Update validator.yml to fix maintenance issues (dependencies, disk space, error throwing)

.github/workflows/check-line-endings.yml

@@ -0,0 +1,44 @@
+
+name: Check for Windows line endings
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+ branches: [ master ]
+
+jobs:
+ check-line-endings:
+ runs-on: ubuntu-latest
+
+ steps:
+
+ - name: Checkout
+ uses: actions/checkout@v2
+ with:
+ # make sure to download directly from the PR's repo, whether that is this repo or a fork
+ # By default github generates a merge commit for each PR in this repo, but only for the one branch under test
+ # but `git-annex` needs access to *two* branches: the current branch and `git-annex`
+ # this might be subtly buggy since it is testing the remote version, not the merged version
+ #
+ # *if* this is not a pull request, this will fall back to its default behaviour.
+ repository: ${{github.event.pull_request.head.repo.full_name}}
+ ref: ${{ github.event.pull_request.head.ref }}
+
+ - name: Install dos2unix
+ run: |
+ # the easy way to do this: convert everything to unix and ask git if that changed anything
+ sudo apt-get install -y dos2unix
+
+ - name: Check line endings
+ run: |
+ # the easy way to do this: convert everything to unix and ask git if that changed anything
+
+ find ./ -path ./.git -prune -o -type f -print0 | xargs -0 dos2unix -q
+ # this version is safer, but more maintenance:
+ #find ./ -path ./.git -prune -o -type f \( ! -name "*.nii" -a ! -name "*.nii.gz" \) -print0 | xargs -0 dos2unix -q
+
+ git diff --stat --exit-code
+ if [ "$?" -ne 0 ]; then
+ echo "error: Windows line endings found."
+ exit 1
+ fi

.github/workflows/lint-participants.yml

@@ -0,0 +1,28 @@
+
+name: Lint participants.tsv file
+on:
+ push:
+ branches: [ master ]
+ pull_request:
+ branches: [ master ]
+
+jobs:
+ lint-participants:
+ runs-on: ubuntu-latest
+
+ steps:
+
+ - name: Checkout
+ uses: actions/checkout@v2
+ with:
+ # make sure to download directly from the PR's repo, whether that is this repo or a fork
+ # By default github generates a merge commit for each PR in this repo, but only for the one branch under test
+ # but `git-annex` needs access to *two* branches: the current branch and `git-annex`
+ # this might be subtly buggy since it is testing the remote version, not the merged version
+ #
+ # *if* this is not a pull request, this will fall back to its default behaviour.
+ repository: ${{github.event.pull_request.head.repo.full_name}}
+ ref: ${{ github.event.pull_request.head.ref }}
+
+ - name: Lint participants.tsv
+ run: .github/workflows/lint-tsv participants.tsv

.github/workflows/validator.yml

@@ -12,8 +12,18 @@ jobs:
 
  steps:
 
+ - name: Gain disk space by remapping temp disk (`/dev/sdb1`)
+ uses: easimon/maximize-build-space@master
+ # We use the above action instead of removing unwanted packages because it only takes about 2s;
+ # by comparison, removing lots of small file is very time-consuming.
+ with:
+ # After freeing up space we could have as much as 87GB available, which is overkill.
+ # So, we set root == 10GB to save room for installing dependencies (leaving us with ~77GB)
+ root-reserve-mb: 10240
+ swap-size-mb: 1024
+
  - name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
  with:
  # make sure to download directly from the PR's repo, whether that is this repo or a fork
  # By default github generates a merge commit for each PR in this repo, but only for the one branch under test
@@ -24,63 +34,33 @@ jobs:
  repository: ${{github.event.pull_request.head.repo.full_name}}
  ref: ${{ github.event.pull_request.head.ref }}
 
- - name: Install dos2unix
- run: |
- # the easy way to do this: convert everything to unix and ask git if that changed anything
- sudo apt-get install -y dos2unix
-
- - name: Check line endings
+ - name: Update apt packages
  run: |
- # the easy way to do this: convert everything to unix and ask git if that changed anything
-
- find ./ -path ./.git -prune -o -type f -print0 | xargs -0 dos2unix -q
- # this version is safer, but more maintenance:
- #find ./ -path ./.git -prune -o -type f \( ! -name "*.nii" -a ! -name "*.nii.gz" \) -print0 | xargs -0 dos2unix -q
-
- git diff --stat --exit-code
- if [ "$?" -ne 0 ]; then
- echo "error: Windows line endings found."
- exit 1
- fi
-
- - name: Lint participants.tsv
- run: .github/workflows/lint-tsv participants.tsv
-
- #- name: Update software
- # run: |
- # # do we want to do this? it's helpful to avoid testing against surprise out-of-date software, but also so slow.
- # sudo apt-get update &&
- # sudo DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade &&
- # sudo DEBIAN_FRONTEND=noninteractive apt-get autoremove
+ # do we want to do this? it's helpful to avoid testing against surprise out-of-date software, but also so slow.
+ sudo apt-get update &&
+ sudo DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade &&
+ sudo DEBIAN_FRONTEND=noninteractive apt-get autoremove
 
  - name: Install git-annex
  run: |
  sudo apt-get install -y git-annex
  git config --global annex.thin true
+ git config --global annex.retry 2
+
+ - name: Setup NodeJS (for bids-validator)
+ uses: actions/setup-node@v4
 
  - name: Install bids-validator
- run: |
- # install proper NodeJS for bids-validator
- curl -sL https://deb.nodesource.com/setup_current.x | sudo -E bash -
- sudo apt-get install -y nodejs
- sudo npm install -g bids-validator
+ run: sudo npm install -g bids-validator
 
  - name: Set up Python
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v5
  with:
- python-version: 3.7
+ python-version: 3.11
 
  - name: Install spine-generic for acquisition parameters check
  run: pip install spinegeneric@git+https:/spine-generic/spine-generic.git@master
 
- #- name: Increase free space
- # run: |
- # # this takes about 2 minutes but saves about 30GB, which is space we might need for git-annex.
- # # annex.thin saves a lot of space, but if the dataset grows beyond what Github can handle
- # # try enabling this.
- # # ref: https:/actions/virtual-environments/issues/2606#issuecomment-772683150
- # sudo rm -rf /usr/local/lib/android /usr/share/dotnet
-
  - name: git config
  run: |
  # this is needed by git-annex so that it can write to the local git-annex branch
@@ -93,13 +73,29 @@ jobs:
 
  - name: Download dataset
  run: |
- git fetch --depth=1 origin git-annex:git-annex # actions/checkout@v2 does a shallow checkout, so it is missing this important branch
+ git fetch --depth=1 origin git-annex:git-annex # actions/checkout@v4 does a shallow checkout, so it is missing this important branch
  git annex init
  git annex get -J8
 
  - name: Checking BIDS compliance
- run: bids-validator --verbose ./
+ if: always()
+ run: |
+ bids-validator --verbose ./ | tee bids.txt
+ printf "\nShort summary:\n\n"
+ # FIXME: This should be replaced with bids-validator settings: A) throw error on warning or B) warning ignores
+ ! grep "\[WARN\]\|\[ERR\]" bids.txt # ! -> Reverse error code so "found == error thrown"
+
  - name: Checking acquisition parameters
- run: sg_params_checker -path-in ./
+ if: always()
+ run: |
+ sg_params_checker -path-in ./ | tee params.txt
+ # FIXME: Checking the output like this should be replaced with SG throwing actual errors
+ ! grep "WARNING" params.txt # ! -> Reverse error code so "found == error thrown"
+
  - name: Checking data consistency
- run: sg_check_data_consistency -path-in ./
+ if: always()
+ run: |
+ sg_check_data_consistency -path-in ./ | tee consistency.txt
+ # FIXME: Checking the output like this should be replaced with SG throwing actual errors
+ ! grep "Warning\|Missing" consistency.txt
+ grep "all good" consistency.txt