From c0af8b2ba6a3782d26acbf9d5ba8a8d003eb0640 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 14 Oct 2024 23:40:47 +1100 Subject: [PATCH 01/21] Fixed dailies.yml --- .github/workflows/dailies.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 459246091..dcb24d3a2 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -19,6 +19,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + python-versions: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' run-daily-dev-tests: uses: ./.github/workflows/dev-testing.yml From 27e1791d2b4231cfcb76e0497f1e5356be8e9bdc Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 14 Oct 2024 23:46:29 +1100 Subject: [PATCH 02/21] Fixed dailies.yml --- .github/workflows/dailies.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index dcb24d3a2..072df090c 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -30,11 +30,17 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + with: + python-versions: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' run-daily-live-service-testing: uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + with: + python-versions: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' run-daily-neuroconv-docker-testing: uses: ./.github/workflows/neuroconv_docker_testing.yml From eedbe9120b084d1304c335935ab292efbafa4b4d Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 14 Oct 2024 23:49:25 +1100 Subject: [PATCH 03/21] Fixed dailies.yml --- .github/workflows/dailies.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 072df090c..96b0b657c 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -32,7 +32,6 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} with: python-versions: '["3.9", "3.10", "3.11", "3.12"]' - os-versions: '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' run-daily-live-service-testing: uses: ./.github/workflows/live-service-testing.yml From 793cd5681bf65c87dc8bf3199d073e11999840cd Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Tue, 15 Oct 2024 18:39:34 +1100 Subject: [PATCH 04/21] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d748d79e6..c369cefa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Deprecations ## Bug Fixes +* Fixed dailies [PR #1113](https://github.com/catalystneuro/neuroconv/pull/1113) ## Features * Using in-house `GenericDataChunkIterator` [PR #1068](https://github.com/catalystneuro/neuroconv/pull/1068) From 977144edbd2d0412b7a47fbc52e916376311d579 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Tue, 15 Oct 2024 18:57:03 +1100 Subject: [PATCH 05/21] debugging: using v0.6.4 for docker tag --- .github/workflows/neuroconv_docker_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/neuroconv_docker_testing.yml b/.github/workflows/neuroconv_docker_testing.yml index c86a83d80..dd5829591 100644 --- a/.github/workflows/neuroconv_docker_testing.yml +++ b/.github/workflows/neuroconv_docker_testing.yml @@ -81,7 +81,7 @@ jobs: - name: Pull docker image run: | - docker pull ghcr.io/catalystneuro/neuroconv:latest + docker pull ghcr.io/catalystneuro/neuroconv:v0.6.4 docker pull ghcr.io/catalystneuro/neuroconv:yaml_variable - name: Run docker tests From 6a8bc86210582a652e46907ea6ad20b5f5ca0a6b Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 21 Oct 2024 16:40:52 -0700 Subject: [PATCH 06/21] switched back to latest --- .github/workflows/neuroconv_docker_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/neuroconv_docker_testing.yml b/.github/workflows/neuroconv_docker_testing.yml index dd5829591..c86a83d80 100644 --- a/.github/workflows/neuroconv_docker_testing.yml +++ b/.github/workflows/neuroconv_docker_testing.yml @@ -81,7 +81,7 @@ jobs: - name: Pull docker image run: | - docker pull ghcr.io/catalystneuro/neuroconv:v0.6.4 + docker pull ghcr.io/catalystneuro/neuroconv:latest docker pull ghcr.io/catalystneuro/neuroconv:yaml_variable - name: Run docker tests From eefe6db6455db8d0a663ed683009305a7544ec8e Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 07:24:17 -0700 Subject: [PATCH 07/21] updated python-versions and os-versions to use a default var --- .github/workflows/dailies.yml | 8 -------- .github/workflows/deploy-tests.yml | 12 ++++++------ .github/workflows/dev-testing.yml | 6 +++--- .github/workflows/doctests.yml | 12 ++++++------ .github/workflows/live-service-testing.yml | 12 ++++++------ .github/workflows/testing.yml | 12 ++++++------ 6 files changed, 27 insertions(+), 35 deletions(-) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 96b0b657c..459246091 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -19,9 +19,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - python-versions: '["3.9", "3.10", "3.11", "3.12"]' - os-versions: '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' run-daily-dev-tests: uses: ./.github/workflows/dev-testing.yml @@ -30,16 +27,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} - with: - python-versions: '["3.9", "3.10", "3.11", "3.12"]' run-daily-live-service-testing: uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} - with: - python-versions: '["3.9", "3.10", "3.11", "3.12"]' - os-versions: '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' run-daily-neuroconv-docker-testing: uses: ./.github/workflows/neuroconv_docker_testing.yml diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 4f67d15de..bef4df981 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -40,8 +40,8 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '' }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: @@ -49,8 +49,8 @@ jobs: if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '' }} run-live-service-tests: @@ -60,8 +60,8 @@ jobs: secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9", "3.10", "3.11", "3.12"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest", "macos-latest", "macos-13", "windows-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '' }} check-final-status: diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 31b5329a8..7d87bb8d9 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -5,9 +5,9 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["3.9", "3.10", "3.11", "3.12"]' + default: ${{ vars.ALL_PYTHON_VERSIONS }} secrets: DANDI_API_KEY: required: true @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(inputs.python-versions) }} + python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index e492eda0c..c95052ede 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -4,14 +4,14 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["3.9", "3.10", "3.11", "3.12"]' + default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + default: ${{ vars.ALL_OS_VERSIONS }} jobs: @@ -21,8 +21,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(inputs.python-versions) }} - os: ${{ fromJson(inputs.os-versions) }} + python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} + os: ${{ fromJson(!inputs.os-versions && vars.ALL_OS_VERSIONS || inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index b9a425a8d..442dcc734 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -5,14 +5,14 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["3.9", "3.10", "3.11", "3.12"]' + default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + default: ${{ vars.ALL_OS_VERSIONS }} secrets: DANDI_API_KEY: @@ -28,8 +28,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(inputs.python-versions) }} - os: ${{ fromJson(inputs.os-versions) }} + python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} + os: ${{ fromJson(!inputs.os-versions && vars.ALL_OS_VERSIONS || inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 06de82c4c..06b2a5503 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,14 +5,14 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["3.9", "3.10", "3.11", "3.12"]' + default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: true + required: false type: string - default: '["ubuntu-latest", "macos-latest", "windows-latest"]' + default: ${{ vars.ALL_OS_VERSIONS }} secrets: @@ -33,8 +33,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(inputs.python-versions) }} - os: ${{ fromJson(inputs.os-versions) }} + python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} + os: ${{ fromJson(!inputs.os-versions && vars.ALL_OS_VERSIONS || inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags From a76fc484e1ab621f7acff859efe7c220ce520985 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 07:30:45 -0700 Subject: [PATCH 08/21] simplified defaults --- .github/workflows/deploy-tests.yml | 12 ++++++------ .github/workflows/dev-testing.yml | 2 +- .github/workflows/doctests.yml | 4 ++-- .github/workflows/live-service-testing.yml | 4 ++-- .github/workflows/testing.yml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index bef4df981..422f6db15 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -40,8 +40,8 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: @@ -49,8 +49,8 @@ jobs: if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} run-live-service-tests: @@ -60,8 +60,8 @@ jobs: secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} check-final-status: diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index 7d87bb8d9..f5efa05dc 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} + python-version: ${{ fromJson(inputs.python-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index c95052ede..27938b2d3 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -21,8 +21,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} - os: ${{ fromJson(!inputs.os-versions && vars.ALL_OS_VERSIONS || inputs.os-versions) }} + python-version: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index 442dcc734..cc07df270 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -28,8 +28,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} - os: ${{ fromJson(!inputs.os-versions && vars.ALL_OS_VERSIONS || inputs.os-versions) }} + python-version: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 06b2a5503..bd458fa29 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -33,8 +33,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ${{ fromJson(!inputs.python-versions && vars.ALL_PYTHON_VERSIONS || inputs.python-versions) }} - os: ${{ fromJson(!inputs.os-versions && vars.ALL_OS_VERSIONS || inputs.os-versions) }} + python-version: ${{ fromJson(inputs.python-versions) }} + os: ${{ fromJson(inputs.os-versions) }} steps: - uses: actions/checkout@v4 - run: git fetch --prune --unshallow --tags From 90e083b9fc4b1a4bbbac22bf186e42643ec39f88 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 07:35:41 -0700 Subject: [PATCH 09/21] debugging required=true --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bd458fa29..c9fc8c8c8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,12 +5,12 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: false + required: true type: string default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: false + required: true type: string default: ${{ vars.ALL_OS_VERSIONS }} From cb74e001c929e33e5ad9893cad302ce31ff98738 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 07:38:59 -0700 Subject: [PATCH 10/21] debugging: manually provide python-verison and os-version --- .github/workflows/deploy-tests.yml | 4 ++-- .github/workflows/testing.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 422f6db15..2715bac6d 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -40,8 +40,8 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9"]' }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest"]' }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c9fc8c8c8..bd458fa29 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,12 +5,12 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: true + required: false type: string default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: true + required: false type: string default: ${{ vars.ALL_OS_VERSIONS }} From d0977f95c59677bc0c4ef706edf90536dad10e8b Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 07:49:38 -0700 Subject: [PATCH 11/21] debugging --- .github/workflows/deploy-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 2715bac6d..c6de3bc11 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -30,6 +30,14 @@ jobs: echo "CHANGELOG.md has not been updated." 0 + debug: + needs: assess-file-changes + if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} + runs-on: ubuntu-latest + steps: + - run: echo $ALL_PYTHON_VERSIONS + - run: echo $ALL_OS_VERSIONS + run-tests: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} From 5c86564524de6c91292555d9b579c986cb9d90d1 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 08:05:31 -0700 Subject: [PATCH 12/21] debugging --- .github/workflows/deploy-tests.yml | 8 -------- .github/workflows/testing.yml | 9 +++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index c6de3bc11..2715bac6d 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -30,14 +30,6 @@ jobs: echo "CHANGELOG.md has not been updated." 0 - debug: - needs: assess-file-changes - if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} - runs-on: ubuntu-latest - steps: - - run: echo $ALL_PYTHON_VERSIONS - - run: echo $ALL_OS_VERSIONS - run-tests: needs: assess-file-changes if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bd458fa29..6e08e1e0b 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -27,6 +27,15 @@ on: workflow_dispatch: jobs: + debug: + name: Debugging + runs-on: ubuntu-latest + steps: + - run: | + echo "Python versions: ${{ inputs.python-versions }}" + echo "OS versions: ${{ inputs.os-versions }}" + echo "parsed python-version: ${{ fromJson(inputs.python-versions) }}" + echo "parsed os-version: ${{ fromJson(inputs.os-versions) }}" run: name: ${{ matrix.os }} Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} From c5eb431a623c4135fc3965a22662ff9ac3d8569c Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 08:11:44 -0700 Subject: [PATCH 13/21] debugging --- .github/workflows/dailies.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 459246091..11b91967a 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -19,6 +19,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + python-versions: '["3.9", "3.10", "3.11", "3.12"]' + os-versions: '["ubuntu-latest", "macos-latest", "windows-latest"]' run-daily-dev-tests: uses: ./.github/workflows/dev-testing.yml From 9b0bdfd7ec884508280ffb43c33d730ff5304549 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 08:15:24 -0700 Subject: [PATCH 14/21] debugging --- .github/workflows/dailies.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 11b91967a..c246d7feb 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -19,9 +19,9 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - python-versions: '["3.9", "3.10", "3.11", "3.12"]' - os-versions: '["ubuntu-latest", "macos-latest", "windows-latest"]' + # with: + # python-versions: '["3.9", "3.10", "3.11", "3.12"]' + # os-versions: '["ubuntu-latest", "macos-latest", "windows-latest"]' run-daily-dev-tests: uses: ./.github/workflows/dev-testing.yml From 8f063ca167749d2b23aa5a78b31c930ddc5f0a2d Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 08:17:40 -0700 Subject: [PATCH 15/21] removed debugging --- .github/workflows/dailies.yml | 3 --- .github/workflows/testing.yml | 9 --------- 2 files changed, 12 deletions(-) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index c246d7feb..459246091 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -19,9 +19,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - # with: - # python-versions: '["3.9", "3.10", "3.11", "3.12"]' - # os-versions: '["ubuntu-latest", "macos-latest", "windows-latest"]' run-daily-dev-tests: uses: ./.github/workflows/dev-testing.yml diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6e08e1e0b..bd458fa29 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -27,15 +27,6 @@ on: workflow_dispatch: jobs: - debug: - name: Debugging - runs-on: ubuntu-latest - steps: - - run: | - echo "Python versions: ${{ inputs.python-versions }}" - echo "OS versions: ${{ inputs.os-versions }}" - echo "parsed python-version: ${{ fromJson(inputs.python-versions) }}" - echo "parsed os-version: ${{ fromJson(inputs.os-versions) }}" run: name: ${{ matrix.os }} Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} From 94b7e496f46090eaff44cc5494c25475baf8b1a5 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Tue, 22 Oct 2024 08:21:38 -0700 Subject: [PATCH 16/21] removed debugging --- .github/workflows/deploy-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 2715bac6d..422f6db15 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -40,8 +40,8 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || '["3.9"]' }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || '["ubuntu-latest"]' }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: From ccd4b9d777c49a2af7316dda37c2e7e6b0178ef1 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Oct 2024 14:10:33 -0700 Subject: [PATCH 17/21] switched deploy-tests to .txt based python versions --- .github/workflows/all_os_versions.txt | 1 + .github/workflows/all_python_versions.txt | 1 + .github/workflows/deploy-tests.yml | 28 +++++++++++++++-------- 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/all_os_versions.txt create mode 100644 .github/workflows/all_python_versions.txt diff --git a/.github/workflows/all_os_versions.txt b/.github/workflows/all_os_versions.txt new file mode 100644 index 000000000..93d2aa0a7 --- /dev/null +++ b/.github/workflows/all_os_versions.txt @@ -0,0 +1 @@ +["ubuntu-latest", "macos-latest", "windows-latest", "macos-13"] diff --git a/.github/workflows/all_python_versions.txt b/.github/workflows/all_python_versions.txt new file mode 100644 index 000000000..7d1db34cd --- /dev/null +++ b/.github/workflows/all_python_versions.txt @@ -0,0 +1 @@ +["3.9", "3.10", "3.11", "3.12"] diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 422f6db15..04a5694a8 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -13,6 +13,16 @@ concurrency: cancel-in-progress: true jobs: + load_python_and_os_versions: + runs-on: ubuntu-latest + outputs: + ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} + ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} + steps: + - id: load_python_versions + run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" + - id: load_os_versions + run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT" assess-file-changes: uses: ./.github/workflows/assess-file-changes.yml @@ -31,7 +41,7 @@ jobs: 0 run-tests: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/testing.yml secrets: @@ -40,28 +50,28 @@ jobs: S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} # If the conversion gallery is the only thing that changed, run doctests only run-doctests-only: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.CONVERSION_GALLERY_CHANGED == 'true' && needs.assess-file-changes.outputs.SOURCE_CHANGED != 'true' }} uses: ./.github/workflows/doctests.yml with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-live-service-tests: - needs: assess-file-changes + needs: [assess-file-changes, load_python_and_os_versions] if: ${{ needs.assess-file-changes.outputs.SOURCE_CHANGED == 'true' }} uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} with: # Ternary operator: condition && value_if_true || value_if_false - python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || vars.ALL_PYTHON_VERSIONS }} - os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || vars.ALL_OS_VERSIONS }} + python-versions: ${{ github.event.pull_request.draft == true && '["3.9"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} check-final-status: From e2808dae13d39833002e512ceef2ccd4d94c35fa Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Oct 2024 14:14:06 -0700 Subject: [PATCH 18/21] debug --- .github/workflows/deploy-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 04a5694a8..6e95cf495 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -19,6 +19,8 @@ jobs: ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} steps: + - id: debug + run: ls - id: load_python_versions run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" - id: load_os_versions From 89b7aac6b5b569b4f10242095bf7241f452cc8f4 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Oct 2024 14:17:06 -0700 Subject: [PATCH 19/21] added checkout --- .github/workflows/deploy-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-tests.yml b/.github/workflows/deploy-tests.yml index 6e95cf495..a18fe8310 100644 --- a/.github/workflows/deploy-tests.yml +++ b/.github/workflows/deploy-tests.yml @@ -19,8 +19,7 @@ jobs: ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} steps: - - id: debug - run: ls + - uses: actions/checkout@v4 - id: load_python_versions run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" - id: load_os_versions From af2d72040dbafde564483b7937894d4b9e00fc9f Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Oct 2024 14:31:31 -0700 Subject: [PATCH 20/21] updated dailies --- .github/workflows/dailies.yml | 23 ++++++++++++++++++++++ .github/workflows/dev-testing.yml | 3 +-- .github/workflows/live-service-testing.yml | 6 ++---- .github/workflows/testing.yml | 7 ++----- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dailies.yml b/.github/workflows/dailies.yml index 459246091..136590652 100644 --- a/.github/workflows/dailies.yml +++ b/.github/workflows/dailies.yml @@ -6,6 +6,18 @@ on: - cron: "0 4 * * *" # Daily at 8PM PST, 11PM EST, 5AM CET to avoid working hours jobs: + load_python_and_os_versions: + runs-on: ubuntu-latest + outputs: + ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }} + ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }} + steps: + - uses: actions/checkout@v4 + - id: load_python_versions + run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT" + - id: load_os_versions + run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT" + build-and-upload-docker-image-dev: uses: ./.github/workflows/build_and_upload_docker_image_dev.yml secrets: @@ -13,25 +25,36 @@ jobs: DOCKER_UPLOADER_PASSWORD: ${{ secrets.DOCKER_UPLOADER_PASSWORD }} run-daily-tests: + needs: load_python_and_os_versions uses: ./.github/workflows/testing.yml secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-daily-dev-tests: + needs: load_python_and_os_versions uses: ./.github/workflows/dev-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} S3_GIN_BUCKET: ${{ secrets.S3_GIN_BUCKET }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} run-daily-live-service-testing: + needs: load_python_and_os_versions uses: ./.github/workflows/live-service-testing.yml secrets: DANDI_API_KEY: ${{ secrets.DANDI_API_KEY }} + with: + python-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }} + os-versions: ${{ needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }} run-daily-neuroconv-docker-testing: uses: ./.github/workflows/neuroconv_docker_testing.yml diff --git a/.github/workflows/dev-testing.yml b/.github/workflows/dev-testing.yml index f5efa05dc..65c011653 100644 --- a/.github/workflows/dev-testing.yml +++ b/.github/workflows/dev-testing.yml @@ -5,9 +5,8 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_PYTHON_VERSIONS }} secrets: DANDI_API_KEY: required: true diff --git a/.github/workflows/live-service-testing.yml b/.github/workflows/live-service-testing.yml index cc07df270..24eda7bc3 100644 --- a/.github/workflows/live-service-testing.yml +++ b/.github/workflows/live-service-testing.yml @@ -5,14 +5,12 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_OS_VERSIONS }} secrets: DANDI_API_KEY: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bd458fa29..d8c5bb9fd 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,15 +5,12 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_OS_VERSIONS }} - secrets: AWS_ACCESS_KEY_ID: From 5ae0f9faee64b768add30bb9054864a36eea5e30 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Wed, 23 Oct 2024 14:38:43 -0700 Subject: [PATCH 21/21] updated doctests --- .github/workflows/doctests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/doctests.yml b/.github/workflows/doctests.yml index 27938b2d3..5d96bf4a3 100644 --- a/.github/workflows/doctests.yml +++ b/.github/workflows/doctests.yml @@ -4,14 +4,12 @@ on: inputs: python-versions: description: 'List of Python versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_PYTHON_VERSIONS }} os-versions: description: 'List of OS versions to use in matrix, as JSON string' - required: false + required: true type: string - default: ${{ vars.ALL_OS_VERSIONS }} jobs: