From aff68535221c1f918ff49d14a6736859dd6cf195 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 24 Apr 2025 11:48:22 +0200 Subject: [PATCH 01/12] template: downgrade nf-schema to fix CI tests --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index d58ca72196..c7d99512cc 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -295,7 +295,7 @@ manifest { {% if nf_schema -%} // Nextflow plugins plugins { - id 'nf-schema@2.3.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-schema@2.2.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet } validation { From 1a2b643d6ecefe10705067088ff51ad9d383f3f8 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 24 Apr 2025 09:49:34 +0000 Subject: [PATCH 02/12] [automated] Update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7527b374cc..c51417d430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # nf-core/tools: Changelog +## v3.2.1dev + +### Template + +- Run awsfulltest after release, and with dev revision on PRs to master ([#3485](https://github.com/nf-core/tools/pull/3485)) +- downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) + ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] ### Template From 49280effef36e2b5915eee67c12a4bb50a04b17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 7 Mar 2025 10:31:22 +0100 Subject: [PATCH 03/12] run awsfulltest after release, and with dev revision on PRs to master --- .../.github/workflows/awsfulltest.yml | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 6805c83a27..ea41ad77fe 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -4,19 +4,20 @@ name: nf-core AWS full size tests # It runs the -profile 'test_full' on AWS batch on: - pull_request: - branches: - - main - - master workflow_dispatch: pull_request_review: types: [submitted] + branches: + - main + - master + release: + types: [published] jobs: run-platform: name: Run AWS full tests - # run only if the PR is approved by at least 2 reviewers and against the master branch or manually triggered - if: github.repository == '{{ name }}' && github.event.review.state == 'approved' && github.event.pull_request.base.ref == 'master' || github.event_name == 'workflow_dispatch' + # run only if the PR is approved by at least 2 reviewers and against the master/main branch or manually triggered + if: github.repository == '{{ name }}' && github.event.review.state == 'approved' && (github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - name: Get PR reviews @@ -43,6 +44,11 @@ jobs: CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required + - name: Set revision variable + id: revision + run: | + echo "revision=${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && github.sha || 'dev' }}" >> "$GITHUB_OUTPUT" + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required @@ -52,12 +58,12 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - revision: ${{ github.sha }} - workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} + revision: ${{ steps.revision.outputs.revision }} + workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ steps.revision.outputs.revision }}{% endraw %} parameters: | { "hook_url": "{% raw %}${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}{% endraw %}", - "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" + "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ steps.revision.outputs.revision }}{% endraw %}" } profiles: test_full From 15cf1f48f69db97ca17b5abd41f817caf256d926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 7 Mar 2025 10:37:52 +0100 Subject: [PATCH 04/12] fix jinja template --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index ea41ad77fe..a6c5e47b88 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -40,7 +40,7 @@ jobs: id: test_variables if: github.event_name != 'workflow_dispatch' run: | - JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}'{% endraw %} + JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}' CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required @@ -53,7 +53,7 @@ jobs: uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) - # on the `test_full.config` test runs with only one set of parameters {%- raw %} + # on the `test_full.config` test runs with only one set of parameters with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} From c761c0ed498fb8320e598726754ef6dc444e23d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 7 Mar 2025 10:39:44 +0100 Subject: [PATCH 05/12] update linting of actions_awsfulltest --- nf_core/pipelines/lint/actions_awsfulltest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/lint/actions_awsfulltest.py b/nf_core/pipelines/lint/actions_awsfulltest.py index 080ae3583e..828646df01 100644 --- a/nf_core/pipelines/lint/actions_awsfulltest.py +++ b/nf_core/pipelines/lint/actions_awsfulltest.py @@ -42,12 +42,14 @@ def actions_awsfulltest(self) -> Dict[str, List[str]]: # Check that the action is only turned on for published releases try: - if wf[True]["pull_request"]["branches"] != ["main", "master"]: + if wf[True]["pull_request_review"]["branches"] != ["main", "master"]: raise AssertionError() if wf[True]["pull_request_review"]["types"] != ["submitted"]: raise AssertionError() if "workflow_dispatch" not in wf[True]: raise AssertionError() + if wf[True]["release"]["types"] != ["published"]: + raise AssertionError() except (AssertionError, KeyError, TypeError): failed.append("`.github/workflows/awsfulltest.yml` is not triggered correctly") else: From 660d734bb8613637d2d2052d5362d354db881a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 12 Mar 2025 13:01:46 +0100 Subject: [PATCH 06/12] remove not available branches on pull_request_review --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 3 --- nf_core/pipelines/lint/actions_awsfulltest.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index a6c5e47b88..81573b0799 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -7,9 +7,6 @@ on: workflow_dispatch: pull_request_review: types: [submitted] - branches: - - main - - master release: types: [published] diff --git a/nf_core/pipelines/lint/actions_awsfulltest.py b/nf_core/pipelines/lint/actions_awsfulltest.py index 828646df01..07142945f2 100644 --- a/nf_core/pipelines/lint/actions_awsfulltest.py +++ b/nf_core/pipelines/lint/actions_awsfulltest.py @@ -42,8 +42,6 @@ def actions_awsfulltest(self) -> Dict[str, List[str]]: # Check that the action is only turned on for published releases try: - if wf[True]["pull_request_review"]["branches"] != ["main", "master"]: - raise AssertionError() if wf[True]["pull_request_review"]["types"] != ["submitted"]: raise AssertionError() if "workflow_dispatch" not in wf[True]: From 87c1324cbe869e6099f16ebc178f9c8883ccb23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 12 Mar 2025 13:09:49 +0100 Subject: [PATCH 07/12] remove checking number of approvals --- .../.github/workflows/awsfulltest.yml | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 81573b0799..8ebe18b401 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -17,30 +17,6 @@ jobs: if: github.repository == '{{ name }}' && github.event.review.state == 'approved' && (github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - name: Get PR reviews - uses: octokit/request-action@v2.x - if: github.event_name != 'workflow_dispatch' - id: check_approvals - continue-on-error: true - with: - route: GET /repos/{%- raw -%}${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews?per_page=100 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Check for approvals - if: ${{ failure() && github.event_name != 'workflow_dispatch' }} - run: | - echo "No review approvals found. At least 2 approvals are required to run this action automatically." - exit 1 - - - name: Check for enough approvals (>=2) - id: test_variables - if: github.event_name != 'workflow_dispatch' - run: | - JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}' - CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') - test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required - - name: Set revision variable id: revision run: | From aac57cbfa948130b0d7c4f1a09a98537aacf50cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 12 Mar 2025 13:14:02 +0100 Subject: [PATCH 08/12] adding jinja template raw --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 8ebe18b401..c514bf0c6b 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -20,7 +20,7 @@ jobs: - name: Set revision variable id: revision run: | - echo "revision=${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && github.sha || 'dev' }}" >> "$GITHUB_OUTPUT" + echo "revision={%- raw -%}${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && github.sha || 'dev' }}" >> "$GITHUB_OUTPUT" - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 From 4093d36cc2119a2e6cfa2a30024eb6c62060ac5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 25 Apr 2025 14:15:35 +0200 Subject: [PATCH 09/12] fix nf-test-tests branch --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 4c1c620adb..2d86160d73 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -30,7 +30,7 @@ GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" -GITLAB_NFTEST_BRANCH = "nf-test-tests-self-hosted-runners" +GITLAB_NFTEST_BRANCH = "nf-test-tests-old" def with_temporary_folder(func: Callable[..., Any]) -> Callable[..., Any]: From 4c91928a0544d96fb739aedd54f2a7981542045c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 29 Apr 2025 12:24:40 +0200 Subject: [PATCH 10/12] bump version to 3.2.1 --- CHANGELOG.md | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c51417d430..c95f5308a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ # nf-core/tools: Changelog -## v3.2.1dev +## [v3.2.1 - Pewter Pangolin Patch](https://github.com/nf-core/tools/releases/tag/3.2.1) - [2025-04-29] ### Template -- Run awsfulltest after release, and with dev revision on PRs to master ([#3485](https://github.com/nf-core/tools/pull/3485)) -- downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) +- Run awsfulltest after release, and with dev revision on PRs to master/main ([#3485](https://github.com/nf-core/tools/pull/3485)) +- Downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] diff --git a/setup.py b/setup.py index d426478786..5b03adcf87 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.2.0" +version = "3.2.1" with open("README.md") as f: readme = f.read() From 32a8d21e24e925fe67ec565af44eed63e94f8d8a Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 29 Apr 2025 12:24:40 +0200 Subject: [PATCH 11/12] bump version to 3.2.1 --- CHANGELOG.md | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c51417d430..c95f5308a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,11 @@ # nf-core/tools: Changelog -## v3.2.1dev +## [v3.2.1 - Pewter Pangolin Patch](https://github.com/nf-core/tools/releases/tag/3.2.1) - [2025-04-29] ### Template -- Run awsfulltest after release, and with dev revision on PRs to master ([#3485](https://github.com/nf-core/tools/pull/3485)) -- downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) +- Run awsfulltest after release, and with dev revision on PRs to master/main ([#3485](https://github.com/nf-core/tools/pull/3485)) +- Downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] diff --git a/setup.py b/setup.py index d426478786..5b03adcf87 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.2.0" +version = "3.2.1" with open("README.md") as f: readme = f.read() From 511126a4c91a8d6e70ee3ae62ea65421572a378d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 29 Apr 2025 16:15:03 +0200 Subject: [PATCH 12/12] fail ci.yml gracefully on latest-everything --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/workflows/ci.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c95f5308a3..6ea219d4ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Run awsfulltest after release, and with dev revision on PRs to master/main ([#3485](https://github.com/nf-core/tools/pull/3485)) - Downgrade nf-schema to fix CI tests ([#3544](https://github.com/nf-core/tools/pull/3544)) +- Fail nextflow run test gracefully for `latest everything` ([#3543](https://github.com/nf-core/tools/pull/3543)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index f3a1073843..abb8680f0a 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -83,5 +83,6 @@ jobs: uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}" + continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }} run: | nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results{% endraw %}