Dev -> Master 2.4.0

.editorconfig

@@ -8,7 +8,7 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = space
 
-[*.{md,yml,yaml,html,css,scss,js,cff}]
+[*.{md,yml,yaml,html,css,scss,js}]
 indent_size = 2
 
 # These files are edited and tested upstream in nf-core/modules

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -45,6 +45,6 @@ body:
         * Nextflow version _(eg. 22.10.1)_
         * Hardware _(eg. HPC, Desktop, Cloud)_
         * Executor _(eg. slurm, local, awsbatch)_
-        * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_
+        * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_
         * OS _(eg. CentOS Linux, macOS, Linux Mint)_
         * Version of nf-core/methylseq _(eg. 1.1, 1.5, 1.8.2)_

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,7 +15,8 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/meth
 
 - [ ] This comment contains a description of changes (with reason).
 - [ ] If you've fixed a bug or added code that should be tested, add tests!
-- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/methylseq/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
+- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/methylseq/tree/master/.github/CONTRIBUTING.md)
+- [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
 - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.

.github/workflows/awsfulltest.yml

@@ -17,7 +17,7 @@ jobs:
         aligner: ["bismark", "bismark_hisat", "bwameth"]
     steps:
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v3
+        uses: seqeralabs/action-tower-launch@v1
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}

.github/workflows/awstest.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
       # Launch workflow using Tower CLI tool action
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v3
+        uses: seqeralabs/action-tower-launch@v1
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}

.github/workflows/branch.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Check PRs
         if: github.repository == 'nf-core/methylseq'
         run: |
-          { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/methylseq ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
+          { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/methylseq ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
 
       # If the above check failed, post a comment on the PR explaining the failure
       # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets

.github/workflows/ci.yml

@@ -1,54 +1,71 @@
-name: nf-core CI
 # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+name: nf-core CI
 on:
   push:
     branches:
-      - dev
+      - "dev"
   pull_request:
+    branches:
+      - "dev"
+      - "master"
   release:
-    types: [published]
+    types:
+      - "published"
 
 env:
   NXF_ANSI_LOG: false
+  NFTEST_VER: "0.7.3"
 
+# cancel-in-progress: true ensures that if a new push is made to the same branch while a previous run is still in progress, the previous run is cancelled
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
-  test:
-    name: Run ${{ matrix.aligner }}-${{ matrix.profile }}-${{ matrix.NXF_VER }}
-    # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}"
+  test-dev:
+    # Only run if the pull request is against the dev branch or if the push is to the dev branch
+    if: github.base_ref == 'dev' || (github.event_name == 'push')
+    name: Run pipeline with test data on dev
     runs-on: ubuntu-latest
+    # Add PR write permissions to allow commenting on the PR
+    permissions:
+      pull-requests: write
+    # Set up the matrix of different pipeline versions and profiles to test
     strategy:
+      # Do not fail fast if one of the tests fails
+      fail-fast: false
       matrix:
         NXF_VER:
-          - "22.10.1"
           - "latest-everything"
         aligner:
-          - bismark
-          - bismark_hisat
-          - bwameth
-        profile: ["docker"] # TODO , "singularity", "conda"]
+          - "bismark"
+          - "bismark_hisat"
+          - "bwameth"
+        profile: # TODO ["singularity", "conda"]
+          - "docker"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v3
 
+      - name: Hash Github Workspace
+        id: hash_workspace
+        run: |
+          echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+
       - name: Cache test data
         id: cache-testdata
         uses: actions/cache@v3
         with:
-          path: ./test-datasets/
-          key: methylseq-test-data
+          path: test-datasets/
+          key: ${{ steps.hash_workspace.outputs.digest }}
 
       - name: Check out test data
         if: steps.cache-testdata.outputs.cache-hit != 'true'
         uses: actions/checkout@v3
         with:
           repository: nf-core/test-datasets
           ref: methylseq
-          path: ./test-datasets/
+          path: test-datasets/
 
       - name: Replace remote paths in samplesheets
         run: |
@@ -67,35 +84,90 @@ jobs:
 
       - name: Install nf-test
         run: |
-          wget -qO- https://code.askimed.com/install/nf-test | bash
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
           sudo mv nf-test /usr/local/bin/
 
-      - name: Set up Singularity
-        if: matrix.profile == 'singularity'
-        uses: eWaterCycle/setup-singularity@v5
+      - name: Run nf-test
+        run: |
+          nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap
+
+      # If the test fails, output the software_versions.yml using the 'batcat' utility
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml
+
+  test-master:
+    # Only run if the pull request is against the master branch
+    if: ${{ (github.base_ref == 'master') }}
+    name: Run pipeline with test data on master
+    runs-on: ubuntu-latest
+    # Add PR write permissions to allow commenting on the PR
+    permissions:
+      pull-requests: write
+    # Set up the matrix of different pipeline versions and profiles to test
+    strategy:
+      matrix:
+        NXF_VER:
+          - "latest-everything"
+        aligner:
+          - "bismark"
+          - "bismark_hisat"
+          - "bwameth"
+        profile: # TODO ["singularity", "conda"]
+          - "docker"
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - name: Hash Github Workspace
+        id: hash_workspace
+        run: |
+          echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
+
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: ${{ steps.hash_workspace.outputs.digest }}
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
         with:
-          singularity-version: 3.7.1
+          repository: nf-core/test-datasets
+          ref: methylseq
+          path: test-datasets/
 
-      - name: Set up miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ./test-datasets/samplesheet/*csv; do
+            sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f
+            sed -i 's=https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/=./test-datasets/=g' $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@v1
         with:
-          auto-update-conda: true
-          channels: conda-forge,bioconda,defaults
-          python-version: ${{ matrix.python-version }}
+          version: "${{ matrix.NXF_VER }}"
 
-      - name: Conda clean
-        if: matrix.profile == 'conda'
-        run: conda clean -a
+      - name: Install nf-test
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
+          sudo mv nf-test /usr/local/bin/
 
       - name: Run nf-test
         run: |
-          nf-test test \
-            --profile=test_ci,test,${{ matrix.profile }} \
-            tests/${{ matrix.aligner }}/*.nf.test \
-            --tap=test.tap
+          nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap
 
-      - uses: pcolby/tap-summary@v1
-        with:
-          path: >-
-            test.tap
+      # If the test fails, output the software_versions.yml using the 'batcat' utility
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml

.github/workflows/clean-up.yml

@@ -0,0 +1,24 @@
+name: "Close user-tagged issues and PRs"
+on:
+  schedule:
+    - cron: "0 0 * * 0" # Once a week
+
+jobs:
+  clean-up:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v7
+        with:
+          stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days."
+          stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful."
+          close-issue-message: "This issue was closed because it has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor and then staled for 20 days with no activity."
+          days-before-stale: 30
+          days-before-close: 20
+          days-before-pr-close: -1
+          any-of-labels: "awaiting-changes,awaiting-feedback"
+          exempt-issue-labels: "WIP"
+          exempt-pr-labels: "WIP"
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/fix-linting.yml

@@ -34,9 +34,9 @@ jobs:
         id: prettier_status
         run: |
           if prettier --check ${GITHUB_WORKSPACE}; then
-            echo "name=result::pass" >> $GITHUB_OUTPUT
+            echo "result=pass" >> $GITHUB_OUTPUT
           else
-            echo "name=result::fail" >> $GITHUB_OUTPUT
+            echo "result=fail" >> $GITHUB_OUTPUT
           fi
 
       - name: Run 'prettier --write'

.github/workflows/linting.yml

@@ -78,7 +78,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.7"
+          python-version: "3.8"
           architecture: "x64"
 
       - name: Install dependencies

.github/workflows/linting_comment.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Get PR number
         id: pr_number
-        run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT
+        run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT
 
       - name: Post PR comment
         uses: marocchino/sticky-pull-request-comment@v2

.nf-core.yml

@@ -1,5 +1,8 @@
 # FIXME - CI currently runs for dev and dsl2 branches, remove the latter before merging to dev
-actions_ci: False
-files_exist:
-  - bin/markdown_to_html.py
 repository_type: pipeline
+lint:
+  files_unchanged:
+    - .gitattributes
+  files_exist:
+    - bin/markdown_to_html.py
+  actions_ci: False

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v2.7.1"
+    hooks:
+      - id: prettier

CHANGELOG.md

@@ -1,5 +1,24 @@
 # nf-core/methylseq
 
+## [v2.4.0](https://github.com/nf-core/methylseq/releases/tag/2.4.0) - 2023-06-02
+
+### Pipeline Updates
+
+- Updated template to nf-core/tools v2.8
+- Add `--bamqc_regions_file` parameter for targeted methylation sequencing data #302
+- ✨ Add NF-TEST tests and snapshots for the pipeline test profile #310
+
+### Bug fixes & refactoring
+
+- 🛠️ update index file channels to explicit value channels #310
+- 🐛 fix `params.test_data_base` in test and test_full configs #310
+- 🤖 GitHub Actions CI - pull_reqest to `dev` tests with NXF_VER `latest-everything` #310
+- 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310
+- 🤖 GitHub Actions CI - `fail-fast` set to false #310
+- 🐛 get to the bottom of index tests #278
+- 🐛 fix `ignore_3prime_r2` param #299
+- 🐛 removed unused directory #297
+
 ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16
 
 ### Pipeline Updates