Merge pull request #347 from nf-core/dev

Dev -> Master 2.5.0
nf-core · Oct 18, 2023 · 053e59e · 053e59e
2 parents 81f989c + e02fd8c
commit 053e59e
Show file tree

Hide file tree

Showing 45 changed files with 1,167 additions and 1,072 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -2,6 +2,7 @@
     "name": "nfcore",
     "image": "nfcore/gitpod:latest",
     "remoteUser": "gitpod",
+    "runArgs": ["--privileged"],
 
     // Configure tool-specific properties.
     "customizations": {

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1 @@
+* @nf-core/methylseq
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -9,7 +9,9 @@ Please use the pre-filled template to save time.
 However, don't be put off by this template - other more general issues and suggestions are welcome!
 Contributions to the code are even more welcome ;)
 
-> If you need help using or modifying nf-core/methylseq then the best place to ask is on the nf-core Slack [#methylseq](https://nfcore.slack.com/channels/methylseq) channel ([join our Slack here](https://nf-co.re/join/slack)).
+:::info
+If you need help using or modifying nf-core/methylseq then the best place to ask is on the nf-core Slack [#methylseq](https://nfcore.slack.com/channels/methylseq) channel ([join our Slack here](https://nf-co.re/join/slack)).
+:::
 
 ## Contribution workflow
 
@@ -116,4 +118,3 @@ To get started:
 Devcontainer specs:
 
 - [DevContainer config](.devcontainer/devcontainer.json)
-- [Dockerfile](.devcontainer/Dockerfile)
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -42,7 +42,7 @@ body:
     attributes:
       label: System information
       description: |
-        * Nextflow version _(eg. 22.10.1)_
+        * Nextflow version _(eg. 23.04.0)_
         * Hardware _(eg. HPC, Desktop, Cloud)_
         * Executor _(eg. slurm, local, awsbatch)_
         * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_

diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml
@@ -17,19 +17,24 @@ jobs:
         aligner: ["bismark", "bismark_hisat", "bwameth"]
     steps:
       - name: Launch workflow via tower
-        uses: seqeralabs/action-tower-launch@v1
+        uses: seqeralabs/action-tower-launch@v2
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/methylseq/work-${{ github.sha }}
           parameters: |
             {
-              "aligner": "${{ matrix.aligner }}",
+              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/methylseq/results-${{ github.sha }}/${{matrix.aligner}}"
+              "aligner": "${{ matrix.aligner }}",
             }
-          profiles: test_full,aws_tower
+          profiles: test_full
+
       - uses: actions/upload-artifact@v3
         with:
           name: Tower debug log file
-          path: tower_action_*.log
+          path: |
+            tower_action_*.log
+            tower_action_*.json
diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml
@@ -15,19 +15,23 @@ jobs:
     steps:
       # Launch workflow using Tower CLI tool action
       - name: Launch workflow via tower
-        uses: seqeralabs/action-tower-launch@v1
+        uses: seqeralabs/action-tower-launch@v2
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/methylseq/work-${{ github.sha }}
           parameters: |
             {
               "aligner": "${{ matrix.aligner }}",
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/methylseq/results-test-${{ github.sha }}/${{matrix.aligner}}"
             }
-          profiles: test,aws_tower
+          profiles: test
+
       - uses: actions/upload-artifact@v3
         with:
           name: Tower debug log file
-          path: tower_action_*.log
+          path: |
+            tower_action_*.log
+            tower_action_*.json
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,58 +21,59 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  define_nxf_versions:
+    name: Choose nextflow versions to test against depending on target branch
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.nxf_versions.outputs.matrix }}
+    steps:
+      - id: nxf_versions
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
+            echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT
+          else
+            echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT
+          fi
+
   test:
     name: Run pipeline with test data
+    needs: define_nxf_versions
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        NXF_VER:
-          - "22.10.1"
-          - "latest-everything"
-        aligner:
+        NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }}
+        test_tags:
           - "bismark"
           - "bismark_hisat"
           - "bwameth"
         profile:
           - "docker"
-    steps:
-      # Skip if it's a pull_request to dev and NXF_VER is '22.10.1'
-      - name: Skip condition
-        id: condition
-        run: |
-          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then
-            echo "skip=true" >> $GITHUB_OUTPUT
-          fi
 
+    steps:
       - name: Check out pipeline code
-        if: steps.condition.outputs.skip != 'true'
         uses: actions/checkout@v3
 
       - name: Hash Github Workspace
-        if: steps.condition.outputs.skip != 'true'
         id: hash_workspace
         run: |
           echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT
 
       - name: Cache test data
-        if: steps.condition.outputs.skip != 'true'
         id: cache-testdata
         uses: actions/cache@v3
         with:
           path: test-datasets/
           key: ${{ steps.hash_workspace.outputs.digest }}
 
       - name: Check out test data
-        if: steps.cache-testdata.outputs.cache-hit != 'true' && steps.condition.outputs.skip != 'true'
         uses: actions/checkout@v3
         with:
           repository: nf-core/test-datasets
           ref: methylseq
           path: test-datasets/
 
       - name: Replace remote paths in samplesheets
-        if: steps.condition.outputs.skip != 'true'
         run: |
           for f in ./test-datasets/samplesheet/*csv; do
             sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f
@@ -83,25 +84,27 @@ jobs:
           done;
 
       - name: Install Nextflow
-        if: steps.condition.outputs.skip != 'true'
         uses: nf-core/setup-nextflow@v1
         with:
           version: "${{ matrix.NXF_VER }}"
 
       - name: Install nf-test
-        if: steps.condition.outputs.skip != 'true'
         run: |
           wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER
           sudo mv nf-test /usr/local/bin/
 
       - name: Run nf-test
-        if: steps.condition.outputs.skip != 'true'
         run: |
-          nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap
+          nf-test test --tag ${{ matrix.test_tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml
 
-      # If the test fails, output the software_versions.yml using the 'batcat' utility
       - name: Output log on failure
         if: failure()
         run: |
           sudo apt install bat > /dev/null
           batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -78,7 +78,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.11"
           architecture: "x64"
 
       - name: Install dependencies

diff --git a/.github/workflows/release-announcments.yml b/.github/workflows/release-announcments.yml
@@ -0,0 +1,68 @@
+name: release-announcements
+# Automatic release toot and tweet anouncements
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  toot:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: rzr/fediverse-action@master
+        with:
+          access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }}
+          host: "mstdn.science" # custom host if not "mastodon.social" (default)
+          # GitHub event payload
+          # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#release
+          message: |
+            Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}!
+
+            Please see the changelog: ${{ github.event.release.html_url }}
+
+  send-tweet:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: pip install tweepy==4.14.0
+      - name: Send tweet
+        shell: python
+        run: |
+          import os
+          import tweepy
+
+          client = tweepy.Client(
+              access_token=os.getenv("TWITTER_ACCESS_TOKEN"),
+              access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"),
+              consumer_key=os.getenv("TWITTER_CONSUMER_KEY"),
+              consumer_secret=os.getenv("TWITTER_CONSUMER_SECRET"),
+          )
+          tweet = os.getenv("TWEET")
+          client.create_tweet(text=tweet)
+        env:
+          TWEET: |
+            Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}!
+
+            Please see the changelog: ${{ github.event.release.html_url }}
+          TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }}
+          TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }}
+          TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }}
+          TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }}
+
+  bsky-post:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: zentered/[email protected]
+        with:
+          post: |
+            Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}!
+
+            Please see the changelog: ${{ github.event.release.html_url }}
+        env:
+          BSKY_IDENTIFIER: ${{ secrets.BSKY_IDENTIFIER }}
+          BSKY_PASSWORD: ${{ secrets.BSKY_PASSWORD }}
+          #
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,4 +1,9 @@
 image: nfcore/gitpod:latest
+tasks:
+  - name: Update Nextflow and setup pre-commit
+    command: |
+      pre-commit install --install-hooks
+      nextflow self-update
 
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack

diff --git a/.nf-core.yml b/.nf-core.yml
@@ -1,8 +1,8 @@
-# FIXME - CI currently runs for dev and dsl2 branches, remove the latter before merging to dev
 repository_type: pipeline
 lint:
   files_unchanged:
     - .gitattributes
   files_exist:
     - bin/markdown_to_html.py
   actions_ci: False
+  multiqc_config: False
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,24 @@
 # nf-core/methylseq
 
+## [v2.5.0](https://github.com/nf-core/methylseq/releases/tag/2.5.0) - 2023-10-18
+
+### Pipeline Updates
+
+- 🔄 Updated template to nf-core/tools v2.9
+- 🔄 Updated template to nf-core/tools v2.10
+- 🔧 Updated nf-core modules for FastQC, samtools sort, samtools flagstat
+  - ❌ Removes problematic `-m` memory assignment for samtools sort [#81](https://github.com/nf-core/methylseq/issues/81)
+- 🧾 Use `fromSamplesheet` from nf-validation [#341](https://github.com/nf-core/methylseq/pull/341)
+- 🚀 Update Maintainers and add CODEOWNERS [#345](https://github.com/nf-core/methylseq/pull/345)
+- ⚙️ Update schema to utilize exists and add more patterns [#342](https://github.com/nf-core/methylseq/pull/342)
+- 📁 Support pipeline-specific configs [#343](https://github.com/nf-core/methylseq/pull/343)
+
+### Bug fixes & refactoring
+
+- 🛠️ Added publishing of coverage (`*cov.gz`) files for NOMe-seq filtered reads for `coverage2cytosine`
+- 🛠️ Wrong display values for "zymo" and "em_seq" presets on help page [#335](https://github.com/nf-core/methylseq/pull/335)
+- 📚 Use new Citation tools functions [#336](https://github.com/nf-core/methylseq/issues/336)
+
 ## [v2.4.0](https://github.com/nf-core/methylseq/releases/tag/2.4.0) - 2023-06-02
 
 ### Pipeline Updates

diff --git a/CITATIONS.md b/CITATIONS.md
@@ -16,12 +16,19 @@
 
 - [FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)
 
-  > <https://www.bioinformatics.babraham.ac.uk/projects/fastqc/>
+  > Andrews, S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online].
+
+- [MultiQC](https://pubmed.ncbi.nlm.nih.gov/27312411/)
+
+  > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: 10.1093/bioinformatics/btw354. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924.
 
 - [Trim Galore!](https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/)
 
   > <https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/>
 
+  <!-- TODO https://github.com/FelixKrueger/TrimGalore/issues/25 -->
+  <!-- https://zenodo.org/records/7598955 -->
+
 - [Bismark](https://doi.org/10.1093/bioinformatics/btr167)
 
   > Felix Krueger, Simon R. Andrews, Bismark: a flexible aligner and methylation caller for Bisulfite-Seq applications, Bioinformatics, Volume 27, Issue 11, 1 June 2011, Pages 1571–1572, doi: [10.1093/bioinformatics/btr167](https://doi.org/10.1093/bioinformatics/btr167)
@@ -42,10 +49,6 @@
 
   > Daley, T., Smith, A. Predicting the molecular complexity of sequencing libraries. Nat Methods 10, 325–327 (2013). doi: [10.1038/nmeth.2375](https://doi.org/10.1038/nmeth.2375)
 
-- [MultiQC](http://dx.doi.org/10.1093/bioinformatics/btw354/)
-
-  > Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. doi: [10.1093/bioinformatics/btw354](http://dx.doi.org/10.1093/bioinformatics/btw354/). Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924.
-
 ## Software packaging/containerisation tools
 
 - [Anaconda](https://anaconda.com)
@@ -62,5 +65,8 @@
 
 - [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241)
 
+  > Merkel, D. (2014). Docker: lightweight linux containers for consistent development and deployment. Linux Journal, 2014(239), 2. doi: 10.5555/2600239.2600241.
+
 - [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/)
+
   > Kurtzer GM, Sochat V, Bauer MW. Singularity: Scientific containers for mobility of compute. PLoS One. 2017 May 11;12(5):e0177459. doi: 10.1371/journal.pone.0177459. eCollection 2017. PubMed PMID: 28494014; PubMed Central PMCID: PMC5426675.