From 0bb0760af5caa1650e42421fcea74306ac130f1d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 17 Dec 2022 01:41:52 +0100 Subject: [PATCH 01/58] Bump to v2.4.0dev --- CHANGELOG.md | 10 ++++++++++ nextflow.config | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6149abd9..58c4afb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # nf-core/methylseq +## v2.4.0dev + +### Pipeline Updates + +- Nothing yet.. + +### Bug fixes & refactoring + +- Nothing yet.. + ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16 ### Pipeline Updates diff --git a/nextflow.config b/nextflow.config index 01bcbe46..a07fac23 100644 --- a/nextflow.config +++ b/nextflow.config @@ -255,7 +255,7 @@ manifest { description = """Methylation (Bisulfite-Sequencing) Best Practice analysis pipeline, part of the nf-core community.""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.1' - version = '2.3.0' + version = '2.4.0dev' doi = '10.5281/zenodo.1343417' } From 02ae19721c3429229ed2d240d52cdfc0f10cfe88 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 19 Dec 2022 12:05:59 +0000 Subject: [PATCH 02/58] Template update for nf-core/tools version 2.7.2 --- .github/workflows/fix-linting.yml | 4 +- .github/workflows/linting_comment.yml | 2 +- CHANGELOG.md | 2 +- lib/WorkflowMain.groovy | 2 +- modules.json | 6 +- modules/local/samplesheet_check.nf | 2 +- .../custom/dumpsoftwareversions/main.nf | 2 +- .../templates/dumpsoftwareversions.py | 99 ++++++++++--------- modules/nf-core/fastqc/main.nf | 40 +++----- modules/nf-core/multiqc/main.nf | 2 +- nextflow.config | 5 +- nextflow_schema.json | 6 -- workflows/methylseq.nf | 2 +- 13 files changed, 84 insertions(+), 90 deletions(-) mode change 100644 => 100755 modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 0b7d9bf2..a2f7f450 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -34,9 +34,9 @@ jobs: id: prettier_status run: | if prettier --check ${GITHUB_WORKSPACE}; then - echo "name=result::pass" >> $GITHUB_OUTPUT + echo "result=pass" >> $GITHUB_OUTPUT else - echo "name=result::fail" >> $GITHUB_OUTPUT + echo "result=fail" >> $GITHUB_OUTPUT fi - name: Run 'prettier --write' diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 39635186..0bbcd30f 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -18,7 +18,7 @@ jobs: - name: Get PR number id: pr_number - run: echo "name=pr_number::$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT + run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment uses: marocchino/sticky-pull-request-comment@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a7b27df..a368ff6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.3.0dev - [date] +## v2.4.0dev - [date] Initial release of nf-core/methylseq, created with the [nf-core](https://nf-co.re/) template. diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 2ab6320a..34c480e7 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -72,7 +72,7 @@ class WorkflowMain { NfcoreTemplate.checkConfigProvided(workflow, log) // Check that conda channels are set-up correctly - if (params.enable_conda) { + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { Utils.checkCondaChannels(log) } diff --git a/modules.json b/modules.json index 78d6d721..3a776935 100644 --- a/modules.json +++ b/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", "installed_by": ["modules"] } } diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index d92983a4..5b505be3 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -2,7 +2,7 @@ process SAMPLESHEET_CHECK { tag "$samplesheet" label 'process_single' - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : 'quay.io/biocontainers/python:3.8.3' }" diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index cebb6e05..3df21765 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,7 +2,7 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 index 787bdb7b..e55b8d43 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,5 +1,9 @@ #!/usr/bin/env python + +"""Provide functions to merge multiple versions.yml files.""" + + import platform from textwrap import dedent @@ -7,6 +11,7 @@ def _make_versions_html(versions): + """Generate a tabular HTML output of all versions for MultiQC.""" html = [ dedent( """\\ @@ -45,47 +50,53 @@ def _make_versions_html(versions): return "\\n".join(html) -versions_this_module = {} -versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, -} - -with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - -# aggregate versions by the module name (derived from fully-qualified process name) -versions_by_module = {} -for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - -versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", -} - -versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), -} - -with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) -with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - -with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) +def main(): + """Load all version files and generate merged output.""" + versions_this_module = {} + versions_this_module["${task.process}"] = { + "python": platform.python_version(), + "yaml": yaml.__version__, + } + + with open("$versions") as f: + versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + + # aggregate versions by the module name (derived from fully-qualified process name) + versions_by_module = {} + for process, process_versions in versions_by_process.items(): + module = process.split(":")[-1] + try: + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) + except KeyError: + versions_by_module[module] = process_versions + + versions_by_module["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version", + } + + versions_mqc = { + "id": "software_versions", + "section_name": "${workflow.manifest.name} Software Versions", + "section_href": "https://github.com/${workflow.manifest.name}", + "plot_type": "html", + "description": "are collected at run time from the software output.", + "data": _make_versions_html(versions_by_module), + } + + with open("software_versions.yml", "w") as f: + yaml.dump(versions_by_module, f, default_flow_style=False) + with open("software_versions_mqc.yml", "w") as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open("versions.yml", "w") as f: + yaml.dump(versions_this_module, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 05730368..9ae58381 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -2,7 +2,7 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'quay.io/biocontainers/fastqc:0.11.9--0' }" @@ -20,30 +20,22 @@ process FASTQC { script: def args = task.ext.args ?: '' - // Add soft-links to original FastQs for consistent naming in pipeline def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) - END_VERSIONS - """ - } + // Make list of old name and new name pairs to use for renaming in the bash while loop + def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } + def rename_to = old_new_pairs*.join(' ').join(' ') + def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + """ + printf "%s %s\\n" $rename_to | while read old_name new_name; do + [ -f "\${new_name}" ] || ln -s \$old_name \$new_name + done + fastqc $args --threads $task.cpus $renamed_files + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ stub: def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index a8159a57..68f66bea 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,7 +1,7 @@ process MULTIQC { label 'process_single' - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + conda "bioconda::multiqc=1.13" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" diff --git a/nextflow.config b/nextflow.config index eede7f24..1e01e96e 100644 --- a/nextflow.config +++ b/nextflow.config @@ -39,7 +39,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - enable_conda = false // Config options @@ -81,7 +80,6 @@ try { profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { - params.enable_conda = true conda.enabled = true docker.enabled = false singularity.enabled = false @@ -90,7 +88,6 @@ profiles { charliecloud.enabled = false } mamba { - params.enable_conda = true conda.enabled = true conda.useMamba = true docker.enabled = false @@ -196,7 +193,7 @@ manifest { description = """Methylation (Bisulfite-Sequencing) Best Practice analysis pipeline, part of the nf-core community.""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.1' - version = '2.3.0dev' + version = '2.4.0dev' doi = '' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 0eb7d3d4..acacb847 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -263,12 +263,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile conda' instead of providing this parameter.", - "hidden": true, - "fa_icon": "fas fa-bacon" } } } diff --git a/workflows/methylseq.nf b/workflows/methylseq.nf index 97433136..3908717d 100644 --- a/workflows/methylseq.nf +++ b/workflows/methylseq.nf @@ -82,7 +82,7 @@ workflow METHYLSEQ { ch_versions = ch_versions.mix(FASTQC.out.versions.first()) CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique{ it.text }.collectFile(name: 'collated_versions.yml') + ch_versions.unique().collectFile(name: 'collated_versions.yml') ) // From f3d8cd7d54c0db3e644f7c3efbcea4fa962d05c3 Mon Sep 17 00:00:00 2001 From: Nathan Spix <56930974+njspix@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:16:33 -0500 Subject: [PATCH 03/58] remove unused output folder --- conf/modules.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index f9c155ae..1827da52 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -229,11 +229,6 @@ process { mode: params.publish_dir_mode, pattern: "*bedGraph.gz" ], - [ - path: { "${params.outdir}/${params.aligner}/methylation_calls/stranded_CpG_report" }, - mode: params.publish_dir_mode, - pattern: "*CpG_report.txt.gz" - ], [ path: { "${params.outdir}/${params.aligner}/methylation_calls/splitting_report" }, mode: params.publish_dir_mode, From 375ae9625546b44582fabd76f20c149ef244c261 Mon Sep 17 00:00:00 2001 From: Nathan Spix <56930974+njspix@users.noreply.github.com> Date: Tue, 14 Feb 2023 14:27:07 -0500 Subject: [PATCH 04/58] remove additional unused directory --- conf/modules.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 1827da52..db8ae124 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -245,11 +245,6 @@ process { withName: BISMARK_COVERAGE2CYTOSINE { ext.args = params.nomeseq ? '--nome-seq' : '' publishDir = [ - [ - path: { "${params.outdir}/bismark/coverage2cytosine/coverage" }, - mode: params.publish_dir_mode, - pattern: "*.cov.gz" - ], publishDir = [ path: { "${params.outdir}/bismark/coverage2cytosine/summaries" }, mode: params.publish_dir_mode, From 9e53d947a56a31d033332d6390aed5e0b293c5d6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 28 Apr 2023 14:24:01 +0000 Subject: [PATCH 05/58] Template update for nf-core/tools version 2.8 --- .editorconfig | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .github/workflows/awsfulltest.yml | 2 +- .github/workflows/awstest.yml | 2 +- .github/workflows/branch.yml | 2 +- .github/workflows/clean-up.yml | 24 ++++ .github/workflows/linting.yml | 2 +- .pre-commit-config.yaml | 5 + README.md | 74 ++++++---- bin/check_samplesheet.py | 3 - conf/base.config | 2 +- conf/igenomes.config | 8 ++ conf/test_full.config | 2 + docs/usage.md | 130 +++++------------- lib/NfcoreSchema.groovy | 4 +- lib/WorkflowMain.groovy | 13 +- lib/WorkflowMethylseq.groovy | 12 +- main.nf | 1 - modules.json | 4 +- modules/local/samplesheet_check.nf | 2 +- .../custom/dumpsoftwareversions/main.nf | 6 +- .../custom/dumpsoftwareversions/meta.yml | 2 + modules/nf-core/multiqc/main.nf | 6 +- modules/nf-core/multiqc/meta.yml | 3 +- nextflow.config | 29 +++- tower.yml | 5 + 27 files changed, 193 insertions(+), 157 deletions(-) create mode 100644 .github/workflows/clean-up.yml create mode 100644 .pre-commit-config.yaml create mode 100644 tower.yml diff --git a/.editorconfig b/.editorconfig index b78de6e6..b6b31907 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js,cff}] +[*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 12cd5bb5..1e3f7273 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -45,6 +45,6 @@ body: * Nextflow version _(eg. 22.10.1)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ - * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter or Charliecloud)_ + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ * Version of nf-core/methylseq _(eg. 1.1, 1.5, 1.8.2)_ diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9dc0985d..08cee127 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,7 +15,8 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/meth - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! -- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/methylseq/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/methylseq/tree/master/.github/CONTRIBUTING.md) +- [ ] If necessary, also make a PR on the nf-core/methylseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index fe752425..07a95b0a 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index c2e6cc9c..51d93888 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 352a5cf7..32d20557 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -13,7 +13,7 @@ jobs: - name: Check PRs if: github.repository == 'nf-core/methylseq' run: | - { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/methylseq ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ ${{github.event.pull_request.head.repo.full_name }} == nf-core/methylseq ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] # If the above check failed, post a comment on the PR explaining the failure # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml new file mode 100644 index 00000000..694e90ec --- /dev/null +++ b/.github/workflows/clean-up.yml @@ -0,0 +1,24 @@ +name: "Close user-tagged issues and PRs" +on: + schedule: + - cron: "0 0 * * 0" # Once a week + +jobs: + clean-up: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v7 + with: + stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." + stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." + close-issue-message: "This issue was closed because it has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor and then staled for 20 days with no activity." + days-before-stale: 30 + days-before-close: 20 + days-before-pr-close: -1 + any-of-labels: "awaiting-changes,awaiting-feedback" + exempt-issue-labels: "WIP" + exempt-pr-labels: "WIP" + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 858d622e..888cb4bc 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.7" + python-version: "3.8" architecture: "x64" - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..0c31cdb9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier diff --git a/README.md b/README.md index 70b0e711..45359e82 100644 --- a/README.md +++ b/README.md @@ -8,57 +8,71 @@ [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/methylseq) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23methylseq-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/methylseq)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23methylseq-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/methylseq)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction - +**nf-core/methylseq** is a bioinformatics pipeline that ... -**nf-core/methylseq** is a bioinformatics best-practice analysis pipeline for Methylation (Bisulfite-Sequencing) Best Practice analysis pipeline, part of the nf-core community.. - -The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It uses Docker/Singularity containers making installation trivial and results highly reproducible. The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. Where possible, these processes have been submitted to and installed from [nf-core/modules](https://github.com/nf-core/modules) in order to make them available to all nf-core pipelines, and to everyone within the Nextflow community! - - - -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources.The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/methylseq/results). - -## Pipeline summary + + 1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) 2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)) -## Quick Start +## Usage + +> **Note** +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how +> to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) +> with `-profile test` before running the workflow on actual data. + + - Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string. +Now, you can run the pipeline using: - > - The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`. - > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. - > - If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. - > - If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs. + -4. Start running your own analysis! +```bash +nextflow run nf-core/methylseq \ + -profile \ + --input samplesheet.csv \ + --outdir +``` - +> **Warning:** +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those +> provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; +> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). - ```bash - nextflow run nf-core/methylseq --input samplesheet.csv --outdir --genome GRCh37 -profile - ``` +For more details, please refer to the [usage documentation](https://nf-co.re/methylseq/usage) and the [parameter documentation](https://nf-co.re/methylseq/parameters). -## Documentation +## Pipeline output -The nf-core/methylseq pipeline comes with documentation about the pipeline [usage](https://nf-co.re/methylseq/usage), [parameters](https://nf-co.re/methylseq/parameters) and [output](https://nf-co.re/methylseq/output). +To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/methylseq/results) tab on the nf-core website pipeline page. +For more details about the output files and reports, please refer to the +[output documentation](https://nf-co.re/methylseq/output). ## Credits diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 11b15572..4a758fe0 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -158,9 +158,6 @@ def sniff_format(handle): peek = read_head(handle) handle.seek(0) sniffer = csv.Sniffer() - if not sniffer.has_header(peek): - logger.critical("The given sample sheet does not appear to contain a header.") - sys.exit(1) dialect = sniffer.sniff(peek) return dialect diff --git a/conf/base.config b/conf/base.config index 35c5aa84..4592c308 100644 --- a/conf/base.config +++ b/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' diff --git a/conf/igenomes.config b/conf/igenomes.config index 7a1b3ac6..3f114377 100644 --- a/conf/igenomes.config +++ b/conf/igenomes.config @@ -36,6 +36,14 @@ params { macs_gsize = "2.7e9" blacklist = "${projectDir}/assets/blacklists/hg38-blacklist.bed" } + 'CHM13' { + fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAIndex/" + bwamem2 = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAmem2Index/" + gtf = "${params.igenomes_base}/Homo_sapiens/NCBI/CHM13/Annotation/Genes/genes.gtf" + gff = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/009/914/755/GCF_009914755.1_T2T-CHM13v2.0/GCF_009914755.1_T2T-CHM13v2.0_genomic.gff.gz" + mito_name = "chrM" + } 'GRCm38' { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/version0.6.0/" diff --git a/conf/test_full.config b/conf/test_full.config index e1595836..27911fa0 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -10,6 +10,8 @@ ---------------------------------------------------------------------------------------- */ +cleanup = true + params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/docs/usage.md b/docs/usage.md index 77dc6373..39849f26 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -71,6 +71,29 @@ work # Directory containing the nextflow working files # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` +If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. + +Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. + +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). +> The above pipeline run specified with a params file in yaml format: + +```bash +nextflow run nf-core/methylseq -profile docker -params-file params.yaml +``` + +with `params.yaml` containing: + +```yaml +input: './samplesheet.csv' +outdir: './results/' +genome: 'GRCh37' +input: 'data' +<...> +``` + +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). + ### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: @@ -87,6 +110,10 @@ First, go to the [nf-core/methylseq releases page](https://github.com/nf-core/me This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. + +> 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. + ## Core Nextflow arguments > **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). @@ -95,7 +122,7 @@ This version number will be logged in reports when you run the pipeline, so that Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -119,8 +146,10 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) - `charliecloud` - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) +- `apptainer` + - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. ### `-resume` @@ -138,102 +167,19 @@ Specify the path to a specific config file (this is a core Nextflow command). Se Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. -For example, if the nf-core/rnaseq pipeline is failing after multiple re-submissions of the `STAR_ALIGN` process due to an exit code of `137` this would indicate that there is an out of memory issue: - -```console -[62/149eb0] NOTE: Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) -Error executing process > 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' - -Caused by: - Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) - -Command executed: - STAR \ - --genomeDir star \ - --readFilesIn WT_REP1_trimmed.fq.gz \ - --runThreadN 2 \ - --outFileNamePrefix WT_REP1. \ - - -Command exit status: - 137 - -Command output: - (empty) - -Command error: - .command.sh: line 9: 30 Killed STAR --genomeDir star --readFilesIn WT_REP1_trimmed.fq.gz --runThreadN 2 --outFileNamePrefix WT_REP1. -Work dir: - /home/pipelinetest/work/9d/172ca5881234073e8d76f2a19c88fb - -Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` -``` - -#### For beginners - -A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. - -#### Advanced option on process level - -To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). -We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/star/align/main.nf`. -If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). -The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. -The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. -Providing you haven't set any other standard nf-core parameters to **cap** the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. -The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. - -```nextflow -process { - withName: 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN' { - memory = 100.GB - } -} -``` - -> **NB:** We specify the full process name i.e. `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN` in the config file because this takes priority over the short name (`STAR_ALIGN`) and allows existing configuration using the full process name to be correctly overridden. -> -> If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. - -### Updating containers (advanced users) - -The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. If for some reason you need to use a different version of a particular tool with the pipeline then you just need to identify the `process` name and override the Nextflow `container` definition for that process using the `withName` declaration. For example, in the [nf-core/viralrecon](https://nf-co.re/viralrecon) pipeline a tool called [Pangolin](https://github.com/cov-lineages/pangolin) has been used during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. Given that the lineage assignments change quite frequently it doesn't make sense to re-release the nf-core/viralrecon everytime a new version of Pangolin has been released. However, you can override the default container used by the pipeline by creating a custom config file and passing it as a command-line argument via `-c custom.config`. - -1. Check the default version used by the pipeline in the module file for [Pangolin](https://github.com/nf-core/viralrecon/blob/a85d5969f9025409e3618d6c280ef15ce417df65/modules/nf-core/software/pangolin/main.nf#L14-L19) -2. Find the latest version of the Biocontainer available on [Quay.io](https://quay.io/repository/biocontainers/pangolin?tag=latest&tab=tags) -3. Create the custom config accordingly: - - - For Docker: +To change the resource requests, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. - ```nextflow - process { - withName: PANGOLIN { - container = 'quay.io/biocontainers/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +### Custom Containers - - For Singularity: +In some cases you may wish to change which container or conda environment a step of the pipeline uses for a particular tool. By default nf-core pipelines use containers and software from the [biocontainers](https://biocontainers.pro/) or [bioconda](https://bioconda.github.io/) projects. However in some cases the pipeline specified version maybe out of date. - ```nextflow - process { - withName: PANGOLIN { - container = 'https://depot.galaxyproject.org/singularity/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +To use a different container from the default container or conda environment specified in a pipeline, please see the [updating tool versions](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. - - For Conda: +### Custom Tool Arguments - ```nextflow - process { - withName: PANGOLIN { - conda = 'bioconda::pangolin=3.0.5' - } - } - ``` +A pipeline might not always support every possible argument or option of a particular tool used in pipeline. Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. -> **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. +To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. ### nf-core/configs diff --git a/lib/NfcoreSchema.groovy b/lib/NfcoreSchema.groovy index 33cd4f6e..9b34804d 100755 --- a/lib/NfcoreSchema.groovy +++ b/lib/NfcoreSchema.groovy @@ -2,6 +2,7 @@ // This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. // +import nextflow.Nextflow import org.everit.json.schema.Schema import org.everit.json.schema.loader.SchemaLoader import org.everit.json.schema.ValidationException @@ -83,6 +84,7 @@ class NfcoreSchema { 'stub-run', 'test', 'w', + 'with-apptainer', 'with-charliecloud', 'with-conda', 'with-dag', @@ -177,7 +179,7 @@ class NfcoreSchema { } if (has_error) { - System.exit(1) + Nextflow.error('Exiting!') } } diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 34c480e7..b86d8184 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the main.nf workflow in the nf-core/methylseq pipeline // +import nextflow.Nextflow + class WorkflowMain { // @@ -21,7 +23,7 @@ class WorkflowMain { // // Generate help string // - public static String help(workflow, params, log) { + public static String help(workflow, params) { def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" def help_string = '' help_string += NfcoreTemplate.logo(workflow, params.monochrome_logs) @@ -34,7 +36,7 @@ class WorkflowMain { // // Generate parameter summary log string // - public static String paramsSummaryLog(workflow, params, log) { + public static String paramsSummaryLog(workflow, params) { def summary_log = '' summary_log += NfcoreTemplate.logo(workflow, params.monochrome_logs) summary_log += NfcoreSchema.paramsSummaryLog(workflow, params) @@ -49,7 +51,7 @@ class WorkflowMain { public static void initialise(workflow, params, log) { // Print help to screen if required if (params.help) { - log.info help(workflow, params, log) + log.info help(workflow, params) System.exit(0) } @@ -61,7 +63,7 @@ class WorkflowMain { } // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params, log) + log.info paramsSummaryLog(workflow, params) // Validate workflow parameters via the JSON schema if (params.validate_params) { @@ -81,8 +83,7 @@ class WorkflowMain { // Check input has been provided if (!params.input) { - log.error "Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'" - System.exit(1) + Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") } } // diff --git a/lib/WorkflowMethylseq.groovy b/lib/WorkflowMethylseq.groovy index 3e466067..29066a19 100755 --- a/lib/WorkflowMethylseq.groovy +++ b/lib/WorkflowMethylseq.groovy @@ -2,6 +2,7 @@ // This file holds several functions specific to the workflow/methylseq.nf in the nf-core/methylseq pipeline // +import nextflow.Nextflow import groovy.text.SimpleTemplateEngine class WorkflowMethylseq { @@ -14,8 +15,7 @@ class WorkflowMethylseq { if (!params.fasta) { - log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - System.exit(1) + Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." } } @@ -61,17 +61,19 @@ class WorkflowMethylseq { def description_html = engine.createTemplate(methods_text).make(meta) return description_html - }// + } + + // // Exit pipeline if incorrect --genome key provided // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } } } diff --git a/main.nf b/main.nf index 40fe491d..d90e67c2 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,6 @@ nf-core/methylseq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/methylseq - Website: https://nf-co.re/methylseq Slack : https://nfcore.slack.com/channels/methylseq ---------------------------------------------------------------------------------------- diff --git a/modules.json b/modules.json index 3a776935..ab58bdf2 100644 --- a/modules.json +++ b/modules.json @@ -7,7 +7,7 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "76cc4938c1f6ea5c7d83fed1eeffc146787f9543", "installed_by": ["modules"] }, "fastqc": { @@ -17,7 +17,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", "installed_by": ["modules"] } } diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index 5b505be3..a88ca35b 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -5,7 +5,7 @@ process SAMPLESHEET_CHECK { conda "conda-forge::python=3.8.3" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/python:3.8.3' : - 'quay.io/biocontainers/python:3.8.3' }" + 'biocontainers/python:3.8.3' }" input: path samplesheet diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 3df21765..800a6099 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 60b546a0..c32657de 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,7 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom + - dump - version tools: - custom: diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 68f66bea..4b604749 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index ebc29b27..f93b5ee5 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: MultiQC description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: @@ -37,7 +38,7 @@ output: description: MultiQC report file pattern: "multiqc_report.html" - data: - type: dir + type: directory description: MultiQC data dir pattern: "multiqc_data" - plots: diff --git a/nextflow.config b/nextflow.config index 1e01e96e..51614373 100644 --- a/nextflow.config +++ b/nextflow.config @@ -78,7 +78,11 @@ try { profiles { - debug { process.beforeScript = 'echo $HOSTNAME' } + debug { + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false + } conda { conda.enabled = true docker.enabled = false @@ -86,6 +90,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } mamba { conda.enabled = true @@ -95,14 +100,18 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } docker { docker.enabled = true + docker.registry = 'quay.io' docker.userEmulation = true + conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' @@ -110,31 +119,49 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + conda.enabled = false docker.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } podman { podman.enabled = true + podman.registry = 'quay.io' + conda.enabled = false docker.enabled = false singularity.enabled = false shifter.enabled = false charliecloud.enabled = false + apptainer.enabled = false } shifter { shifter.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false charliecloud.enabled = false + apptainer.enabled = false } charliecloud { charliecloud.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false + apptainer.enabled = false + } + apptainer { + apptainer.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } gitpod { executor.name = 'local' diff --git a/tower.yml b/tower.yml new file mode 100644 index 00000000..787aedfe --- /dev/null +++ b/tower.yml @@ -0,0 +1,5 @@ +reports: + multiqc_report.html: + display: "MultiQC HTML report" + samplesheet.csv: + display: "Auto-created samplesheet with collated metadata and FASTQ paths" From 069b0f05678514d3e33e3e64e9211d49a41d328b Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 16 May 2023 12:41:35 -0500 Subject: [PATCH 06/58] style: versions => ch_versions To match the template --- workflows/methylseq.nf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/workflows/methylseq.nf b/workflows/methylseq.nf index f86a6181..7cab0b5c 100644 --- a/workflows/methylseq.nf +++ b/workflows/methylseq.nf @@ -87,13 +87,13 @@ def multiqc_report = [] workflow METHYLSEQ { - versions = Channel.empty() + ch_versions = Channel.empty() // // SUBWORKFLOW: Prepare any required reference genome indices // PREPARE_GENOME() - versions = versions.mix(PREPARE_GENOME.out.versions) + ch_versions = ch_versions.mix(PREPARE_GENOME.out.versions) // // SUBWORKFLOW: Read in samplesheet, validate and stage input files @@ -117,7 +117,7 @@ workflow METHYLSEQ { return [ meta, fastq.flatten() ] } .set { ch_fastq } - versions = versions.mix(INPUT_CHECK.out.versions) + ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) // // MODULE: Concatenate FastQ files from same sample if required @@ -128,7 +128,7 @@ workflow METHYLSEQ { .reads .mix(ch_fastq.single) .set { ch_cat_fastq } - versions = versions.mix(CAT_FASTQ.out.versions.first()) + ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first()) // // MODULE: Run FastQC @@ -136,7 +136,7 @@ workflow METHYLSEQ { FASTQC ( ch_cat_fastq ) - versions = versions.mix(FASTQC.out.versions.first()) + ch_versions = ch_versions.mix(FASTQC.out.versions.first()) /* * MODULE: Run TrimGalore! @@ -144,7 +144,7 @@ workflow METHYLSEQ { if (!params.skip_trimming) { TRIMGALORE(ch_cat_fastq) reads = TRIMGALORE.out.reads - versions = versions.mix(TRIMGALORE.out.versions.first()) + ch_versions = ch_versions.mix(TRIMGALORE.out.versions.first()) } else { reads = ch_cat_fastq } @@ -167,7 +167,7 @@ workflow METHYLSEQ { params.skip_deduplication || params.rrbs, params.cytosine_report || params.nomeseq ) - versions = versions.mix(BISMARK.out.versions.unique{ it.baseName }) + ch_versions = ch_versions.mix(BISMARK.out.versions.unique{ it.baseName }) ch_bam = BISMARK.out.bam ch_dedup = BISMARK.out.dedup ch_aligner_mqc = BISMARK.out.mqc @@ -182,7 +182,7 @@ workflow METHYLSEQ { PREPARE_GENOME.out.fasta_index, params.skip_deduplication || params.rrbs, ) - versions = versions.mix(BWAMETH.out.versions.unique{ it.baseName }) + ch_versions = ch_versions.mix(BWAMETH.out.versions.unique{ it.baseName }) ch_bam = BWAMETH.out.bam ch_dedup = BWAMETH.out.dedup ch_aligner_mqc = BWAMETH.out.mqc @@ -195,7 +195,7 @@ workflow METHYLSEQ { ch_dedup, [] ) - versions = versions.mix(QUALIMAP_BAMQC.out.versions.first()) + ch_versions = ch_versions.mix(QUALIMAP_BAMQC.out.versions.first()) /* * MODULE: Run Preseq @@ -238,7 +238,7 @@ workflow METHYLSEQ { ch_multiqc_logo.toList() ) multiqc_report = MULTIQC.out.report.toList() - versions = versions.mix(MULTIQC.out.versions) + ch_versions = ch_versions.mix(MULTIQC.out.versions) } } From 842083e16be3b8957b815ca8cdd9be03f9b83596 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 16 May 2023 12:42:57 -0500 Subject: [PATCH 07/58] chore: Add version update to CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5704ca46..6625542a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Pipeline Updates -- Nothing yet.. +- Updated template to nf-core/tools v2.8 ### Bug fixes & refactoring From 500c0ff40c965f2fc1d7c4f856b433a792642a2a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 16 May 2023 12:47:21 -0500 Subject: [PATCH 08/58] chore: Run nf-core modules update --- modules.json | 50 +++++++++---------- modules/nf-core/bismark/align/main.nf | 4 +- .../nf-core/bismark/coverage2cytosine/main.nf | 4 +- modules/nf-core/bismark/deduplicate/main.nf | 4 +- .../nf-core/bismark/genomepreparation/main.nf | 4 +- .../bismark/methylationextractor/main.nf | 4 +- modules/nf-core/bismark/report/main.nf | 4 +- modules/nf-core/bismark/summary/main.nf | 4 +- modules/nf-core/bwameth/align/main.nf | 6 +-- modules/nf-core/bwameth/index/main.nf | 4 +- modules/nf-core/cat/fastq/main.nf | 4 +- modules/nf-core/cat/fastq/meta.yml | 3 +- .../custom/dumpsoftwareversions/main.nf | 2 +- .../templates/dumpsoftwareversions.py | 1 + modules/nf-core/fastqc/main.nf | 2 +- modules/nf-core/methyldackel/extract/main.nf | 4 +- modules/nf-core/methyldackel/extract/meta.yml | 9 ++-- modules/nf-core/methyldackel/mbias/main.nf | 4 +- modules/nf-core/methyldackel/mbias/meta.yml | 9 ++-- modules/nf-core/multiqc/main.nf | 2 +- modules/nf-core/picard/markduplicates/main.nf | 12 ++--- modules/nf-core/preseq/lcextrap/main.nf | 4 +- modules/nf-core/preseq/lcextrap/meta.yml | 2 +- modules/nf-core/qualimap/bamqc/main.nf | 8 +-- modules/nf-core/qualimap/bamqccram/main.nf | 10 ++-- modules/nf-core/qualimap/bamqccram/meta.yml | 2 +- modules/nf-core/samtools/faidx/main.nf | 6 +-- modules/nf-core/samtools/flagstat/main.nf | 6 +-- modules/nf-core/samtools/flagstat/meta.yml | 2 +- modules/nf-core/samtools/index/main.nf | 6 +-- modules/nf-core/samtools/index/meta.yml | 2 +- modules/nf-core/samtools/sort/main.nf | 16 ++++-- modules/nf-core/samtools/sort/meta.yml | 2 +- modules/nf-core/samtools/stats/main.nf | 6 +-- modules/nf-core/samtools/stats/meta.yml | 14 +++--- modules/nf-core/trimgalore/main.nf | 21 ++++---- modules/nf-core/trimgalore/meta.yml | 2 +- 37 files changed, 129 insertions(+), 120 deletions(-) diff --git a/modules.json b/modules.json index d2ff9f8e..2b51dee0 100644 --- a/modules.json +++ b/modules.json @@ -7,127 +7,127 @@ "nf-core": { "bismark/align": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/coverage2cytosine": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/deduplicate": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/genomepreparation": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/methylationextractor": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/report": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bismark/summary": { "branch": "master", - "git_sha": "e84cc8f07607ad7970db9b6cf79b815b78d9cacd", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bwameth/align": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "bwameth/index": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "cat/fastq": { "branch": "master", - "git_sha": "76cc4938c1f6ea5c7d83fed1eeffc146787f9543", + "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", "installed_by": ["modules"] }, "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "8022c68e7403eecbd8ba9c49496f69f8c49d50f0", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "methyldackel/extract": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "methyldackel/mbias": { "branch": "master", - "git_sha": "cdaaf00a580a2b3378926b08dd44db1de44ed7b5", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "picard/markduplicates": { "branch": "master", - "git_sha": "eca65aa4a5e2e192ac44d6962c8f9260f314ffb8", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "preseq/lcextrap": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "qualimap/bamqc": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "qualimap/bamqccram": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220", "installed_by": ["modules"] }, "samtools/faidx": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/flagstat": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/index": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/sort": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "samtools/stats": { "branch": "master", - "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] }, "trimgalore": { "branch": "master", - "git_sha": "b51a69e30973c71950225c817ad07a3337d22c40", + "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", "installed_by": ["modules"] } } diff --git a/modules/nf-core/bismark/align/main.nf b/modules/nf-core/bismark/align/main.nf index eccb8550..d36f6726 100644 --- a/modules/nf-core/bismark/align/main.nf +++ b/modules/nf-core/bismark/align/main.nf @@ -2,10 +2,10 @@ process BISMARK_ALIGN { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/bismark/coverage2cytosine/main.nf b/modules/nf-core/bismark/coverage2cytosine/main.nf index 5051f34c..f5429700 100644 --- a/modules/nf-core/bismark/coverage2cytosine/main.nf +++ b/modules/nf-core/bismark/coverage2cytosine/main.nf @@ -2,10 +2,10 @@ process BISMARK_COVERAGE2CYTOSINE { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::bismark=0.23.0" : null) + conda "bioconda::bismark=0.23.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(coverage_file) diff --git a/modules/nf-core/bismark/deduplicate/main.nf b/modules/nf-core/bismark/deduplicate/main.nf index be75b54d..7e238d77 100644 --- a/modules/nf-core/bismark/deduplicate/main.nf +++ b/modules/nf-core/bismark/deduplicate/main.nf @@ -2,10 +2,10 @@ process BISMARK_DEDUPLICATE { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/bismark/genomepreparation/main.nf b/modules/nf-core/bismark/genomepreparation/main.nf index 695a10f8..8f424443 100644 --- a/modules/nf-core/bismark/genomepreparation/main.nf +++ b/modules/nf-core/bismark/genomepreparation/main.nf @@ -2,10 +2,10 @@ process BISMARK_GENOMEPREPARATION { tag "$fasta" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: path fasta, stageAs: "BismarkIndex/*" diff --git a/modules/nf-core/bismark/methylationextractor/main.nf b/modules/nf-core/bismark/methylationextractor/main.nf index b8018ac7..2a3e5d22 100644 --- a/modules/nf-core/bismark/methylationextractor/main.nf +++ b/modules/nf-core/bismark/methylationextractor/main.nf @@ -2,10 +2,10 @@ process BISMARK_METHYLATIONEXTRACTOR { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/bismark/report/main.nf b/modules/nf-core/bismark/report/main.nf index 744c6cd1..804a141d 100644 --- a/modules/nf-core/bismark/report/main.nf +++ b/modules/nf-core/bismark/report/main.nf @@ -2,10 +2,10 @@ process BISMARK_REPORT { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: tuple val(meta), path(align_report), path(dedup_report), path(splitting_report), path(mbias) diff --git a/modules/nf-core/bismark/summary/main.nf b/modules/nf-core/bismark/summary/main.nf index 60854fba..399f960d 100644 --- a/modules/nf-core/bismark/summary/main.nf +++ b/modules/nf-core/bismark/summary/main.nf @@ -1,10 +1,10 @@ process BISMARK_SUMMARY { label 'process_low' - conda (params.enable_conda ? "bioconda::bismark=0.24.0" : null) + conda "bioconda::bismark=0.24.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bismark:0.24.0--hdfd78af_0' : - 'quay.io/biocontainers/bismark:0.24.0--hdfd78af_0' }" + 'biocontainers/bismark:0.24.0--hdfd78af_0' }" input: val(bam) diff --git a/modules/nf-core/bwameth/align/main.nf b/modules/nf-core/bwameth/align/main.nf index 6d9c4431..47304035 100644 --- a/modules/nf-core/bwameth/align/main.nf +++ b/modules/nf-core/bwameth/align/main.nf @@ -2,10 +2,10 @@ process BWAMETH_ALIGN { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? "bioconda::bwameth=0.2.2" : null) + conda "bioconda::bwameth=0.2.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bwameth:0.2.2--py_1' : - 'quay.io/biocontainers/bwameth:0.2.2--py_1' }" + 'biocontainers/bwameth:0.2.2--py_1' }" input: tuple val(meta), path(reads) @@ -24,7 +24,7 @@ process BWAMETH_ALIGN { def prefix = task.ext.prefix ?: "${meta.id}" def read_group = meta.read_group ? "-R ${meta.read_group}" : "" """ - INDEX=`find -L ${index} -name "*.bwameth.c2t" | sed 's/.bwameth.c2t//'` + INDEX=`find -L ${index} -name "*.bwameth.c2t" | sed 's/\\.bwameth.c2t\$//'` # Modify the timestamps so that bwameth doesn't complain about building the index # See https://github.com/nf-core/methylseq/pull/217 diff --git a/modules/nf-core/bwameth/index/main.nf b/modules/nf-core/bwameth/index/main.nf index 4c106826..1c789885 100644 --- a/modules/nf-core/bwameth/index/main.nf +++ b/modules/nf-core/bwameth/index/main.nf @@ -2,10 +2,10 @@ process BWAMETH_INDEX { tag "$fasta" label 'process_high' - conda (params.enable_conda ? "bioconda::bwameth=0.2.2" : null) + conda "bioconda::bwameth=0.2.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bwameth:0.2.2--py_1' : - 'quay.io/biocontainers/bwameth:0.2.2--py_1' }" + 'biocontainers/bwameth:0.2.2--py_1' }" input: path fasta, stageAs: "bwameth/*" diff --git a/modules/nf-core/cat/fastq/main.nf b/modules/nf-core/cat/fastq/main.nf index 4fa365d3..5021e6fc 100644 --- a/modules/nf-core/cat/fastq/main.nf +++ b/modules/nf-core/cat/fastq/main.nf @@ -2,10 +2,10 @@ process CAT_FASTQ { tag "$meta.id" label 'process_single' - conda (params.enable_conda ? "conda-forge::sed=4.7" : null) + conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'ubuntu:20.04' }" + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(reads, stageAs: "input*/*") diff --git a/modules/nf-core/cat/fastq/meta.yml b/modules/nf-core/cat/fastq/meta.yml index c836598e..8a39e309 100644 --- a/modules/nf-core/cat/fastq/meta.yml +++ b/modules/nf-core/cat/fastq/meta.yml @@ -1,6 +1,7 @@ name: cat_fastq description: Concatenates fastq files keywords: + - cat - fastq - concatenate tools: @@ -16,7 +17,7 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - reads: - type: list + type: file description: | List of input FastQ files to be concatenated. output: diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index 800a6099..ebc87273 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -5,7 +5,7 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index b10e0a5c..da033408 100755 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -4,6 +4,7 @@ """Provide functions to merge multiple versions.yml files.""" +import yaml import platform from textwrap import dedent diff --git a/modules/nf-core/fastqc/main.nf b/modules/nf-core/fastqc/main.nf index 9ae58381..07d5e433 100644 --- a/modules/nf-core/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -5,7 +5,7 @@ process FASTQC { conda "bioconda::fastqc=0.11.9" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'quay.io/biocontainers/fastqc:0.11.9--0' }" + 'biocontainers/fastqc:0.11.9--0' }" input: tuple val(meta), path(reads) diff --git a/modules/nf-core/methyldackel/extract/main.nf b/modules/nf-core/methyldackel/extract/main.nf index 2ad1dc68..32951459 100644 --- a/modules/nf-core/methyldackel/extract/main.nf +++ b/modules/nf-core/methyldackel/extract/main.nf @@ -2,10 +2,10 @@ process METHYLDACKEL_EXTRACT { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? 'bioconda::methyldackel=0.6.0' : null) + conda "bioconda::methyldackel=0.6.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/methyldackel:0.6.0--h22771d5_0' : - 'quay.io/biocontainers/methyldackel:0.6.0--h22771d5_0' }" + 'biocontainers/methyldackel:0.6.0--h22771d5_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/methyldackel/extract/meta.yml b/modules/nf-core/methyldackel/extract/meta.yml index a831df16..9d777bb1 100644 --- a/modules/nf-core/methyldackel/extract/meta.yml +++ b/modules/nf-core/methyldackel/extract/meta.yml @@ -13,11 +13,10 @@ keywords: tools: - methyldackel: description: | - A (mostly) universal methylation extractor - for BS-seq experiments. - homepage: https://github.com/brentp/bwa-meth - documentation: https://github.com/brentp/bwa-meth - arxiv: arXiv:1401.1129 + Methylation caller from MethylDackel, a (mostly) universal methylation extractor + for methyl-seq experiments. + homepage: https://github.com/dpryan79/MethylDackel + documentation: https://github.com/dpryan79/MethylDackel/blob/master/README.md licence: ["MIT"] input: - meta: diff --git a/modules/nf-core/methyldackel/mbias/main.nf b/modules/nf-core/methyldackel/mbias/main.nf index 755e62f8..b30d3391 100644 --- a/modules/nf-core/methyldackel/mbias/main.nf +++ b/modules/nf-core/methyldackel/mbias/main.nf @@ -2,10 +2,10 @@ process METHYLDACKEL_MBIAS { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? 'bioconda::methyldackel=0.6.0' : null) + conda "bioconda::methyldackel=0.6.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/methyldackel:0.6.0--h22771d5_0' : - 'quay.io/biocontainers/methyldackel:0.6.0--h22771d5_0' }" + 'biocontainers/methyldackel:0.6.0--h22771d5_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/methyldackel/mbias/meta.yml b/modules/nf-core/methyldackel/mbias/meta.yml index 370f052a..75217232 100644 --- a/modules/nf-core/methyldackel/mbias/meta.yml +++ b/modules/nf-core/methyldackel/mbias/meta.yml @@ -14,11 +14,10 @@ keywords: tools: - methyldackel: description: | - A (mostly) universal methylation extractor - for BS-seq experiments. - homepage: https://github.com/brentp/bwa-meth - documentation: https://github.com/brentp/bwa-meth - arxiv: arXiv:1401.1129 + Read position methylation bias tools from MethylDackel, a (mostly) universal extractor + for methyl-seq experiments. + homepage: https://github.com/dpryan79/MethylDackel + documentation: https://github.com/dpryan79/MethylDackel/blob/master/README.md licence: ["MIT"] input: - meta: diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 4b604749..1fc387be 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -4,7 +4,7 @@ process MULTIQC { conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/picard/markduplicates/main.nf b/modules/nf-core/picard/markduplicates/main.nf index d1f3aaa1..7ba8448f 100644 --- a/modules/nf-core/picard/markduplicates/main.nf +++ b/modules/nf-core/picard/markduplicates/main.nf @@ -2,10 +2,10 @@ process PICARD_MARKDUPLICATES { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::picard=2.27.4" : null) + conda "bioconda::picard=3.0.0" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/picard:2.27.4--hdfd78af_0' : - 'quay.io/biocontainers/picard:2.27.4--hdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/picard:3.0.0--hdfd78af_1' : + 'biocontainers/picard:3.0.0--hdfd78af_1' }" input: tuple val(meta), path(bam) @@ -24,15 +24,15 @@ process PICARD_MARKDUPLICATES { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def avail_mem = 3 + def avail_mem = 3072 if (!task.memory) { log.info '[Picard MarkDuplicates] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' } else { - avail_mem = task.memory.giga + avail_mem = (task.memory.mega*0.8).intValue() } """ picard \\ - -Xmx${avail_mem}g \\ + -Xmx${avail_mem}M \\ MarkDuplicates \\ $args \\ --INPUT $bam \\ diff --git a/modules/nf-core/preseq/lcextrap/main.nf b/modules/nf-core/preseq/lcextrap/main.nf index a98a922c..12546f0a 100644 --- a/modules/nf-core/preseq/lcextrap/main.nf +++ b/modules/nf-core/preseq/lcextrap/main.nf @@ -3,10 +3,10 @@ process PRESEQ_LCEXTRAP { label 'process_single' label 'error_ignore' - conda (params.enable_conda ? "bioconda::preseq=3.1.2" : null) + conda "bioconda::preseq=3.1.2" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/preseq:3.1.2--h445547b_2': - 'quay.io/biocontainers/preseq:3.1.2--h445547b_2' }" + 'biocontainers/preseq:3.1.2--h445547b_2' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/preseq/lcextrap/meta.yml b/modules/nf-core/preseq/lcextrap/meta.yml index f1be05a2..1391961c 100755 --- a/modules/nf-core/preseq/lcextrap/meta.yml +++ b/modules/nf-core/preseq/lcextrap/meta.yml @@ -10,7 +10,7 @@ tools: homepage: http://smithlabresearch.org/software/preseq/ documentation: http://smithlabresearch.org/wp-content/uploads/manual.pdf tool_dev_url: https://github.com/smithlabcode/preseq - doi: "" + licence: ["GPL"] input: diff --git a/modules/nf-core/qualimap/bamqc/main.nf b/modules/nf-core/qualimap/bamqc/main.nf index 3bfcb4c1..fef7307a 100644 --- a/modules/nf-core/qualimap/bamqc/main.nf +++ b/modules/nf-core/qualimap/bamqc/main.nf @@ -2,10 +2,10 @@ process QUALIMAP_BAMQC { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::qualimap=2.2.2d" : null) + conda "bioconda::qualimap=2.2.2d" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/qualimap:2.2.2d--1' : - 'quay.io/biocontainers/qualimap:2.2.2d--1' }" + 'biocontainers/qualimap:2.2.2d--1' }" input: tuple val(meta), path(bam) @@ -23,7 +23,7 @@ process QUALIMAP_BAMQC { prefix = task.ext.prefix ?: "${meta.id}" def collect_pairs = meta.single_end ? '' : '--collect-overlap-pairs' - def memory = task.memory.toGiga() + "G" + def memory = (task.memory.mega*0.8).intValue() + 'M' def regions = gff ? "--gff $gff" : '' def strandedness = 'non-strand-specific' @@ -34,7 +34,7 @@ process QUALIMAP_BAMQC { } """ unset DISPLAY - mkdir tmp + mkdir -p tmp export _JAVA_OPTIONS=-Djava.io.tmpdir=./tmp qualimap \\ --java-mem-size=$memory \\ diff --git a/modules/nf-core/qualimap/bamqccram/main.nf b/modules/nf-core/qualimap/bamqccram/main.nf index e136b8e2..327d8557 100644 --- a/modules/nf-core/qualimap/bamqccram/main.nf +++ b/modules/nf-core/qualimap/bamqccram/main.nf @@ -2,10 +2,10 @@ process QUALIMAP_BAMQCCRAM { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::qualimap=2.2.2d bioconda::samtools=1.15.1" : null) + conda "bioconda::qualimap=2.2.2d bioconda::samtools=1.16.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:61f6d4658ac88635fc37623af50bba77561988ab-0' : - 'quay.io/biocontainers/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:61f6d4658ac88635fc37623af50bba77561988ab-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:00d3c18496ddf07ea580fd00d1dd203cf31ab630-0' : + 'biocontainers/mulled-v2-d3934ca6bb4e61334891ffa2e9a4c87a530e3188:00d3c18496ddf07ea580fd00d1dd203cf31ab630-0' }" input: tuple val(meta), path(cram), path(crai) @@ -25,7 +25,7 @@ process QUALIMAP_BAMQCCRAM { prefix = task.ext.prefix ?: "${meta.id}" def collect_pairs = meta.single_end ? '' : '--collect-overlap-pairs' - def memory = task.memory.toGiga() + "G" + def memory = (task.memory.mega*0.8).intValue() + 'M' def regions = gff ? "--gff $gff" : '' def strandedness = 'non-strand-specific' @@ -36,7 +36,7 @@ process QUALIMAP_BAMQCCRAM { } """ unset DISPLAY - mkdir tmp + mkdir -p tmp export _JAVA_OPTIONS=-Djava.io.tmpdir=./tmp samtools view -hb -T ${fasta} ${cram} | diff --git a/modules/nf-core/qualimap/bamqccram/meta.yml b/modules/nf-core/qualimap/bamqccram/meta.yml index d72f203d..ea6c4d92 100644 --- a/modules/nf-core/qualimap/bamqccram/meta.yml +++ b/modules/nf-core/qualimap/bamqccram/meta.yml @@ -40,7 +40,7 @@ output: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - results: - type: dir + type: directory description: Qualimap results dir pattern: "*/*" - versions: diff --git a/modules/nf-core/samtools/faidx/main.nf b/modules/nf-core/samtools/faidx/main.nf index ef940db2..4dd0e5b0 100644 --- a/modules/nf-core/samtools/faidx/main.nf +++ b/modules/nf-core/samtools/faidx/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_FAIDX { tag "$fasta" label 'process_single' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(fasta) diff --git a/modules/nf-core/samtools/flagstat/main.nf b/modules/nf-core/samtools/flagstat/main.nf index c3152aca..eb7e72fc 100644 --- a/modules/nf-core/samtools/flagstat/main.nf +++ b/modules/nf-core/samtools/flagstat/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_FLAGSTAT { tag "$meta.id" label 'process_single' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam), path(bai) diff --git a/modules/nf-core/samtools/flagstat/meta.yml b/modules/nf-core/samtools/flagstat/meta.yml index 95269063..954225df 100644 --- a/modules/nf-core/samtools/flagstat/meta.yml +++ b/modules/nf-core/samtools/flagstat/meta.yml @@ -14,7 +14,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf index e04e63e8..0b20aa4b 100644 --- a/modules/nf-core/samtools/index/main.nf +++ b/modules/nf-core/samtools/index/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_INDEX { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input) diff --git a/modules/nf-core/samtools/index/meta.yml b/modules/nf-core/samtools/index/meta.yml index e5cadbc2..8bd2fa6f 100644 --- a/modules/nf-core/samtools/index/meta.yml +++ b/modules/nf-core/samtools/index/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf index ab7f1cca..1e5181d4 100644 --- a/modules/nf-core/samtools/sort/main.nf +++ b/modules/nf-core/samtools/sort/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_SORT { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(bam) @@ -21,9 +21,17 @@ process SAMTOOLS_SORT { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def sort_memory = (task.memory.mega/task.cpus).intValue() if ("$bam" == "${prefix}.bam") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!" """ - samtools sort $args -@ $task.cpus -o ${prefix}.bam -T $prefix $bam + samtools sort \\ + $args \\ + -@ $task.cpus \\ + -m ${sort_memory}M \\ + -o ${prefix}.bam \\ + -T $prefix \\ + $bam + cat <<-END_VERSIONS > versions.yml "${task.process}": samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') diff --git a/modules/nf-core/samtools/sort/meta.yml b/modules/nf-core/samtools/sort/meta.yml index 09289751..07328431 100644 --- a/modules/nf-core/samtools/sort/meta.yml +++ b/modules/nf-core/samtools/sort/meta.yml @@ -12,7 +12,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: diff --git a/modules/nf-core/samtools/stats/main.nf b/modules/nf-core/samtools/stats/main.nf index 9b0c3867..eb7f098b 100644 --- a/modules/nf-core/samtools/stats/main.nf +++ b/modules/nf-core/samtools/stats/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_STATS { tag "$meta.id" label 'process_single' - conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) + conda "bioconda::samtools=1.17" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : - 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' : + 'biocontainers/samtools:1.17--h00cdaf9_0' }" input: tuple val(meta), path(input), path(input_index) diff --git a/modules/nf-core/samtools/stats/meta.yml b/modules/nf-core/samtools/stats/meta.yml index cac50b1c..1d68a5d8 100644 --- a/modules/nf-core/samtools/stats/meta.yml +++ b/modules/nf-core/samtools/stats/meta.yml @@ -13,7 +13,7 @@ tools: short DNA sequence read alignments in the SAM, BAM and CRAM formats, written by Heng Li. These files are generated as output by short read aligners like BWA. homepage: http://www.htslib.org/ - documentation: hhttp://www.htslib.org/doc/samtools.html + documentation: http://www.htslib.org/doc/samtools.html doi: 10.1093/bioinformatics/btp352 licence: ["MIT"] input: @@ -23,13 +23,13 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - input: - type: file - description: BAM/CRAM file from alignment - pattern: "*.{bam,cram}" + type: file + description: BAM/CRAM file from alignment + pattern: "*.{bam,cram}" - input_index: - type: file - description: BAI/CRAI file from alignment - pattern: "*.{bai,crai}" + type: file + description: BAI/CRAI file from alignment + pattern: "*.{bai,crai}" - fasta: type: optional file description: Reference file the CRAM was created with diff --git a/modules/nf-core/trimgalore/main.nf b/modules/nf-core/trimgalore/main.nf index 5b45e4d7..dcb77ae7 100644 --- a/modules/nf-core/trimgalore/main.nf +++ b/modules/nf-core/trimgalore/main.nf @@ -2,22 +2,21 @@ process TRIMGALORE { tag "$meta.id" label 'process_high' - conda (params.enable_conda ? 'bioconda::trim-galore=0.6.7' : null) + conda "bioconda::trim-galore=0.6.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/trim-galore:0.6.7--hdfd78af_0' : - 'quay.io/biocontainers/trim-galore:0.6.7--hdfd78af_0' }" + 'biocontainers/trim-galore:0.6.7--hdfd78af_0' }" input: tuple val(meta), path(reads) output: - tuple val(meta), path("*{trimmed,val}*.fq.gz"), emit: reads - tuple val(meta), path("*report.txt") , emit: log - path "versions.yml" , emit: versions - - tuple val(meta), path("*unpaired*.fq.gz") , emit: unpaired, optional: true - tuple val(meta), path("*.html") , emit: html , optional: true - tuple val(meta), path("*.zip") , emit: zip , optional: true + tuple val(meta), path("*{3prime,5prime,trimmed,val}*.fq.gz"), emit: reads + tuple val(meta), path("*report.txt") , emit: log , optional: true + tuple val(meta), path("*unpaired*.fq.gz") , emit: unpaired, optional: true + tuple val(meta), path("*.html") , emit: html , optional: true + tuple val(meta), path("*.zip") , emit: zip , optional: true + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -38,10 +37,12 @@ process TRIMGALORE { // Added soft-links to original fastqs for consistent naming in MultiQC def prefix = task.ext.prefix ?: "${meta.id}" if (meta.single_end) { + def args_list = args.split("\\s(?=--)").toList() + args_list.removeAll { it.toLowerCase().contains('_r2 ') } """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz trim_galore \\ - $args \\ + ${args_list.join(' ')} \\ --cores $cores \\ --gzip \\ ${prefix}.fastq.gz diff --git a/modules/nf-core/trimgalore/meta.yml b/modules/nf-core/trimgalore/meta.yml index 439f566d..f84c4d77 100644 --- a/modules/nf-core/trimgalore/meta.yml +++ b/modules/nf-core/trimgalore/meta.yml @@ -36,7 +36,7 @@ output: description: | List of input adapter trimmed FastQ files of size 1 and 2 for single-end and paired-end data, respectively. - pattern: "*.{fq.gz}" + pattern: "*{3prime,5prime,trimmed,val}*.fq.gz" - unpaired: type: file description: | From 0629fd08b5edfc294ab154eae00593fa34d293d5 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 16 May 2023 12:49:22 -0500 Subject: [PATCH 09/58] fixup! Merge branch 'dev' into nf-core-template-merge-2.8 --- nextflow_schema.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index f0296d71..5eb0af93 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -500,11 +500,6 @@ "description": "Show all params when using `--help`", "hidden": true, "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters." - }, - "enable_conda": { - "type": "boolean", - "description": "Run this workflow with Conda. You can also use '-profile condo' instead of providing this parameter.", - "hidden": true } } }, From 96385cd1a876c7a671342dc99e479145fa7dc253 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 16 May 2023 14:39:23 -0400 Subject: [PATCH 10/58] bump-nf-tests-snapshots-2.8.0 --- nf-test.config | 5 +- tests/bismark/main.nf.test | 88 +++++++++++++++++++-------- tests/bismark/main.nf.test.snap | 48 ++++++++------- tests/bismark_hisat/main.nf.test | 44 +++++++++----- tests/bismark_hisat/main.nf.test.snap | 36 +++++------ tests/bwameth/main.nf.test | 51 ++++++++++------ tests/bwameth/main.nf.test.snap | 42 +++++++------ tests/clipping/main.nf.test | 25 -------- tests/clipping/main.nf.test.snap | 8 --- tests/nextflow.config | 1 - 10 files changed, 194 insertions(+), 154 deletions(-) delete mode 100644 tests/clipping/main.nf.test delete mode 100644 tests/clipping/main.nf.test.snap delete mode 100644 tests/nextflow.config diff --git a/nf-test.config b/nf-test.config index d8fd6cbe..f978b336 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,7 +1,6 @@ config { - testsDir "tests" workDir ".nf-test" - profile "test,docker" - + configFile "nextflow.config" + profile "" } diff --git a/tests/bismark/main.nf.test b/tests/bismark/main.nf.test index 51567718..5695ef84 100644 --- a/tests/bismark/main.nf.test +++ b/tests/bismark/main.nf.test @@ -2,15 +2,17 @@ import com.askimed.nf.test.util.FileUtil; nextflow_pipeline { - name "bismark" + name "Test Workflow main.nf" script "main.nf" + profile "test,docker" + tag "pipeline" - test("Single End") { + test("Bismark Single End") { when { params { - aligner = "bismark" + aligner = "bismark" save_reference = true - outdir = "$outputDir" + outdir = "$outputDir" } } @@ -24,19 +26,21 @@ nextflow_pipeline { new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() ) - // Test results - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() - assert new File("$outputDir/multiqc/bismark/multiqc_report.html").exists() - - assert new File("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt").exists() - assert new File("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt").exists() - assert path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt").md5 == "3038a90a6e513c3b1c0de540439eb101" + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) } } - test("Single End with index") { + test("Bismark Single End With Index") { when { params { aligner = "bismark" @@ -49,12 +53,10 @@ nextflow_pipeline { then { assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() } } - test("Single End RRBS") { + test("Bismark Single End With RRBS") { when { params { aligner = "bismark" @@ -66,19 +68,26 @@ nextflow_pipeline { } then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_bismark_bt2_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2.merged_bismark_bt2_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_rrbs") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) } } - test("Single End RRBS with index") { + test("Bismark Single End With RRBS With Index") { when { params { aligner = "bismark" skip_trimming = true rrbs = true - save_reference = true // Generated by previous test bismark_index = ".nf-test/indexes/BismarkIndex/" cytosine_report = true @@ -88,12 +97,10 @@ nextflow_pipeline { then { assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() } } - test("Single End NOMe-seq") { + test("Bismark Single End NOMe-seq") { when { params { aligner = "bismark" @@ -103,8 +110,35 @@ nextflow_pipeline { } then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_nomeseq") }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) + } + } + + test("Bismark Clipping Params") { + when { + params { + aligner = "bismark" + em_seq = true + clip_r1 = 2 + outdir = "$outputDir" + } + } + + then { + assertAll( + { assert workflow.success }, + { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } + ) } } diff --git a/tests/bismark/main.nf.test.snap b/tests/bismark/main.nf.test.snap index f7ba587c..4c5609dc 100644 --- a/tests/bismark/main.nf.test.snap +++ b/tests/bismark/main.nf.test.snap @@ -1,32 +1,38 @@ { - "Single End RRBS with index": { + "single_end_nomeseq": { "content": [ - 32 + "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,292c73be0dfaad92ee7d54ec3e5d0dd6", + "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,d70ee229a29b68db8634fa7d568b7c53", + "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", + "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", + "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", + "genome_results.txt:md5,29e911b75d7160e964530e9607110164", + "software_versions.yml:md5,f281f6db0fb109fd50575688149859e3" ], - "timestamp": "2022-12-07T14:59:14+0000" + "timestamp": "2023-04-25T16:12:43+0000" }, - "Single End NOMe-seq": { + "single_end": { "content": [ - 39 + "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,292c73be0dfaad92ee7d54ec3e5d0dd6", + "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,d70ee229a29b68db8634fa7d568b7c53", + "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", + "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", + "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", + "genome_results.txt:md5,29e911b75d7160e964530e9607110164", + "software_versions.yml:md5,f281f6db0fb109fd50575688149859e3" ], - "timestamp": "2022-12-07T14:59:14+0000" + "timestamp": "2023-04-25T16:12:43+0000" }, - "Single End with index": { + "single_end_rrbs": { "content": [ - 38 + "Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt:md5,9f72ee663677fb74fde3752b08350858", + "SRR389222_sub1_bismark_bt2_splitting_report.txt:md5,f59eac7a30ec7f1f690aac921a5591c0", + "SRR389222_sub2.merged_bismark_bt2_splitting_report.txt:md5,43ca6c76e47ce2eb1c8dd6027475b5a6", + "genome_results.txt:md5,41becdcecee6ebbdd46f472629731892", + "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", + "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e", + "software_versions.yml:md5,986af83c1bf68311b22432ed217161bd" ], - "timestamp": "2022-12-07T14:59:14+0000" - }, - "Single End": { - "content": [ - 36 - ], - "timestamp": "2022-12-07T14:59:14+0000" - }, - "Single End RRBS": { - "content": [ - 30 - ], - "timestamp": "2022-12-07T14:59:14+0000" + "timestamp": "2023-04-25T16:12:43+0000" } } \ No newline at end of file diff --git a/tests/bismark_hisat/main.nf.test b/tests/bismark_hisat/main.nf.test index 0daa6f7c..645b2dc5 100644 --- a/tests/bismark_hisat/main.nf.test +++ b/tests/bismark_hisat/main.nf.test @@ -2,10 +2,12 @@ import com.askimed.nf.test.util.FileUtil; nextflow_pipeline { - name "bismark_hisat" + name "Test Workflow main.nf" script "main.nf" + profile "test,docker" + tag "pipeline" - test("Single End") { + test("Bismark Hisat Single End") { when { params { aligner = "bismark_hisat" @@ -24,13 +26,21 @@ nextflow_pipeline { new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() ) - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub1_trimmed_bismark_hisat2.deduplicated_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end") }, + { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } + ) } } - test("Single End with index") { + test("Bismark Hisat Single End With Index") { when { params { aligner = "bismark_hisat" @@ -43,12 +53,10 @@ nextflow_pipeline { then { assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() } } - test("Single End RRBS") { + test("Bismark Hisat Single End With RRBS") { when { params { aligner = "bismark_hisat" @@ -60,13 +68,21 @@ nextflow_pipeline { } then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub1_bismark_hisat2_splitting_report.txt"), + path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_rrbs") }, + { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } + ) } } - test("Single End RRBS with index") { + test("Bismark Hisat Single End With RRBS With Index") { when { params { aligner = "bismark_hisat" @@ -81,8 +97,6 @@ nextflow_pipeline { then { assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() } } diff --git a/tests/bismark_hisat/main.nf.test.snap b/tests/bismark_hisat/main.nf.test.snap index 38d4603c..89f048f1 100644 --- a/tests/bismark_hisat/main.nf.test.snap +++ b/tests/bismark_hisat/main.nf.test.snap @@ -1,26 +1,26 @@ { - "Single End RRBS with index": { + "single_end": { "content": [ - 29 + "Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt:md5,4e3ab3792be5eedde05d468a7e7f17e4", + "SRR389222_sub1_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4020063dd8610b58b7b39c3ca62cef17", + "SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4db2155fdfae1cb9c7c758edba256e2e", + "genome_results.txt:md5,cd5d3b03821c038eb3bda24beb3ae938", + "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", + "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec", + "software_versions.yml:md5,f281f6db0fb109fd50575688149859e3" ], - "timestamp": "2022-12-07T15:38:47+0000" + "timestamp": "2023-04-25T16:37:08+0000" }, - "Single End with index": { + "single_end_rrbs": { "content": [ - 35 + "Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt:md5,f59c6bf8089c7dde3225cc577e0260e8", + "SRR389222_sub1_bismark_hisat2_splitting_report.txt:md5,754ed7bf4d75acee466c48c05a11d331", + "SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt:md5,0629e1f8b4727efe8fd69500783eec61", + "genome_results.txt:md5,3afd1eb7d4dc72a05e4d4de887d791e3", + "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", + "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02", + "software_versions.yml:md5,986af83c1bf68311b22432ed217161bd" ], - "timestamp": "2022-12-07T15:38:47+0000" - }, - "Single End": { - "content": [ - 36 - ], - "timestamp": "2022-12-07T15:38:47+0000" - }, - "Single End RRBS": { - "content": [ - 30 - ], - "timestamp": "2022-12-07T15:38:47+0000" + "timestamp": "2023-04-25T16:37:08+0000" } } \ No newline at end of file diff --git a/tests/bwameth/main.nf.test b/tests/bwameth/main.nf.test index 53b14dfa..8e83bf84 100644 --- a/tests/bwameth/main.nf.test +++ b/tests/bwameth/main.nf.test @@ -2,10 +2,12 @@ import com.askimed.nf.test.util.FileUtil; nextflow_pipeline { - name "bwameth" + name "Test Workflow main.nf" script "main.nf" + profile "test,docker" + tag "pipeline" - test("Single End") { + test("Bwameth Single End") { when { params { aligner = "bwameth" @@ -24,16 +26,23 @@ nextflow_pipeline { new File(".nf-test/indexes/bwameth/").getAbsolutePath() ) - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - assert new File("$outputDir/multiqc/bwameth/multiqc_report.html").exists() - assert new File("$outputDir/bwameth/deduplicated/SRR389222_sub1.markdup.sorted.bam").exists() - assert new File("$outputDir/bwameth/reference_genome/bwameth/genome.fa.bwameth.c2t").exists() - // FIXME assert snapshot(workflow).match() + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub1.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub2.flagstat"), + path("$outputDir/methyldackel/Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub1.markdup.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub2.markdup.sorted_CpG.bedGraph"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end") } + ) } } - test("Single End with index") { + test("Bwameth Single End With Index") { when { params { aligner = "bwameth" @@ -46,12 +55,10 @@ nextflow_pipeline { then { assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() } } - test("Single End RRBS") { + test("Bwameth Single End RRBS") { when { params { aligner = "bwameth" @@ -64,13 +71,23 @@ nextflow_pipeline { } then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() + assertAll( + { assert workflow.success }, + { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub1.flagstat"), + path("$outputDir/bwameth/alignments/samtools_stats/SRR389222_sub2.flagstat"), + path("$outputDir/methyldackel/Ecoli_10K_methylated.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub1.sorted_CpG.bedGraph"), + path("$outputDir/methyldackel/SRR389222_sub2.sorted_CpG.bedGraph"), + path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), + path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_rrbs") } + ) } } - test("Single End RRBS with index") { + test("Bwameth Single End RRBS With Index") { when { params { aligner = "bwameth" @@ -85,8 +102,6 @@ nextflow_pipeline { then { assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // FIXME assert snapshot(workflow).match() } } diff --git a/tests/bwameth/main.nf.test.snap b/tests/bwameth/main.nf.test.snap index b9ed3a35..0cf17518 100644 --- a/tests/bwameth/main.nf.test.snap +++ b/tests/bwameth/main.nf.test.snap @@ -1,26 +1,32 @@ { - "Single End RRBS with index": { + "single_end": { "content": [ - 34 + "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", + "SRR389222_sub1.flagstat:md5,ad8159a7683b9c3057bf54ccb77a2ddc", + "SRR389222_sub2.flagstat:md5,2166a52f81f9b3881e8ac060876aa2d9", + "Ecoli_10K_methylated.markdup.sorted_CpG.bedGraph:md5,d3bce01078b6e7749f3906bf9faad5b5", + "SRR389222_sub1.markdup.sorted_CpG.bedGraph:md5,0944ad0e3b05791bf6dfd06be7ca4af1", + "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", + "genome_results.txt:md5,1ee2e29ac41c2cd7e6a0f00741e8d16e", + "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", + "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0", + "software_versions.yml:md5,e08c216cc6de759f0a6dc82d96a9a014" ], - "timestamp": "2022-12-07T15:05:21+0000" + "timestamp": "2023-04-25T16:44:12+0000" }, - "Single End with index": { + "single_end_rrbs": { "content": [ - 43 + "Ecoli_10K_methylated.flagstat:md5,7c3bae0f9c700dde88785a7e94c6ae2c", + "SRR389222_sub1.flagstat:md5,897d500a710a56a7098172167fa71108", + "SRR389222_sub2.flagstat:md5,2d680c5657fc029b703b217be4af68e2", + "Ecoli_10K_methylated.sorted_CpG.bedGraph:md5,3656c54896013b76de978ab95c3f5975", + "SRR389222_sub1.sorted_CpG.bedGraph:md5,4475a9462ce86a019015b69b82a9d5a2", + "SRR389222_sub2.sorted_CpG.bedGraph:md5,18b8845d9d985d3c203ca6c9c40aab1b", + "genome_results.txt:md5,ca30867e16c5ca25c850c81d88563d41", + "genome_results.txt:md5,f01c68dd46ca9ec0af3d6ec926763b75", + "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0", + "software_versions.yml:md5,206a9e52256fca71c3ca921ac8ef78f7" ], - "timestamp": "2022-12-07T15:05:21+0000" - }, - "Single End": { - "content": [ - 44 - ], - "timestamp": "2022-12-07T15:05:21+0000" - }, - "Single End RRBS": { - "content": [ - 35 - ], - "timestamp": "2022-12-07T15:05:21+0000" + "timestamp": "2023-04-25T16:44:12+0000" } } \ No newline at end of file diff --git a/tests/clipping/main.nf.test b/tests/clipping/main.nf.test deleted file mode 100644 index 6e1cc9c9..00000000 --- a/tests/clipping/main.nf.test +++ /dev/null @@ -1,25 +0,0 @@ -nextflow_pipeline { - - name "Clipping" - script "main.nf" - - test("Should run with clipping params") { - when { - params { - aligner = "bismark" - em_seq = true - clip_r1 = 2 - outdir = "$outputDir" - } - } - - then { - assert workflow.success - assert snapshot(workflow.trace.tasks().size()).match() - // TODO NFCORE_METHYLSEQ:METHYLSEQ:TRIMGALORE should include - // --three_prime_clip_r1 10 from the --em_seq preset - // --clip_r1 2 correctly overriding from --clip_r1 - } - } - -} diff --git a/tests/clipping/main.nf.test.snap b/tests/clipping/main.nf.test.snap deleted file mode 100644 index dde3180a..00000000 --- a/tests/clipping/main.nf.test.snap +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Should run with clipping params": { - "content": [ - 36 - ], - "timestamp": "2022-12-07T15:00:07+0000" - } -} \ No newline at end of file diff --git a/tests/nextflow.config b/tests/nextflow.config deleted file mode 100644 index 8c256c7b..00000000 --- a/tests/nextflow.config +++ /dev/null @@ -1 +0,0 @@ -// This file is left intentionally blank From ff457ea5e558258e1da4dbfa2a41995aa3026c3f Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 16 May 2023 16:45:28 -0400 Subject: [PATCH 11/58] update-ci-test-min-nxf-ver --- .github/workflows/ci.yml | 39 +++++++++++++++++++-------- tests/bismark/main.nf.test.snap | 12 ++++----- tests/bismark_hisat/main.nf.test.snap | 8 +++--- tests/bwameth/main.nf.test.snap | 8 +++--- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7652b0f8..d4c92f13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,25 +16,42 @@ concurrency: cancel-in-progress: true jobs: + get_version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.get_version.outputs.version }} + + steps: + - name: Check out pipeline code + uses: actions/checkout@v3 + + - name: Get Minimum supported Nextflow version + id: get_version + run: | + VERSION=$(grep 'nextflowVersion' nextflow.config | awk -F'= ' '{print $2}' | tr -d '!>=" ') + echo "::set-output name=version::$VERSION" + test: - name: Run ${{ matrix.aligner }}-${{ matrix.profile }}-${{ matrix.NXF_VER }} - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + name: Run ${{ matrix.profile }} pipeline test + needs: get_version + + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/fetchngs') }}" + runs-on: ubuntu-latest + env: + NXF_VER: ${{ needs.get_version.outputs.version }} + strategy: matrix: - NXF_VER: - - "22.10.1" - - "latest-everything" - aligner: - - bismark - - bismark_hisat - - bwameth - profile: ["docker"] # TODO , "singularity", "conda"] + profile: ["docker"] + steps: - name: Check out pipeline code uses: actions/checkout@v3 + - name: Use Nextflow version + run: echo "Using Nextflow version $NXF_VER" + - name: Cache test data id: cache-testdata uses: actions/cache@v3 diff --git a/tests/bismark/main.nf.test.snap b/tests/bismark/main.nf.test.snap index 4c5609dc..6d3a563b 100644 --- a/tests/bismark/main.nf.test.snap +++ b/tests/bismark/main.nf.test.snap @@ -7,9 +7,9 @@ "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164", - "software_versions.yml:md5,f281f6db0fb109fd50575688149859e3" + "software_versions.yml:md5,be66a55d78874ca301b4043fa994d846" ], - "timestamp": "2023-04-25T16:12:43+0000" + "timestamp": "2023-05-16T20:24:22+0000" }, "single_end": { "content": [ @@ -19,9 +19,9 @@ "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164", - "software_versions.yml:md5,f281f6db0fb109fd50575688149859e3" + "software_versions.yml:md5,be66a55d78874ca301b4043fa994d846" ], - "timestamp": "2023-04-25T16:12:43+0000" + "timestamp": "2023-05-16T20:24:22+0000" }, "single_end_rrbs": { "content": [ @@ -31,8 +31,8 @@ "genome_results.txt:md5,41becdcecee6ebbdd46f472629731892", "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e", - "software_versions.yml:md5,986af83c1bf68311b22432ed217161bd" + "software_versions.yml:md5,0660a77e92fd716304141e5e1bfc1f40" ], - "timestamp": "2023-04-25T16:12:43+0000" + "timestamp": "2023-05-16T20:24:22+0000" } } \ No newline at end of file diff --git a/tests/bismark_hisat/main.nf.test.snap b/tests/bismark_hisat/main.nf.test.snap index 89f048f1..35ac35cf 100644 --- a/tests/bismark_hisat/main.nf.test.snap +++ b/tests/bismark_hisat/main.nf.test.snap @@ -7,9 +7,9 @@ "genome_results.txt:md5,cd5d3b03821c038eb3bda24beb3ae938", "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec", - "software_versions.yml:md5,f281f6db0fb109fd50575688149859e3" + "software_versions.yml:md5,be66a55d78874ca301b4043fa994d846" ], - "timestamp": "2023-04-25T16:37:08+0000" + "timestamp": "2023-05-16T20:34:34+0000" }, "single_end_rrbs": { "content": [ @@ -19,8 +19,8 @@ "genome_results.txt:md5,3afd1eb7d4dc72a05e4d4de887d791e3", "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02", - "software_versions.yml:md5,986af83c1bf68311b22432ed217161bd" + "software_versions.yml:md5,0660a77e92fd716304141e5e1bfc1f40" ], - "timestamp": "2023-04-25T16:37:08+0000" + "timestamp": "2023-05-16T20:34:34+0000" } } \ No newline at end of file diff --git a/tests/bwameth/main.nf.test.snap b/tests/bwameth/main.nf.test.snap index 0cf17518..c3e730d9 100644 --- a/tests/bwameth/main.nf.test.snap +++ b/tests/bwameth/main.nf.test.snap @@ -10,9 +10,9 @@ "genome_results.txt:md5,1ee2e29ac41c2cd7e6a0f00741e8d16e", "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0", - "software_versions.yml:md5,e08c216cc6de759f0a6dc82d96a9a014" + "software_versions.yml:md5,512d88ff90ba5c782b5841521f132b01" ], - "timestamp": "2023-04-25T16:44:12+0000" + "timestamp": "2023-05-16T20:43:17+0000" }, "single_end_rrbs": { "content": [ @@ -25,8 +25,8 @@ "genome_results.txt:md5,ca30867e16c5ca25c850c81d88563d41", "genome_results.txt:md5,f01c68dd46ca9ec0af3d6ec926763b75", "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0", - "software_versions.yml:md5,206a9e52256fca71c3ca921ac8ef78f7" + "software_versions.yml:md5,72fa60f30d868c1f2e903c333bbc0ecf" ], - "timestamp": "2023-04-25T16:44:12+0000" + "timestamp": "2023-05-16T20:43:17+0000" } } \ No newline at end of file From e8fa8a90b8b977ea08fbe1d05991f13038494f34 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 16 May 2023 17:47:39 -0400 Subject: [PATCH 12/58] update-ci-nf-test-path --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4c92f13..424097c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,10 +107,7 @@ jobs: - name: Run nf-test run: | - nf-test test \ - --profile=test_ci,test,${{ matrix.profile }} \ - tests/${{ matrix.aligner }}/*.nf.test \ - --tap=test.tap + nf-test test tests/ --tap=test.tap - uses: pcolby/tap-summary@v1 with: From 4007189d18b6f42eb76cfc140e92992f66cecd28 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 16 May 2023 18:57:28 -0400 Subject: [PATCH 13/58] update-ci-use-GITHUB_OUTPUT-nxf-ver --- .github/workflows/ci.yml | 4 ++-- .nf-core.yml | 2 ++ tests/{ => pipeline}/bismark/main.nf.test | 0 tests/{ => pipeline}/bismark/main.nf.test.snap | 0 tests/{ => pipeline}/bismark_hisat/main.nf.test | 0 tests/{ => pipeline}/bismark_hisat/main.nf.test.snap | 0 tests/{ => pipeline}/bwameth/main.nf.test | 0 tests/{ => pipeline}/bwameth/main.nf.test.snap | 0 8 files changed, 4 insertions(+), 2 deletions(-) rename tests/{ => pipeline}/bismark/main.nf.test (100%) rename tests/{ => pipeline}/bismark/main.nf.test.snap (100%) rename tests/{ => pipeline}/bismark_hisat/main.nf.test (100%) rename tests/{ => pipeline}/bismark_hisat/main.nf.test.snap (100%) rename tests/{ => pipeline}/bwameth/main.nf.test (100%) rename tests/{ => pipeline}/bwameth/main.nf.test.snap (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 424097c7..5e8778b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: id: get_version run: | VERSION=$(grep 'nextflowVersion' nextflow.config | awk -F'= ' '{print $2}' | tr -d '!>=" ') - echo "::set-output name=version::$VERSION" + echo "nxf_version=$VERSION" >> $GITHUB_OUTPUT test: name: Run ${{ matrix.profile }} pipeline test @@ -107,7 +107,7 @@ jobs: - name: Run nf-test run: | - nf-test test tests/ --tap=test.tap + nf-test test tests/pipeline/ --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/.nf-core.yml b/.nf-core.yml index 83758400..572161fb 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -2,4 +2,6 @@ actions_ci: False files_exist: - bin/markdown_to_html.py +files_unchanged: + - .gitattributes repository_type: pipeline diff --git a/tests/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test similarity index 100% rename from tests/bismark/main.nf.test rename to tests/pipeline/bismark/main.nf.test diff --git a/tests/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap similarity index 100% rename from tests/bismark/main.nf.test.snap rename to tests/pipeline/bismark/main.nf.test.snap diff --git a/tests/bismark_hisat/main.nf.test b/tests/pipeline/bismark_hisat/main.nf.test similarity index 100% rename from tests/bismark_hisat/main.nf.test rename to tests/pipeline/bismark_hisat/main.nf.test diff --git a/tests/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap similarity index 100% rename from tests/bismark_hisat/main.nf.test.snap rename to tests/pipeline/bismark_hisat/main.nf.test.snap diff --git a/tests/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test similarity index 100% rename from tests/bwameth/main.nf.test rename to tests/pipeline/bwameth/main.nf.test diff --git a/tests/bwameth/main.nf.test.snap b/tests/pipeline/bwameth/main.nf.test.snap similarity index 100% rename from tests/bwameth/main.nf.test.snap rename to tests/pipeline/bwameth/main.nf.test.snap From 23eb11e60dc8d355d2dae00826b20c96575aa946 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 16 May 2023 19:30:25 -0400 Subject: [PATCH 14/58] fix-nf-core-yml-lint-actions-ci-false --- .nf-core.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.nf-core.yml b/.nf-core.yml index 572161fb..798bca42 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,7 +1,8 @@ # FIXME - CI currently runs for dev and dsl2 branches, remove the latter before merging to dev -actions_ci: False -files_exist: - - bin/markdown_to_html.py -files_unchanged: - - .gitattributes repository_type: pipeline +lint: + files_unchanged: + - .gitattributes + files_exist: + - bin/markdown_to_html.py + actions_ci: False From bf8255d62b7f324bca4547a0a0d42af86da10f36 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 16 May 2023 19:37:14 -0400 Subject: [PATCH 15/58] ci-fix-nxf-ver-output --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e8778b0..f8ae144a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: id: get_version run: | VERSION=$(grep 'nextflowVersion' nextflow.config | awk -F'= ' '{print $2}' | tr -d '!>=" ') - echo "nxf_version=$VERSION" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT test: name: Run ${{ matrix.profile }} pipeline test From bd8e4d29eb537e87b660fa598a0673435c6e4846 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 24 May 2023 00:43:54 -0400 Subject: [PATCH 16/58] revert-to-testing-multiple-nxf-vers --- .github/workflows/ci.yml | 48 +++++++---------- nf-test.config | 2 +- tests/pipeline/bismark/main.nf.test | 30 ++++++++--- tests/pipeline/bismark/main.nf.test.snap | 51 +++++++++++++++---- tests/pipeline/bismark_hisat/main.nf.test | 17 +++++-- .../pipeline/bismark_hisat/main.nf.test.snap | 42 ++++++++++++--- tests/pipeline/bwameth/main.nf.test | 17 +++++-- tests/pipeline/bwameth/main.nf.test.snap | 42 ++++++++++++--- 8 files changed, 181 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8ae144a..5f7d868a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,42 +16,25 @@ concurrency: cancel-in-progress: true jobs: - get_version: - runs-on: ubuntu-latest - outputs: - version: ${{ steps.get_version.outputs.version }} - - steps: - - name: Check out pipeline code - uses: actions/checkout@v3 - - - name: Get Minimum supported Nextflow version - id: get_version - run: | - VERSION=$(grep 'nextflowVersion' nextflow.config | awk -F'= ' '{print $2}' | tr -d '!>=" ') - echo "version=$VERSION" >> $GITHUB_OUTPUT - test: - name: Run ${{ matrix.profile }} pipeline test - needs: get_version - - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/fetchngs') }}" - + name: Run ${{ matrix.aligner }}-${{ matrix.profile }}-${{ matrix.NXF_VER }} + # Only run on push if this is the nf-core dev branch (merged PRs) + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" runs-on: ubuntu-latest - env: - NXF_VER: ${{ needs.get_version.outputs.version }} - strategy: matrix: - profile: ["docker"] - + NXF_VER: + - "22.10.1" + - "latest-everything" + aligner: + - bismark + - bismark_hisat + - bwameth + profile: ["docker"] # TODO , "singularity", "conda"] steps: - name: Check out pipeline code uses: actions/checkout@v3 - - name: Use Nextflow version - run: echo "Using Nextflow version $NXF_VER" - - name: Cache test data id: cache-testdata uses: actions/cache@v3 @@ -107,7 +90,14 @@ jobs: - name: Run nf-test run: | - nf-test test tests/pipeline/ --tap=test.tap + nf-test test tests/pipeline/${{ matrix.aligner }}/ --tap=test.tap + + # If the test fails, output the software_versions.yml using the 'batcat' utility + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /home/runner/work/fetchngs/fetchngs/.nf-test/tests/*/output/pipeline_info/software_versions.yml - uses: pcolby/tap-summary@v1 with: diff --git a/nf-test.config b/nf-test.config index f978b336..7d6adb94 100644 --- a/nf-test.config +++ b/nf-test.config @@ -2,5 +2,5 @@ config { testsDir "tests" workDir ".nf-test" configFile "nextflow.config" - profile "" + profile "test,docker" } diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test index 5695ef84..456687aa 100644 --- a/tests/pipeline/bismark/main.nf.test +++ b/tests/pipeline/bismark/main.nf.test @@ -4,7 +4,6 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" - profile "test,docker" tag "pipeline" test("Bismark Single End") { @@ -26,6 +25,10 @@ nextflow_pipeline { new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() ) + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), @@ -33,8 +36,8 @@ nextflow_pipeline { path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end") }, + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, + { assert snapshot(softwareVersions).match("software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -68,6 +71,10 @@ nextflow_pipeline { } then { + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt"), @@ -75,8 +82,8 @@ nextflow_pipeline { path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2.merged_bismark_bt2_splitting_report.txt"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_rrbs") }, + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, + { assert snapshot(softwareVersions).match("software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -110,6 +117,10 @@ nextflow_pipeline { } then { + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), @@ -117,8 +128,8 @@ nextflow_pipeline { path("$outputDir/bismark/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_nomeseq") }, + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_nomeseq") }, + { assert snapshot(softwareVersions).match("software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -135,8 +146,13 @@ nextflow_pipeline { } then { + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, + { assert snapshot(softwareVersions).match("software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } diff --git a/tests/pipeline/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap index 6d3a563b..bbc83ba6 100644 --- a/tests/pipeline/bismark/main.nf.test.snap +++ b/tests/pipeline/bismark/main.nf.test.snap @@ -1,4 +1,40 @@ { + "software_versions": { + "content": [ + { + "BISMARK_ALIGN": { + "bismark": "0.24.0" + }, + "BISMARK_GENOMEPREPARATION": { + "bismark": "0.24.0" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "TRIMGALORE": { + "cutadapt": "3.4", + "trimgalore": "0.6.7" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T04:26:11+0000" + }, "single_end_nomeseq": { "content": [ "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,292c73be0dfaad92ee7d54ec3e5d0dd6", @@ -6,10 +42,9 @@ "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", - "genome_results.txt:md5,29e911b75d7160e964530e9607110164", - "software_versions.yml:md5,be66a55d78874ca301b4043fa994d846" + "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-16T20:24:22+0000" + "timestamp": "2023-05-24T04:26:11+0000" }, "single_end": { "content": [ @@ -18,10 +53,9 @@ "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", - "genome_results.txt:md5,29e911b75d7160e964530e9607110164", - "software_versions.yml:md5,be66a55d78874ca301b4043fa994d846" + "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-16T20:24:22+0000" + "timestamp": "2023-05-24T04:26:11+0000" }, "single_end_rrbs": { "content": [ @@ -30,9 +64,8 @@ "SRR389222_sub2.merged_bismark_bt2_splitting_report.txt:md5,43ca6c76e47ce2eb1c8dd6027475b5a6", "genome_results.txt:md5,41becdcecee6ebbdd46f472629731892", "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", - "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e", - "software_versions.yml:md5,0660a77e92fd716304141e5e1bfc1f40" + "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e" ], - "timestamp": "2023-05-16T20:24:22+0000" + "timestamp": "2023-05-24T04:26:11+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bismark_hisat/main.nf.test b/tests/pipeline/bismark_hisat/main.nf.test index 645b2dc5..a7d21503 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test +++ b/tests/pipeline/bismark_hisat/main.nf.test @@ -4,7 +4,6 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" - profile "test,docker" tag "pipeline" test("Bismark Hisat Single End") { @@ -26,6 +25,10 @@ nextflow_pipeline { new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() ) + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt"), @@ -33,8 +36,8 @@ nextflow_pipeline { path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end") }, + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, + { assert snapshot(softwareVersions).match("software_versions") }, { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } ) } @@ -68,6 +71,10 @@ nextflow_pipeline { } then { + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt"), @@ -75,8 +82,8 @@ nextflow_pipeline { path("$outputDir/bismark_hisat/methylation_calls/splitting_report/SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_rrbs") }, + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, + { assert snapshot(softwareVersions).match("software_versions") }, { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } ) } diff --git a/tests/pipeline/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap index 35ac35cf..59126776 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test.snap +++ b/tests/pipeline/bismark_hisat/main.nf.test.snap @@ -1,4 +1,36 @@ { + "software_versions": { + "content": [ + { + "BISMARK_ALIGN": { + "bismark": "0.24.0" + }, + "BISMARK_GENOMEPREPARATION": { + "bismark": "0.24.0" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T04:34:55+0000" + }, "single_end": { "content": [ "Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt:md5,4e3ab3792be5eedde05d468a7e7f17e4", @@ -6,10 +38,9 @@ "SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4db2155fdfae1cb9c7c758edba256e2e", "genome_results.txt:md5,cd5d3b03821c038eb3bda24beb3ae938", "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", - "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec", - "software_versions.yml:md5,be66a55d78874ca301b4043fa994d846" + "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec" ], - "timestamp": "2023-05-16T20:34:34+0000" + "timestamp": "2023-05-24T04:34:55+0000" }, "single_end_rrbs": { "content": [ @@ -18,9 +49,8 @@ "SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt:md5,0629e1f8b4727efe8fd69500783eec61", "genome_results.txt:md5,3afd1eb7d4dc72a05e4d4de887d791e3", "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", - "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02", - "software_versions.yml:md5,0660a77e92fd716304141e5e1bfc1f40" + "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02" ], - "timestamp": "2023-05-16T20:34:34+0000" + "timestamp": "2023-05-24T04:34:55+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test index 8e83bf84..5cd33c9b 100644 --- a/tests/pipeline/bwameth/main.nf.test +++ b/tests/pipeline/bwameth/main.nf.test @@ -4,7 +4,6 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" - profile "test,docker" tag "pipeline" test("Bwameth Single End") { @@ -26,6 +25,10 @@ nextflow_pipeline { new File(".nf-test/indexes/bwameth/").getAbsolutePath() ) + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), @@ -36,8 +39,8 @@ nextflow_pipeline { path("$outputDir/methyldackel/SRR389222_sub2.markdup.sorted_CpG.bedGraph"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end") } + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, + { assert snapshot(softwareVersions).match("software_versions") } ) } } @@ -71,6 +74,10 @@ nextflow_pipeline { } then { + // Remove Nextflow version from software_versions.yml + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } + assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), @@ -81,8 +88,8 @@ nextflow_pipeline { path("$outputDir/methyldackel/SRR389222_sub2.sorted_CpG.bedGraph"), path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), - path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt"), - path("$outputDir/pipeline_info/software_versions.yml")).match("single_end_rrbs") } + path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, + { assert snapshot(softwareVersions).match("software_versions") } ) } } diff --git a/tests/pipeline/bwameth/main.nf.test.snap b/tests/pipeline/bwameth/main.nf.test.snap index c3e730d9..af5853bf 100644 --- a/tests/pipeline/bwameth/main.nf.test.snap +++ b/tests/pipeline/bwameth/main.nf.test.snap @@ -1,4 +1,36 @@ { + "software_versions": { + "content": [ + { + "BWAMETH_ALIGN": { + "bwameth": "0.2.2" + }, + "BWAMETH_INDEX": { + "bwameth": "0.2.2" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T04:42:22+0000" + }, "single_end": { "content": [ "Ecoli_10K_methylated.flagstat:md5,f520e87cc4050799a0dbed72909572ec", @@ -9,10 +41,9 @@ "SRR389222_sub2.markdup.sorted_CpG.bedGraph:md5,0a3c19af55d1625e91903ae25d1d2098", "genome_results.txt:md5,1ee2e29ac41c2cd7e6a0f00741e8d16e", "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", - "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0", - "software_versions.yml:md5,512d88ff90ba5c782b5841521f132b01" + "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0" ], - "timestamp": "2023-05-16T20:43:17+0000" + "timestamp": "2023-05-24T04:42:22+0000" }, "single_end_rrbs": { "content": [ @@ -24,9 +55,8 @@ "SRR389222_sub2.sorted_CpG.bedGraph:md5,18b8845d9d985d3c203ca6c9c40aab1b", "genome_results.txt:md5,ca30867e16c5ca25c850c81d88563d41", "genome_results.txt:md5,f01c68dd46ca9ec0af3d6ec926763b75", - "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0", - "software_versions.yml:md5,72fa60f30d868c1f2e903c333bbc0ecf" + "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0" ], - "timestamp": "2023-05-16T20:43:17+0000" + "timestamp": "2023-05-24T04:42:22+0000" } } \ No newline at end of file From 7cd0ae5f7d6a73fd80cb0b90818dcc3974664ff9 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 24 May 2023 00:45:41 -0400 Subject: [PATCH 17/58] fix-trailing-whitespace --- tests/pipeline/bwameth/main.nf.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipeline/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test index 5cd33c9b..f885f129 100644 --- a/tests/pipeline/bwameth/main.nf.test +++ b/tests/pipeline/bwameth/main.nf.test @@ -40,7 +40,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("software_versions") } + { assert snapshot(softwareVersions).match("software_versions") } ) } } @@ -89,7 +89,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("software_versions") } + { assert snapshot(softwareVersions).match("software_versions") } ) } } From 45349b2581174b5d4b7c6cdb6ca730be10fe0c01 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 24 May 2023 10:39:11 -0400 Subject: [PATCH 18/58] update-snapshots --- tests/pipeline/bismark/main.nf.test | 8 +- tests/pipeline/bismark/main.nf.test.snap | 114 +++++++++++++++++- tests/pipeline/bismark_hisat/main.nf.test | 4 +- .../pipeline/bismark_hisat/main.nf.test.snap | 44 ++++++- tests/pipeline/bwameth/main.nf.test | 4 +- tests/pipeline/bwameth/main.nf.test.snap | 44 ++++++- 6 files changed, 197 insertions(+), 21 deletions(-) diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test index 456687aa..ee2189a2 100644 --- a/tests/pipeline/bismark/main.nf.test +++ b/tests/pipeline/bismark/main.nf.test @@ -37,7 +37,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("software_versions") }, + { assert snapshot(softwareVersions).match("single_end_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -83,7 +83,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("software_versions") }, + { assert snapshot(softwareVersions).match("single_end_rrbs_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -129,7 +129,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_nomeseq") }, - { assert snapshot(softwareVersions).match("software_versions") }, + { assert snapshot(softwareVersions).match("single_end_nomeseq_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -152,7 +152,7 @@ nextflow_pipeline { assertAll( { assert workflow.success }, - { assert snapshot(softwareVersions).match("software_versions") }, + { assert snapshot(softwareVersions).match("clipping_params_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } diff --git a/tests/pipeline/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap index bbc83ba6..833588ae 100644 --- a/tests/pipeline/bismark/main.nf.test.snap +++ b/tests/pipeline/bismark/main.nf.test.snap @@ -1,5 +1,5 @@ { - "software_versions": { + "clipping_params_software_versions": { "content": [ { "BISMARK_ALIGN": { @@ -33,7 +33,79 @@ } } ], - "timestamp": "2023-05-24T04:26:11+0000" + "timestamp": "2023-05-24T14:17:30+0000" + }, + "single_end_software_versions": { + "content": [ + { + "BISMARK_ALIGN": { + "bismark": "0.24.0" + }, + "BISMARK_GENOMEPREPARATION": { + "bismark": "0.24.0" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "TRIMGALORE": { + "cutadapt": "3.4", + "trimgalore": "0.6.7" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T14:17:30+0000" + }, + "single_end_nomeseq_software_versions": { + "content": [ + { + "BISMARK_ALIGN": { + "bismark": "0.24.0" + }, + "BISMARK_GENOMEPREPARATION": { + "bismark": "0.24.0" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "TRIMGALORE": { + "cutadapt": "3.4", + "trimgalore": "0.6.7" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T14:17:30+0000" }, "single_end_nomeseq": { "content": [ @@ -44,7 +116,39 @@ "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-24T04:26:11+0000" + "timestamp": "2023-05-24T14:17:30+0000" + }, + "single_end_rrbs_software_versions": { + "content": [ + { + "BISMARK_ALIGN": { + "bismark": "0.24.0" + }, + "BISMARK_GENOMEPREPARATION": { + "bismark": "0.24.0" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T14:17:30+0000" }, "single_end": { "content": [ @@ -55,7 +159,7 @@ "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-24T04:26:11+0000" + "timestamp": "2023-05-24T14:17:30+0000" }, "single_end_rrbs": { "content": [ @@ -66,6 +170,6 @@ "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e" ], - "timestamp": "2023-05-24T04:26:11+0000" + "timestamp": "2023-05-24T14:17:30+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bismark_hisat/main.nf.test b/tests/pipeline/bismark_hisat/main.nf.test index a7d21503..b3ec30cc 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test +++ b/tests/pipeline/bismark_hisat/main.nf.test @@ -37,7 +37,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("software_versions") }, + { assert snapshot(softwareVersions).match("single_end_software_versions") }, { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } ) } @@ -83,7 +83,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("software_versions") }, + { assert snapshot(softwareVersions).match("single_end_rrbs_software_versions") }, { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } ) } diff --git a/tests/pipeline/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap index 59126776..65137010 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test.snap +++ b/tests/pipeline/bismark_hisat/main.nf.test.snap @@ -1,5 +1,41 @@ { - "software_versions": { + "single_end_software_versions": { + "content": [ + { + "BISMARK_ALIGN": { + "bismark": "0.24.0" + }, + "BISMARK_GENOMEPREPARATION": { + "bismark": "0.24.0" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "TRIMGALORE": { + "cutadapt": "3.4", + "trimgalore": "0.6.7" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T14:22:27+0000" + }, + "single_end_rrbs_software_versions": { "content": [ { "BISMARK_ALIGN": { @@ -29,7 +65,7 @@ } } ], - "timestamp": "2023-05-24T04:34:55+0000" + "timestamp": "2023-05-24T14:22:27+0000" }, "single_end": { "content": [ @@ -40,7 +76,7 @@ "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec" ], - "timestamp": "2023-05-24T04:34:55+0000" + "timestamp": "2023-05-24T14:22:27+0000" }, "single_end_rrbs": { "content": [ @@ -51,6 +87,6 @@ "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02" ], - "timestamp": "2023-05-24T04:34:55+0000" + "timestamp": "2023-05-24T14:22:27+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test index f885f129..b06aeb28 100644 --- a/tests/pipeline/bwameth/main.nf.test +++ b/tests/pipeline/bwameth/main.nf.test @@ -40,7 +40,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("software_versions") } + { assert snapshot(softwareVersions).match("single_end_software_versions") } ) } } @@ -89,7 +89,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("software_versions") } + { assert snapshot(softwareVersions).match("single_end_rrbs_software_versions") } ) } } diff --git a/tests/pipeline/bwameth/main.nf.test.snap b/tests/pipeline/bwameth/main.nf.test.snap index af5853bf..a1c89198 100644 --- a/tests/pipeline/bwameth/main.nf.test.snap +++ b/tests/pipeline/bwameth/main.nf.test.snap @@ -1,5 +1,41 @@ { - "software_versions": { + "single_end_software_versions": { + "content": [ + { + "BWAMETH_ALIGN": { + "bwameth": "0.2.2" + }, + "BWAMETH_INDEX": { + "bwameth": "0.2.2" + }, + "CAT_FASTQ": { + "cat": "8.30" + }, + "CUSTOM_DUMPSOFTWAREVERSIONS": { + "python": "3.11.0", + "yaml": "6.0" + }, + "FASTQC": { + "fastqc": "0.11.9" + }, + "QUALIMAP_BAMQC": { + "qualimap": "2.2.2-dev" + }, + "SAMPLESHEET_CHECK": { + "python": "3.8.3" + }, + "TRIMGALORE": { + "cutadapt": "3.4", + "trimgalore": "0.6.7" + }, + "Workflow": { + "nf-core/methylseq": "2.4.0dev" + } + } + ], + "timestamp": "2023-05-24T14:29:18+0000" + }, + "single_end_rrbs_software_versions": { "content": [ { "BWAMETH_ALIGN": { @@ -29,7 +65,7 @@ } } ], - "timestamp": "2023-05-24T04:42:22+0000" + "timestamp": "2023-05-24T14:29:18+0000" }, "single_end": { "content": [ @@ -43,7 +79,7 @@ "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0" ], - "timestamp": "2023-05-24T04:42:22+0000" + "timestamp": "2023-05-24T14:29:18+0000" }, "single_end_rrbs": { "content": [ @@ -57,6 +93,6 @@ "genome_results.txt:md5,f01c68dd46ca9ec0af3d6ec926763b75", "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0" ], - "timestamp": "2023-05-24T04:42:22+0000" + "timestamp": "2023-05-24T14:29:18+0000" } } \ No newline at end of file From 8739fb850afe35f00386c5f410214b85e770f69d Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Wed, 24 May 2023 11:40:21 -0400 Subject: [PATCH 19/58] Update tests/pipeline/bismark/main.nf.test Co-authored-by: Edmund Miller --- tests/pipeline/bismark/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test index ee2189a2..3c4ca82c 100644 --- a/tests/pipeline/bismark/main.nf.test +++ b/tests/pipeline/bismark/main.nf.test @@ -16,7 +16,7 @@ nextflow_pipeline { } then { - // Clean up anything from previous runs + // issue #278: Clean up anything from previous runs FileUtil.deleteDirectory(new File(".nf-test/indexes/")); new File(".nf-test/indexes/").mkdirs() // Copy the ref genome to a stable location for next tests From f0e7697e5995e9a83c2b2a1363729c08335e79df Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 24 May 2023 22:29:14 -0400 Subject: [PATCH 20/58] CI-add-master-pull-request-tests --- .github/workflows/ci.yml | 93 ++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f7d868a..ba5de73a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: - dev pull_request: + branches: + - dev + - master release: types: [published] @@ -16,20 +19,14 @@ concurrency: cancel-in-progress: true jobs: - test: - name: Run ${{ matrix.aligner }}-${{ matrix.profile }}-${{ matrix.NXF_VER }} - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq') }}" + test-dev: + if: github.base_ref == 'dev' || github.event_name == 'push' + name: Run pipeline with test data on dev runs-on: ubuntu-latest strategy: matrix: - NXF_VER: - - "22.10.1" - - "latest-everything" - aligner: - - bismark - - bismark_hisat - - bwameth + NXF_VER: ["latest-everything"] + aligner: ["bismark", "bismark_hisat", "bwameth"] profile: ["docker"] # TODO , "singularity", "conda"] steps: - name: Check out pipeline code @@ -70,34 +67,76 @@ jobs: wget -qO- https://code.askimed.com/install/nf-test | bash sudo mv nf-test /usr/local/bin/ - - name: Set up Singularity - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-singularity@v5 + - name: Run nf-test + run: | + nf-test test tests/pipeline/${{ matrix.aligner }}/ --tap=test.tap + + # If the test fails, output the software_versions.yml using the 'batcat' utility + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml + + - uses: pcolby/tap-summary@v1 with: - singularity-version: 3.7.1 + path: >- + test.tap + + test-master: + if: github.base_ref == 'master' + name: Run pipeline with test data on master + runs-on: ubuntu-latest + strategy: + matrix: + NXF_VER: ["22.10.1", "latest-everything"] + aligner: ["bismark", "bismark_hisat", "bwameth"] + profile: ["docker"] # TODO , "singularity", "conda"] + steps: + - name: Check out pipeline code + uses: actions/checkout@v3 - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@v2 + - name: Cache test data + id: cache-testdata + uses: actions/cache@v3 with: - auto-update-conda: true - channels: conda-forge,bioconda,defaults - python-version: ${{ matrix.python-version }} + path: ./test-datasets/ + key: methylseq-test-data - - name: Conda clean - if: matrix.profile == 'conda' - run: conda clean -a + - name: Check out test data + if: steps.cache-testdata.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: nf-core/test-datasets + ref: methylseq + path: ./test-datasets/ - - name: Run nf-test + - name: Replace remote paths in samplesheets run: | - nf-test test tests/pipeline/${{ matrix.aligner }}/ --tap=test.tap + for f in ./test-datasets/samplesheet/*csv; do + sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f + sed -i 's=https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/=./test-datasets/=g' $f + echo "========== $f ============" + cat $f + echo "========================================" + done; + + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Install nf-test + run: | + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ # If the test fails, output the software_versions.yml using the 'batcat' utility - name: Output log on failure if: failure() run: | sudo apt install bat > /dev/null - batcat --decorations=always --color=always /home/runner/work/fetchngs/fetchngs/.nf-test/tests/*/output/pipeline_info/software_versions.yml + batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - uses: pcolby/tap-summary@v1 with: From 8e21898d5ac1bb41c73699dba8b97e471707a0d9 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 24 May 2023 22:30:29 -0400 Subject: [PATCH 21/58] fix-trailing-whitespace --- tests/pipeline/bismark/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test index 3c4ca82c..364e64e1 100644 --- a/tests/pipeline/bismark/main.nf.test +++ b/tests/pipeline/bismark/main.nf.test @@ -16,7 +16,7 @@ nextflow_pipeline { } then { - // issue #278: Clean up anything from previous runs + // issue #278: Clean up anything from previous runs FileUtil.deleteDirectory(new File(".nf-test/indexes/")); new File(".nf-test/indexes/").mkdirs() // Copy the ref genome to a stable location for next tests From 5b7ffa335779fce1962975cb86c33b48ef1f6d70 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Thu, 25 May 2023 16:03:22 -0400 Subject: [PATCH 22/58] CI-add-comment-pr-test-tap-results --- .github/workflows/ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba5de73a..e89ef344 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,10 +78,11 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - - uses: pcolby/tap-summary@v1 + # Comment on the PR with the test results + - name: PR comment with file + uses: thollander/actions-comment-pull-request@v2 with: - path: >- - test.tap + filePath: test.tap test-master: if: github.base_ref == 'master' @@ -138,7 +139,8 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - - uses: pcolby/tap-summary@v1 + # Comment on the PR with the test results + - name: PR comment with file + uses: thollander/actions-comment-pull-request@v2 with: - path: >- - test.tap + filePath: test.tap From 6665d534ebe37ee350dcb3716659b2383d73c268 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Thu, 25 May 2023 20:18:28 -0400 Subject: [PATCH 23/58] CI-add-pr-comment-if-logic --- .github/workflows/ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e89ef344..37a93a48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ -name: nf-core CI # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +name: nf-core CI on: push: branches: @@ -20,14 +20,16 @@ concurrency: jobs: test-dev: + # Only run if the pull request is against the dev branch or if the push is to the dev branch if: github.base_ref == 'dev' || github.event_name == 'push' name: Run pipeline with test data on dev runs-on: ubuntu-latest + # Set up the matrix of different pipeline versions and profiles to test strategy: matrix: NXF_VER: ["latest-everything"] aligner: ["bismark", "bismark_hisat", "bwameth"] - profile: ["docker"] # TODO , "singularity", "conda"] + profile: ["docker"] # TODO ["singularity", "conda"] steps: - name: Check out pipeline code uses: actions/checkout@v3 @@ -80,19 +82,22 @@ jobs: # Comment on the PR with the test results - name: PR comment with file + if: github.event_name == 'pull_request' uses: thollander/actions-comment-pull-request@v2 with: filePath: test.tap test-master: + # Only run if the pull request is against the master branch if: github.base_ref == 'master' name: Run pipeline with test data on master runs-on: ubuntu-latest + # Set up the matrix of different pipeline versions and profiles to test strategy: matrix: NXF_VER: ["22.10.1", "latest-everything"] aligner: ["bismark", "bismark_hisat", "bwameth"] - profile: ["docker"] # TODO , "singularity", "conda"] + profile: ["docker"] # TODO ["singularity", "conda"] steps: - name: Check out pipeline code uses: actions/checkout@v3 @@ -141,6 +146,7 @@ jobs: # Comment on the PR with the test results - name: PR comment with file + if: github.event_name == 'pull_request' uses: thollander/actions-comment-pull-request@v2 with: filePath: test.tap From cb9323dc4d9ab4cb89203b53f27c70e056c7cc48 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Mon, 29 May 2023 21:48:22 -0400 Subject: [PATCH 24/58] 303-add-bamqc-regions-file-param --- CHANGELOG.md | 1 + nextflow.config | 2 ++ nextflow_schema.json | 16 ++++++++++++++++ workflows/methylseq.nf | 2 +- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6625542a..3ced00bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Pipeline Updates - Updated template to nf-core/tools v2.8 +- Add `--bamqc_regions_file` parameter for targeted methylation sequencing data. ### Bug fixes & refactoring diff --git a/nextflow.config b/nextflow.config index 55957aa0..cc28ece8 100644 --- a/nextflow.config +++ b/nextflow.config @@ -16,6 +16,8 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false + // Qualimap options + bamqc_regions_file = null // MultiQC options multiqc_config = null multiqc_title = null diff --git a/nextflow_schema.json b/nextflow_schema.json index 5eb0af93..abe2a784 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -374,6 +374,19 @@ }, "fa_icon": "far fa-circle" }, + "qualimap_options": { + "title": "Qualimap Options", + "type": "object", + "description": "Qualimap configurations", + "default": "", + "properties": { + "bamqc_regions_file": { + "type": "string", + "description": "A GFF or BED file containing the target regions which will be passed to Qualimap/Bamqc.", + "help_text": "Setting this option could be useful if you want calculate coverage stats over a list of regions, i.e. for targeted methylation sequencing data." + } + } + }, "skip_pipeline_steps": { "title": "Skip pipeline steps", "type": "object", @@ -620,6 +633,9 @@ { "$ref": "#/definitions/bwa_meth_options" }, + { + "$ref": "#/definitions/qualimap_options" + }, { "$ref": "#/definitions/skip_pipeline_steps" }, diff --git a/workflows/methylseq.nf b/workflows/methylseq.nf index 7cab0b5c..9ce22207 100644 --- a/workflows/methylseq.nf +++ b/workflows/methylseq.nf @@ -193,7 +193,7 @@ workflow METHYLSEQ { */ QUALIMAP_BAMQC ( ch_dedup, - [] + params.bamqc_regions_file ? Channel.fromPath( params.bamqc_regions_file, checkIfExists: true ).toList() : [] ) ch_versions = ch_versions.mix(QUALIMAP_BAMQC.out.versions.first()) From 299e5c507319f35606ef25c273fc1e0e608f0bc8 Mon Sep 17 00:00:00 2001 From: Nathan Spix <56930974+njspix@users.noreply.github.com> Date: Thu, 1 Jun 2023 08:27:52 -0400 Subject: [PATCH 25/58] removed unused directory --- conf/modules.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index f9c155ae..1827da52 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -229,11 +229,6 @@ process { mode: params.publish_dir_mode, pattern: "*bedGraph.gz" ], - [ - path: { "${params.outdir}/${params.aligner}/methylation_calls/stranded_CpG_report" }, - mode: params.publish_dir_mode, - pattern: "*CpG_report.txt.gz" - ], [ path: { "${params.outdir}/${params.aligner}/methylation_calls/splitting_report" }, mode: params.publish_dir_mode, From 70b167cef430237aaa0e6b587cb1565b18c0c08a Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 09:48:58 -0400 Subject: [PATCH 26/58] add-params-to-test-data-base-config --- conf/test.config | 6 +++--- conf/test_full.config | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/test.config b/conf/test.config index 6843627b..ca5e25cb 100644 --- a/conf/test.config +++ b/conf/test.config @@ -17,9 +17,9 @@ params { max_time = 6.h // Input data - input = "$test_data_base/samplesheet/samplesheet_test.csv" + input = "${params.test_data_base}/samplesheet/samplesheet_test.csv" // Genome references - fasta = "$test_data_base/reference/genome.fa" - fasta_index = "$test_data_base/reference/genome.fa.fai" + fasta = "${params.test_data_base}/reference/genome.fa" + fasta_index = "${params.test_data_base}/reference/genome.fa.fai" } diff --git a/conf/test_full.config b/conf/test_full.config index 96dbe7b9..60dbdf96 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -17,7 +17,7 @@ params { config_profile_description = 'Full test dataset to check pipeline function' // Input data - input = "https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/samplesheet/samplesheet_full.csv" + input = "${params.test_data_base}/samplesheet/samplesheet_full.csv" genome = 'GRCh38' } From 83d249be9626f56b485b32f67bef2d8e027099d7 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 10:07:18 -0400 Subject: [PATCH 27/58] update-index-files-to-value-channels --- subworkflows/local/prepare_genome.nf | 47 ++++++++++++++-------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf index faf87194..bb12053a 100644 --- a/subworkflows/local/prepare_genome.nf +++ b/subworkflows/local/prepare_genome.nf @@ -9,16 +9,15 @@ include { SAMTOOLS_FAIDX } from '../../modules/nf-core/samtools/fai workflow PREPARE_GENOME { main: - versions = Channel.empty() - - fasta = Channel.empty() - bismark_index = Channel.empty() - bwameth_index = Channel.empty() - fasta_index = Channel.empty() + ch_versions = Channel.empty() + ch_fasta = Channel.empty() + ch_bismark_index = Channel.empty() + ch_bwameth_index = Channel.empty() + ch_fasta_index = Channel.empty() // FASTA, if supplied if (params.fasta) { - fasta = file(params.fasta) + ch_fasta = Channel.value(file(params.fasta)) } // Aligner: bismark or bismark_hisat @@ -28,11 +27,11 @@ workflow PREPARE_GENOME { * Generate bismark index if not supplied */ if (params.bismark_index) { - bismark_index = file(params.bismark_index) + ch_bismark_index = Channel.value(file(params.bismark_index)) } else { - BISMARK_GENOMEPREPARATION(fasta) - bismark_index = BISMARK_GENOMEPREPARATION.out.index - versions = versions.mix(BISMARK_GENOMEPREPARATION.out.versions) + BISMARK_GENOMEPREPARATION(ch_fasta) + ch_bismark_index = BISMARK_GENOMEPREPARATION.out.index + ch_versions = versions.mix(BISMARK_GENOMEPREPARATION.out.versions) } } @@ -43,30 +42,30 @@ workflow PREPARE_GENOME { * Generate bwameth index if not supplied */ if (params.bwa_meth_index) { - bwameth_index = file(params.bwa_meth_index) + ch_bwameth_index = Channel.value(file(params.bwa_meth_index)) } else { - BWAMETH_INDEX(fasta) - bwameth_index = BWAMETH_INDEX.out.index - versions = versions.mix(BWAMETH_INDEX.out.versions) + BWAMETH_INDEX(ch_fasta) + ch_bwameth_index = BWAMETH_INDEX.out.index + ch_versions = versions.mix(BWAMETH_INDEX.out.versions) } /* * Generate fasta index if not supplied */ if (params.fasta_index) { - fasta_index = file(params.fasta_index) + ch_fasta_index = Channel.value(file(params.fasta_index)) } else { - SAMTOOLS_FAIDX([[:], fasta]) - fasta_index = SAMTOOLS_FAIDX.out.fai.map{ return(it[1])} - versions = versions.mix(SAMTOOLS_FAIDX.out.versions) + SAMTOOLS_FAIDX([[:], ch_fasta]) + ch_fasta_index = SAMTOOLS_FAIDX.out.fai.map{ return(it[1])} + ch_versions = versions.mix(SAMTOOLS_FAIDX.out.versions) } } emit: - fasta - bismark_index - bwameth_index - fasta_index - versions + fasta = ch_fasta // channel: path(genome.fasta) + bismark_index = ch_bismark_index // channel: path(genome.fasta) + bwameth_index = ch_bwameth_index // channel: path(genome.fasta) + fasta_index = ch_fasta_index // channel: path(genome.fasta) + versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] } From 7e334f19f81f36ea4fa4d67785008d3cdd5f445d Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 10:07:39 -0400 Subject: [PATCH 28/58] add-default-docker-registry-quay --- nextflow.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nextflow.config b/nextflow.config index cc28ece8..39a7e2c1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -256,6 +256,12 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Set default registry for Docker and Podman independent of -profile +// Will not be used unless Docker / Podman are enabled +// Set to your registry if you have a mirror of containers +docker.registry = 'quay.io' +podman.registry = 'quay.io' + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true From 380baf56d3e4ce1423d82aa8f89b28ffb2d3aef8 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 10:24:30 -0400 Subject: [PATCH 29/58] CI-fail-fast-false-and-cache-testdata-github-digest --- .github/workflows/ci.yml | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37a93a48..ec58fb2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,18 +2,16 @@ name: nf-core CI on: push: - branches: - - dev + branches: ["dev"] pull_request: - branches: - - dev - - master + branches: ["dev", "master"] release: types: [published] env: NXF_ANSI_LOG: false +# cancel-in-progress: true ensures that if a new push is made to the same branch while a previous run is still in progress, the previous run is cancelled concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true @@ -26,6 +24,8 @@ jobs: runs-on: ubuntu-latest # Set up the matrix of different pipeline versions and profiles to test strategy: + # Do not fail fast if one of the tests fails + fail-fast: false matrix: NXF_VER: ["latest-everything"] aligner: ["bismark", "bismark_hisat", "bwameth"] @@ -34,12 +34,17 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v3 + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: - path: ./test-datasets/ - key: methylseq-test-data + path: test-datasets/ + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -47,7 +52,7 @@ jobs: with: repository: nf-core/test-datasets ref: methylseq - path: ./test-datasets/ + path: test-datasets/ - name: Replace remote paths in samplesheets run: | @@ -102,12 +107,17 @@ jobs: - name: Check out pipeline code uses: actions/checkout@v3 + - name: Hash Github Workspace + id: hash_workspace + run: | + echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT + - name: Cache test data id: cache-testdata uses: actions/cache@v3 with: - path: ./test-datasets/ - key: methylseq-test-data + path: test-datasets/ + key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data if: steps.cache-testdata.outputs.cache-hit != 'true' @@ -115,7 +125,7 @@ jobs: with: repository: nf-core/test-datasets ref: methylseq - path: ./test-datasets/ + path: test-datasets/ - name: Replace remote paths in samplesheets run: | From ade05489089ff9b7013de72c388fa844df0130bd Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 10:32:29 -0400 Subject: [PATCH 30/58] fix-ch-versions-variable --- subworkflows/local/prepare_genome.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/prepare_genome.nf b/subworkflows/local/prepare_genome.nf index bb12053a..8fa2cc6e 100644 --- a/subworkflows/local/prepare_genome.nf +++ b/subworkflows/local/prepare_genome.nf @@ -31,7 +31,7 @@ workflow PREPARE_GENOME { } else { BISMARK_GENOMEPREPARATION(ch_fasta) ch_bismark_index = BISMARK_GENOMEPREPARATION.out.index - ch_versions = versions.mix(BISMARK_GENOMEPREPARATION.out.versions) + ch_versions = ch_versions.mix(BISMARK_GENOMEPREPARATION.out.versions) } } @@ -46,7 +46,7 @@ workflow PREPARE_GENOME { } else { BWAMETH_INDEX(ch_fasta) ch_bwameth_index = BWAMETH_INDEX.out.index - ch_versions = versions.mix(BWAMETH_INDEX.out.versions) + ch_versions = ch_versions.mix(BWAMETH_INDEX.out.versions) } /* @@ -57,7 +57,7 @@ workflow PREPARE_GENOME { } else { SAMTOOLS_FAIDX([[:], ch_fasta]) ch_fasta_index = SAMTOOLS_FAIDX.out.fai.map{ return(it[1])} - ch_versions = versions.mix(SAMTOOLS_FAIDX.out.versions) + ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) } } From 457977069a75035b26967962c3e64e60e4ce71e8 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 21:05:44 -0400 Subject: [PATCH 31/58] add-UTILS-removeNextflowVersion-lib-update-snapshots --- nf-test.config | 10 ++ tests/pipeline/bismark/main.nf.test | 65 ++++----- tests/pipeline/bismark/main.nf.test.snap | 138 ++---------------- tests/pipeline/bismark_hisat/main.nf.test | 50 +++---- .../pipeline/bismark_hisat/main.nf.test.snap | 68 +-------- tests/pipeline/bwameth/main.nf.test | 38 ++--- tests/pipeline/bwameth/main.nf.test.snap | 68 +-------- tests/pipeline/lib/UTILS.groovy | 11 ++ 8 files changed, 105 insertions(+), 343 deletions(-) create mode 100644 tests/pipeline/lib/UTILS.groovy diff --git a/nf-test.config b/nf-test.config index 7d6adb94..7d705067 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,6 +1,16 @@ config { + // location for all nf-tests testsDir "tests" + + // nf-test directory including temporary files for each test workDir ".nf-test" + + // location of library folder that is added automatically to the classpath + libDir "tests/pipeline/lib/" + + // location of an optional nextflow.config file specific for executing tests configFile "nextflow.config" + + // run all test with the defined docker profile from the main nextflow.config profile "test,docker" } diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test index 364e64e1..a07ed645 100644 --- a/tests/pipeline/bismark/main.nf.test +++ b/tests/pipeline/bismark/main.nf.test @@ -4,15 +4,16 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" - tag "pipeline" + tag "bismark" - test("Bismark Single End") { + test("Bismark Single-End") { when { params { aligner = "bismark" save_reference = true outdir = "$outputDir" } + } then { @@ -25,10 +26,6 @@ nextflow_pipeline { new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() ) - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), @@ -37,20 +34,20 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("single_end_software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } } - test("Bismark Single End With Index") { + test("Bismark Single-End With Index") { when { params { - aligner = "bismark" + aligner = "bismark" // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" + bismark_index = ".nf-test/indexes/BismarkIndex/" cytosine_report = true - outdir = "$outputDir" + outdir = "$outputDir" } } @@ -59,22 +56,18 @@ nextflow_pipeline { } } - test("Bismark Single End With RRBS") { + test("Bismark Single-End With RRBS") { when { params { - aligner = "bismark" - skip_trimming = true + aligner = "bismark" + skip_trimming = true save_reference = true - rrbs = true - outdir = "$outputDir" + rrbs = true + outdir = "$outputDir" } } then { - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt"), @@ -83,22 +76,22 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("single_end_rrbs_software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_rrbs_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } } - test("Bismark Single End With RRBS With Index") { + test("Bismark Single-End With RRBS With Index") { when { params { - aligner = "bismark" - skip_trimming = true - rrbs = true + aligner = "bismark" + skip_trimming = true + rrbs = true // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" + bismark_index = ".nf-test/indexes/BismarkIndex/" cytosine_report = true - outdir = "$outputDir" + outdir = "$outputDir" } } @@ -107,7 +100,7 @@ nextflow_pipeline { } } - test("Bismark Single End NOMe-seq") { + test("Bismark Single-End NOMe-seq") { when { params { aligner = "bismark" @@ -117,10 +110,6 @@ nextflow_pipeline { } then { - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt"), @@ -129,7 +118,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_nomeseq") }, - { assert snapshot(softwareVersions).match("single_end_nomeseq_software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_nomeseq_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } @@ -139,20 +128,16 @@ nextflow_pipeline { when { params { aligner = "bismark" - em_seq = true + em_seq = true clip_r1 = 2 - outdir = "$outputDir" + outdir = "$outputDir" } } then { - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, - { assert snapshot(softwareVersions).match("clipping_params_software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("clipping_params_software_versions") }, { assert new File("$outputDir/bismark/summary/bismark_summary_report.txt").exists() } ) } diff --git a/tests/pipeline/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap index 833588ae..9daf67f6 100644 --- a/tests/pipeline/bismark/main.nf.test.snap +++ b/tests/pipeline/bismark/main.nf.test.snap @@ -1,111 +1,21 @@ { "clipping_params_software_versions": { "content": [ - { - "BISMARK_ALIGN": { - "bismark": "0.24.0" - }, - "BISMARK_GENOMEPREPARATION": { - "bismark": "0.24.0" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "TRIMGALORE": { - "cutadapt": "3.4", - "trimgalore": "0.6.7" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" }, "single_end_software_versions": { "content": [ - { - "BISMARK_ALIGN": { - "bismark": "0.24.0" - }, - "BISMARK_GENOMEPREPARATION": { - "bismark": "0.24.0" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "TRIMGALORE": { - "cutadapt": "3.4", - "trimgalore": "0.6.7" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" }, "single_end_nomeseq_software_versions": { "content": [ - { - "BISMARK_ALIGN": { - "bismark": "0.24.0" - }, - "BISMARK_GENOMEPREPARATION": { - "bismark": "0.24.0" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "TRIMGALORE": { - "cutadapt": "3.4", - "trimgalore": "0.6.7" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" }, "single_end_nomeseq": { "content": [ @@ -116,39 +26,13 @@ "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" }, "single_end_rrbs_software_versions": { "content": [ - { - "BISMARK_ALIGN": { - "bismark": "0.24.0" - }, - "BISMARK_GENOMEPREPARATION": { - "bismark": "0.24.0" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" }, "single_end": { "content": [ @@ -159,7 +43,7 @@ "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" }, "single_end_rrbs": { "content": [ @@ -170,6 +54,6 @@ "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e" ], - "timestamp": "2023-05-24T14:17:30+0000" + "timestamp": "2023-05-28T00:41:58+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bismark_hisat/main.nf.test b/tests/pipeline/bismark_hisat/main.nf.test index b3ec30cc..48a4cffa 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test +++ b/tests/pipeline/bismark_hisat/main.nf.test @@ -4,14 +4,14 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" - tag "pipeline" + tag "bismark_hisat" - test("Bismark Hisat Single End") { + test("Bismark Hisat Single-End") { when { params { - aligner = "bismark_hisat" + aligner = "bismark_hisat" save_reference = true - outdir = "$outputDir" + outdir = "$outputDir" } } @@ -25,10 +25,6 @@ nextflow_pipeline { new File(".nf-test/indexes/BismarkIndex/").getAbsolutePath() ) - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt"), @@ -37,20 +33,20 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("single_end_software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_software_versions") }, { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } ) } } - test("Bismark Hisat Single End With Index") { + test("Bismark Hisat Single-End With Index") { when { params { - aligner = "bismark_hisat" + aligner = "bismark_hisat" // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" + bismark_index = ".nf-test/indexes/BismarkIndex/" local_alignment = true - outdir = "$outputDir" + outdir = "$outputDir" } } @@ -59,22 +55,18 @@ nextflow_pipeline { } } - test("Bismark Hisat Single End With RRBS") { + test("Bismark Hisat Single-End With RRBS") { when { params { - aligner = "bismark_hisat" + aligner = "bismark_hisat" save_reference = true - skip_trimming = true - rrbs = true - outdir = "$outputDir" + skip_trimming = true + rrbs = true + outdir = "$outputDir" } } then { - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bismark_hisat/methylation_calls/splitting_report/Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt"), @@ -83,22 +75,22 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("single_end_rrbs_software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_rrbs_software_versions") }, { assert new File("$outputDir/bismark_hisat/summary/bismark_summary_report.txt").exists() } ) } } - test("Bismark Hisat Single End With RRBS With Index") { + test("Bismark Hisat Single-End With RRBS With Index") { when { params { - aligner = "bismark_hisat" - skip_trimming = true - rrbs = true + aligner = "bismark_hisat" + skip_trimming = true + rrbs = true // Generated by previous test - bismark_index = ".nf-test/indexes/BismarkIndex/" + bismark_index = ".nf-test/indexes/BismarkIndex/" local_alignment = true - outdir = "$outputDir" + outdir = "$outputDir" } } diff --git a/tests/pipeline/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap index 65137010..f4497a3f 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test.snap +++ b/tests/pipeline/bismark_hisat/main.nf.test.snap @@ -1,71 +1,15 @@ { "single_end_software_versions": { "content": [ - { - "BISMARK_ALIGN": { - "bismark": "0.24.0" - }, - "BISMARK_GENOMEPREPARATION": { - "bismark": "0.24.0" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "TRIMGALORE": { - "cutadapt": "3.4", - "trimgalore": "0.6.7" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:22:27+0000" + "timestamp": "2023-05-28T00:47:31+0000" }, "single_end_rrbs_software_versions": { "content": [ - { - "BISMARK_ALIGN": { - "bismark": "0.24.0" - }, - "BISMARK_GENOMEPREPARATION": { - "bismark": "0.24.0" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:22:27+0000" + "timestamp": "2023-05-28T00:47:31+0000" }, "single_end": { "content": [ @@ -76,7 +20,7 @@ "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec" ], - "timestamp": "2023-05-24T14:22:27+0000" + "timestamp": "2023-05-28T00:47:31+0000" }, "single_end_rrbs": { "content": [ @@ -87,6 +31,6 @@ "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02" ], - "timestamp": "2023-05-24T14:22:27+0000" + "timestamp": "2023-05-28T00:47:31+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test index b06aeb28..d7e33c63 100644 --- a/tests/pipeline/bwameth/main.nf.test +++ b/tests/pipeline/bwameth/main.nf.test @@ -4,14 +4,14 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" - tag "pipeline" + tag "bwameth" - test("Bwameth Single End") { + test("Bwameth Single-End") { when { params { - aligner = "bwameth" + aligner = "bwameth" save_reference = true - outdir = "$outputDir" + outdir = "$outputDir" } } @@ -25,10 +25,6 @@ nextflow_pipeline { new File(".nf-test/indexes/bwameth/").getAbsolutePath() ) - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), @@ -40,19 +36,19 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end") }, - { assert snapshot(softwareVersions).match("single_end_software_versions") } + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_software_versions") } ) } } - test("Bwameth Single End With Index") { + test("Bwameth Single-End With Index") { when { params { - aligner = "bwameth" + aligner = "bwameth" // Generated by previous test bwa_meth_index = ".nf-test/indexes/bwameth/" - methyl_kit = true - outdir = "$outputDir" + methyl_kit = true + outdir = "$outputDir" } } @@ -61,23 +57,19 @@ nextflow_pipeline { } } - test("Bwameth Single End RRBS") { + test("Bwameth Single-End RRBS") { when { params { - aligner = "bwameth" + aligner = "bwameth" save_reference = true - skip_trimming = true - rrbs = true + skip_trimming = true + rrbs = true save_reference = true - outdir = "$outputDir" + outdir = "$outputDir" } } then { - // Remove Nextflow version from software_versions.yml - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml - if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } - assertAll( { assert workflow.success }, { assert snapshot(path("$outputDir/bwameth/alignments/samtools_stats/Ecoli_10K_methylated.flagstat"), @@ -89,7 +81,7 @@ nextflow_pipeline { path("$outputDir/qualimap/Ecoli_10K_methylated/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub1/genome_results.txt"), path("$outputDir/qualimap/SRR389222_sub2/genome_results.txt")).match("single_end_rrbs") }, - { assert snapshot(softwareVersions).match("single_end_rrbs_software_versions") } + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("single_end_rrbs_software_versions") } ) } } diff --git a/tests/pipeline/bwameth/main.nf.test.snap b/tests/pipeline/bwameth/main.nf.test.snap index a1c89198..1014e5f0 100644 --- a/tests/pipeline/bwameth/main.nf.test.snap +++ b/tests/pipeline/bwameth/main.nf.test.snap @@ -1,71 +1,15 @@ { "single_end_software_versions": { "content": [ - { - "BWAMETH_ALIGN": { - "bwameth": "0.2.2" - }, - "BWAMETH_INDEX": { - "bwameth": "0.2.2" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "TRIMGALORE": { - "cutadapt": "3.4", - "trimgalore": "0.6.7" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:29:18+0000" + "timestamp": "2023-05-28T00:53:43+0000" }, "single_end_rrbs_software_versions": { "content": [ - { - "BWAMETH_ALIGN": { - "bwameth": "0.2.2" - }, - "BWAMETH_INDEX": { - "bwameth": "0.2.2" - }, - "CAT_FASTQ": { - "cat": "8.30" - }, - "CUSTOM_DUMPSOFTWAREVERSIONS": { - "python": "3.11.0", - "yaml": "6.0" - }, - "FASTQC": { - "fastqc": "0.11.9" - }, - "QUALIMAP_BAMQC": { - "qualimap": "2.2.2-dev" - }, - "SAMPLESHEET_CHECK": { - "python": "3.8.3" - }, - "Workflow": { - "nf-core/methylseq": "2.4.0dev" - } - } + "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-24T14:29:18+0000" + "timestamp": "2023-05-28T00:53:43+0000" }, "single_end": { "content": [ @@ -79,7 +23,7 @@ "genome_results.txt:md5,2c755b15ae5ad4a1d29395320a6264ca", "genome_results.txt:md5,3c7d14ea070df510941c55f72706a7a0" ], - "timestamp": "2023-05-24T14:29:18+0000" + "timestamp": "2023-05-28T00:53:43+0000" }, "single_end_rrbs": { "content": [ @@ -93,6 +37,6 @@ "genome_results.txt:md5,f01c68dd46ca9ec0af3d6ec926763b75", "genome_results.txt:md5,f4513f5aa1622937ccdcfe92a6f877c0" ], - "timestamp": "2023-05-24T14:29:18+0000" + "timestamp": "2023-05-28T00:53:43+0000" } } \ No newline at end of file diff --git a/tests/pipeline/lib/UTILS.groovy b/tests/pipeline/lib/UTILS.groovy new file mode 100644 index 00000000..a76b18a7 --- /dev/null +++ b/tests/pipeline/lib/UTILS.groovy @@ -0,0 +1,11 @@ +// Function to remove Nextflow version from software_versions.yml + +class UTILS { + public static String removeNextflowVersion(outputDir) { + def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + if (softwareVersions.containsKey("Workflow")) { + softwareVersions.Workflow.remove("Nextflow") + } + return softwareVersions + } +} \ No newline at end of file From 4307580031e9d1b3d74c0d68914ce49a3583f0de Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 21:06:43 -0400 Subject: [PATCH 32/58] add-ci-fast-option-for-dev --- .github/workflows/ci.yml | 4 ++-- tests/config/tags.yml | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/config/tags.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec58fb2c..6f20fdc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ concurrency: jobs: test-dev: # Only run if the pull request is against the dev branch or if the push is to the dev branch - if: github.base_ref == 'dev' || github.event_name == 'push' + if: ${{ (github.base_ref == 'dev' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq')) && !contains(github.event.head_commit.message, '[ci fast]') }} name: Run pipeline with test data on dev runs-on: ubuntu-latest # Set up the matrix of different pipeline versions and profiles to test @@ -94,7 +94,7 @@ jobs: test-master: # Only run if the pull request is against the master branch - if: github.base_ref == 'master' + if: ${{ (github.base_ref == 'master' && github.repository == 'nf-core/methylseq') }} name: Run pipeline with test data on master runs-on: ubuntu-latest # Set up the matrix of different pipeline versions and profiles to test diff --git a/tests/config/tags.yml b/tests/config/tags.yml new file mode 100644 index 00000000..d76f56d6 --- /dev/null +++ b/tests/config/tags.yml @@ -0,0 +1,11 @@ +# tags for tests + + +bismark: + + +bismark_hisat: + + +bwameth: + From 49a1e617b32c4824463c1396cf5a44fb6c4f63ae Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 21:06:57 -0400 Subject: [PATCH 33/58] update-changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ced00bb..57ff8d8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,15 @@ - Updated template to nf-core/tools v2.8 - Add `--bamqc_regions_file` parameter for targeted methylation sequencing data. +- ✨ Add NF-TEST tests and snapshots for the pipeline test profile ### Bug fixes & refactoring -- Nothing yet.. +- 🛠️ update index file channels to explicit value channels (`877ad9f`) +- 🐛 fix `params.test_data_base` in test and test_full configs (`1105d4f`) +- 🤖 GitHub Actions CI - pull_reqest to `dev` tests with NXF_VER `latest-everything` +- 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` +- 🤖 GitHub Actions CI - `fail-fast` set to false ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16 From d4c407d7cd23c2408ef6325fd869189ceead028e Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 21:08:05 -0400 Subject: [PATCH 34/58] init-tests-tags-yml --- tests/config/tags.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/config/tags.yml b/tests/config/tags.yml index d76f56d6..db5bda67 100644 --- a/tests/config/tags.yml +++ b/tests/config/tags.yml @@ -1,11 +1,7 @@ # tags for tests - bismark: - bismark_hisat: - bwameth: - From 8517f52cade34df3d294488d0ffd0f4fdbb745a9 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 27 May 2023 21:11:14 -0400 Subject: [PATCH 35/58] fix-no-new-line --- tests/pipeline/lib/UTILS.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/lib/UTILS.groovy b/tests/pipeline/lib/UTILS.groovy index a76b18a7..311403cc 100644 --- a/tests/pipeline/lib/UTILS.groovy +++ b/tests/pipeline/lib/UTILS.groovy @@ -8,4 +8,4 @@ class UTILS { } return softwareVersions } -} \ No newline at end of file +} From ed7645401d4647ed2131eb49aef203df4dd8806d Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sun, 28 May 2023 12:36:47 -0400 Subject: [PATCH 36/58] CI-add-nf-test-ver-to-matrix[ci ckip] --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f20fdc4..f0ed92e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ concurrency: jobs: test-dev: # Only run if the pull request is against the dev branch or if the push is to the dev branch - if: ${{ (github.base_ref == 'dev' || (github.event_name == 'push' && github.repository == 'nf-core/methylseq')) && !contains(github.event.head_commit.message, '[ci fast]') }} + if: ${{ (github.base_ref == 'dev' || (github.event_name == 'push')) && !contains(github.event.head_commit.message, '[ci skip]') }} name: Run pipeline with test data on dev runs-on: ubuntu-latest # Set up the matrix of different pipeline versions and profiles to test @@ -28,6 +28,7 @@ jobs: fail-fast: false matrix: NXF_VER: ["latest-everything"] + NFTEST_VER: ["0.7.3"] aligner: ["bismark", "bismark_hisat", "bwameth"] profile: ["docker"] # TODO ["singularity", "conda"] steps: @@ -71,7 +72,7 @@ jobs: - name: Install nf-test run: | - wget -qO- https://code.askimed.com/install/nf-test | bash + wget -qO- https://code.askimed.com/install/nf-test | bash -s ${{ matrix.NFTEST_VER }} sudo mv nf-test /usr/local/bin/ - name: Run nf-test @@ -94,13 +95,14 @@ jobs: test-master: # Only run if the pull request is against the master branch - if: ${{ (github.base_ref == 'master' && github.repository == 'nf-core/methylseq') }} + if: ${{ (github.base_ref == 'master') }} name: Run pipeline with test data on master runs-on: ubuntu-latest # Set up the matrix of different pipeline versions and profiles to test strategy: matrix: NXF_VER: ["22.10.1", "latest-everything"] + NFTEST_VER: ["0.7.3"] aligner: ["bismark", "bismark_hisat", "bwameth"] profile: ["docker"] # TODO ["singularity", "conda"] steps: @@ -144,7 +146,7 @@ jobs: - name: Install nf-test run: | - wget -qO- https://code.askimed.com/install/nf-test | bash + wget -qO- https://code.askimed.com/install/nf-test | bash -s ${{ matrix.NFTEST_VER }} sudo mv nf-test /usr/local/bin/ # If the test fails, output the software_versions.yml using the 'batcat' utility From cd0166f6e579987ce49d0f2f77672a602418b17e Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sun, 28 May 2023 15:21:10 -0400 Subject: [PATCH 37/58] add-missing-run-nf-test-master-branch --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0ed92e1..6d82428d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,6 +149,10 @@ jobs: wget -qO- https://code.askimed.com/install/nf-test | bash -s ${{ matrix.NFTEST_VER }} sudo mv nf-test /usr/local/bin/ + - name: Run nf-test + run: | + nf-test test tests/pipeline/${{ matrix.aligner }}/ --tap=test.tap + # If the test fails, output the software_versions.yml using the 'batcat' utility - name: Output log on failure if: failure() From c4730622e86d2e735183a04876fd90516d990bb1 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Mon, 29 May 2023 19:11:19 -0400 Subject: [PATCH 38/58] CI-update-pr-commenter-to-upsert --- .github/workflows/ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d82428d..40ee71e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,11 +87,12 @@ jobs: batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml # Comment on the PR with the test results - - name: PR comment with file - if: github.event_name == 'pull_request' + - name: Comment PR with execution number uses: thollander/actions-comment-pull-request@v2 with: - filePath: test.tap + message: | + _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ + comment_tag: execution test-master: # Only run if the pull request is against the master branch @@ -161,8 +162,9 @@ jobs: batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml # Comment on the PR with the test results - - name: PR comment with file - if: github.event_name == 'pull_request' + - name: Comment PR with execution number uses: thollander/actions-comment-pull-request@v2 with: - filePath: test.tap + message: | + _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ + comment_tag: execution From 509121df4476d022d7687cd0b1572b019ba15707 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 30 May 2023 21:46:09 -0400 Subject: [PATCH 39/58] CI-update-pr-commenter-msg-reactions --- .github/workflows/ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40ee71e2..f304d8a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: if: ${{ (github.base_ref == 'dev' || (github.event_name == 'push')) && !contains(github.event.head_commit.message, '[ci skip]') }} name: Run pipeline with test data on dev runs-on: ubuntu-latest + # Add PR write permissions to allow commenting on the PR + permissions: + pull-requests: write # Set up the matrix of different pipeline versions and profiles to test strategy: # Do not fail fast if one of the tests fails @@ -92,13 +95,20 @@ jobs: with: message: | _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ + ``` + $(cat *.tap) + ``` comment_tag: execution + reactions: eyes, rocket test-master: # Only run if the pull request is against the master branch if: ${{ (github.base_ref == 'master') }} name: Run pipeline with test data on master runs-on: ubuntu-latest + # Add PR write permissions to allow commenting on the PR + permissions: + pull-requests: write # Set up the matrix of different pipeline versions and profiles to test strategy: matrix: @@ -167,4 +177,8 @@ jobs: with: message: | _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ + ``` + $(cat *.tap) + ``` comment_tag: execution + reactions: eyes, rocket From 44479c9a6553968826068a730e9c9d3631919b1c Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 30 May 2023 22:41:54 -0400 Subject: [PATCH 40/58] CI-update-pr-commenter-concat-test-results --- .github/workflows/ci.yml | 35 ++++++++++++++++++++--------------- nextflow.config | 6 ------ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f304d8a6..c8a4fa3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,7 @@ on: env: NXF_ANSI_LOG: false + NFTEST_VER: "0.7.3" # cancel-in-progress: true ensures that if a new push is made to the same branch while a previous run is still in progress, the previous run is cancelled concurrency: @@ -19,7 +20,7 @@ concurrency: jobs: test-dev: # Only run if the pull request is against the dev branch or if the push is to the dev branch - if: ${{ (github.base_ref == 'dev' || (github.event_name == 'push')) && !contains(github.event.head_commit.message, '[ci skip]') }} + if: github.base_ref == 'dev' || (github.event_name == 'push') name: Run pipeline with test data on dev runs-on: ubuntu-latest # Add PR write permissions to allow commenting on the PR @@ -31,7 +32,6 @@ jobs: fail-fast: false matrix: NXF_VER: ["latest-everything"] - NFTEST_VER: ["0.7.3"] aligner: ["bismark", "bismark_hisat", "bwameth"] profile: ["docker"] # TODO ["singularity", "conda"] steps: @@ -75,7 +75,7 @@ jobs: - name: Install nf-test run: | - wget -qO- https://code.askimed.com/install/nf-test | bash -s ${{ matrix.NFTEST_VER }} + wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER sudo mv nf-test /usr/local/bin/ - name: Run nf-test @@ -89,17 +89,20 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml + # Concatenate all the test results into a single file + - name: Concatenate test results + id: concat + run: | + echo "TEST_RESULTS=$(cat *.tap)" >> $GITHUB_OUTPUT + # Comment on the PR with the test results - name: Comment PR with execution number uses: thollander/actions-comment-pull-request@v2 with: message: | - _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ - ``` - $(cat *.tap) - ``` + _(execution **${{ steps.concat.outputs.TEST_RESULTS }}**)_ comment_tag: execution - reactions: eyes, rocket + reactions: eyes test-master: # Only run if the pull request is against the master branch @@ -113,7 +116,6 @@ jobs: strategy: matrix: NXF_VER: ["22.10.1", "latest-everything"] - NFTEST_VER: ["0.7.3"] aligner: ["bismark", "bismark_hisat", "bwameth"] profile: ["docker"] # TODO ["singularity", "conda"] steps: @@ -157,7 +159,7 @@ jobs: - name: Install nf-test run: | - wget -qO- https://code.askimed.com/install/nf-test | bash -s ${{ matrix.NFTEST_VER }} + wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER sudo mv nf-test /usr/local/bin/ - name: Run nf-test @@ -171,14 +173,17 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml + # Concatenate all the test results into a single file + - name: Concatenate test results + id: concat + run: | + echo "TEST_RESULTS=$(cat *.tap)" >> $GITHUB_OUTPUT + # Comment on the PR with the test results - name: Comment PR with execution number uses: thollander/actions-comment-pull-request@v2 with: message: | - _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ - ``` - $(cat *.tap) - ``` + _(execution **${{ steps.concat.outputs.TEST_RESULTS }}**)_ comment_tag: execution - reactions: eyes, rocket + reactions: eyes diff --git a/nextflow.config b/nextflow.config index 39a7e2c1..cc28ece8 100644 --- a/nextflow.config +++ b/nextflow.config @@ -256,12 +256,6 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] -// Set default registry for Docker and Podman independent of -profile -// Will not be used unless Docker / Podman are enabled -// Set to your registry if you have a mirror of containers -docker.registry = 'quay.io' -podman.registry = 'quay.io' - def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true From 78f4a730d4d5a99100a70afd9f58f4888861411d Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Tue, 30 May 2023 23:31:30 -0400 Subject: [PATCH 41/58] CI-save-commenter-battle-for-later --- .github/workflows/ci.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8a4fa3d..8d35f756 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,18 +89,12 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - # Concatenate all the test results into a single file - - name: Concatenate test results - id: concat - run: | - echo "TEST_RESULTS=$(cat *.tap)" >> $GITHUB_OUTPUT - # Comment on the PR with the test results - name: Comment PR with execution number uses: thollander/actions-comment-pull-request@v2 with: message: | - _(execution **${{ steps.concat.outputs.TEST_RESULTS }}**)_ + _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ comment_tag: execution reactions: eyes @@ -173,17 +167,11 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - # Concatenate all the test results into a single file - - name: Concatenate test results - id: concat - run: | - echo "TEST_RESULTS=$(cat *.tap)" >> $GITHUB_OUTPUT - # Comment on the PR with the test results - name: Comment PR with execution number uses: thollander/actions-comment-pull-request@v2 with: message: | - _(execution **${{ steps.concat.outputs.TEST_RESULTS }}**)_ + _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ comment_tag: execution reactions: eyes From a304497c095a450a054dd251b4a314a6dce55a1f Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 31 May 2023 12:56:13 -0400 Subject: [PATCH 42/58] rm-docker-hardcode --- nf-test.config | 2 +- tests/pipeline/bismark/main.nf.test | 1 + tests/pipeline/bismark_hisat/main.nf.test | 1 + tests/pipeline/bwameth/main.nf.test | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nf-test.config b/nf-test.config index 7d705067..cd432d43 100644 --- a/nf-test.config +++ b/nf-test.config @@ -12,5 +12,5 @@ config { configFile "nextflow.config" // run all test with the defined docker profile from the main nextflow.config - profile "test,docker" + profile "" } diff --git a/tests/pipeline/bismark/main.nf.test b/tests/pipeline/bismark/main.nf.test index a07ed645..2bd75c9c 100644 --- a/tests/pipeline/bismark/main.nf.test +++ b/tests/pipeline/bismark/main.nf.test @@ -5,6 +5,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" tag "bismark" + tag "pipeline" test("Bismark Single-End") { when { diff --git a/tests/pipeline/bismark_hisat/main.nf.test b/tests/pipeline/bismark_hisat/main.nf.test index 48a4cffa..a368b722 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test +++ b/tests/pipeline/bismark_hisat/main.nf.test @@ -5,6 +5,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" tag "bismark_hisat" + tag "pipeline" test("Bismark Hisat Single-End") { when { diff --git a/tests/pipeline/bwameth/main.nf.test b/tests/pipeline/bwameth/main.nf.test index d7e33c63..95d783c0 100644 --- a/tests/pipeline/bwameth/main.nf.test +++ b/tests/pipeline/bwameth/main.nf.test @@ -5,6 +5,7 @@ nextflow_pipeline { name "Test Workflow main.nf" script "main.nf" tag "bwameth" + tag "pipeline" test("Bwameth Single-End") { when { From b9e47d31be87b8446b27a166525adb1d424e3ebb Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 31 May 2023 12:56:43 -0400 Subject: [PATCH 43/58] CI-add-matrix-profile-nf-test-run --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d35f756..1b2a5b02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,11 +2,15 @@ name: nf-core CI on: push: - branches: ["dev"] + branches: + - "dev" pull_request: - branches: ["dev", "master"] + branches: + - "dev" + - "master" release: - types: [published] + types: + - "published" env: NXF_ANSI_LOG: false @@ -31,9 +35,14 @@ jobs: # Do not fail fast if one of the tests fails fail-fast: false matrix: - NXF_VER: ["latest-everything"] - aligner: ["bismark", "bismark_hisat", "bwameth"] - profile: ["docker"] # TODO ["singularity", "conda"] + NXF_VER: + - "latest-everything" + aligner: + - "bismark" + - "bismark_hisat" + - "bwameth" + profile: # TODO ["singularity", "conda"] + - "docker" steps: - name: Check out pipeline code uses: actions/checkout@v3 @@ -80,7 +89,7 @@ jobs: - name: Run nf-test run: | - nf-test test tests/pipeline/${{ matrix.aligner }}/ --tap=test.tap + nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}"" --tap=test.tap # If the test fails, output the software_versions.yml using the 'batcat' utility - name: Output log on failure @@ -109,9 +118,14 @@ jobs: # Set up the matrix of different pipeline versions and profiles to test strategy: matrix: - NXF_VER: ["22.10.1", "latest-everything"] - aligner: ["bismark", "bismark_hisat", "bwameth"] - profile: ["docker"] # TODO ["singularity", "conda"] + NXF_VER: + - "latest-everything" + aligner: + - "bismark" + - "bismark_hisat" + - "bwameth" + profile: # TODO ["singularity", "conda"] + - "docker" steps: - name: Check out pipeline code uses: actions/checkout@v3 @@ -158,7 +172,7 @@ jobs: - name: Run nf-test run: | - nf-test test tests/pipeline/${{ matrix.aligner }}/ --tap=test.tap + nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}"" --tap=test.tap # If the test fails, output the software_versions.yml using the 'batcat' utility - name: Output log on failure From a290e0c3438cdb3a1b2a349385047312c496250c Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 31 May 2023 12:59:01 -0400 Subject: [PATCH 44/58] fix-docker-podman-registry-tower-scoping --- nextflow.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index cc28ece8..5ee81113 100644 --- a/nextflow.config +++ b/nextflow.config @@ -163,7 +163,6 @@ profiles { } docker { docker.enabled = true - docker.registry = 'quay.io' docker.userEmulation = true conda.enabled = false singularity.enabled = false @@ -187,7 +186,6 @@ profiles { } podman { podman.enabled = true - podman.registry = 'quay.io' conda.enabled = false docker.enabled = false singularity.enabled = false @@ -256,6 +254,12 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Set default registry for Docker and Podman independent of -profile +// Will not be used unless Docker / Podman are enabled +// Set to your registry if you have a mirror of containers +docker.registry = 'quay.io' +podman.registry = 'quay.io' + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true From 810bcdc2f2971d7d18481d0afea437d863a80c6a Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 31 May 2023 13:03:37 -0400 Subject: [PATCH 45/58] fix-typo --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b2a5b02..dc39e93d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: - name: Run nf-test run: | - nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}"" --tap=test.tap + nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap # If the test fails, output the software_versions.yml using the 'batcat' utility - name: Output log on failure @@ -172,7 +172,7 @@ jobs: - name: Run nf-test run: | - nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}"" --tap=test.tap + nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap # If the test fails, output the software_versions.yml using the 'batcat' utility - name: Output log on failure From 2697d306afc4c6b838f92c4f3bd1209aaffa374f Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 31 May 2023 22:54:22 -0400 Subject: [PATCH 46/58] 311-fix-ignore_3prime_r2 --- CHANGELOG.md | 1 + conf/modules.config | 2 +- lib/WorkflowMethylseq.groovy | 4 +++- nextflow.config | 2 ++ nextflow_schema.json | 16 ++++++++++++++- tests/pipeline/bismark/main.nf.test.snap | 20 +++++++++---------- .../pipeline/bismark_hisat/main.nf.test.snap | 12 +++++------ 7 files changed, 38 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57ff8d8b..1cb718fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - 🤖 GitHub Actions CI - pull_reqest to `dev` tests with NXF_VER `latest-everything` - 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` - 🤖 GitHub Actions CI - `fail-fast` set to false +- 🐛 fix `ignore_3prime_r2` param #299 ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16 diff --git a/conf/modules.config b/conf/modules.config index db8ae124..6bf158ba 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -211,7 +211,7 @@ process { params.nomeseq ? '--CX' : '', meta.single_end ? '' : (params.no_overlap ? ' --no_overlap' : '--include_overlap'), meta.single_end ? '' : (params.ignore_r2 > 0 ? "--ignore_r2 ${params.ignore_r2}" : ""), - meta.single_end ? '' : (params.ignore_3prime_r2 > 0 ? "--ignore_r2 ${params.ignore_3prime_r2}": "") + meta.single_end ? '' : (params.ignore_3prime_r2 > 0 ? "--ignore_3prime_r2 ${params.ignore_3prime_r2}": "") ].join(' ').trim() } publishDir = [ [ diff --git a/lib/WorkflowMethylseq.groovy b/lib/WorkflowMethylseq.groovy index 29066a19..f74108d9 100755 --- a/lib/WorkflowMethylseq.groovy +++ b/lib/WorkflowMethylseq.groovy @@ -15,7 +15,9 @@ class WorkflowMethylseq { if (!params.fasta) { - Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." + if (!params.bismark_index || params.aligner != 'bismark') { + Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." + } } } diff --git a/nextflow.config b/nextflow.config index 5ee81113..a304abed 100644 --- a/nextflow.config +++ b/nextflow.config @@ -65,7 +65,9 @@ params { // Bismark default is 0.2 (L,0,-0.2), Bowtie2 default is 0.6 (L,0,-0.6) meth_cutoff = null no_overlap = true + ignore = 0 ignore_r2 = 2 + ignore_3prime = 0 ignore_3prime_r2 = 2 known_splices = null local_alignment = false diff --git a/nextflow_schema.json b/nextflow_schema.json index abe2a784..8425b549 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -92,7 +92,7 @@ "mimetype": "text/plain", "pattern": "^\\S+\\.fn?a(sta)?$", "description": "Path to FASTA genome file", - "help_text": "If you have no genome reference available, the pipeline can build one using a FASTA file. This requires additional time and resources, so it's better to use a pre-build index if possible. You can use the command line option `--save_reference` to keep the generated references so that they can be added to your config and used again in the future." + "help_text": "If you have no genome reference available, the pipeline can build one using a FASTA file. This requires additional time and resources, so it's better to use a pre-build index if possible. You can use the command line option `--save_reference` to keep the generated references so that they can be added to your config and used again in the future. If aligner is Bismark and bismark_index is specified, this parameter is ignored." }, "fasta_index": { "type": "string", @@ -301,6 +301,13 @@ "help_text": "For paired-end reads it is theoretically possible that read_1 and read_2 overlap. To avoid scoring overlapping methylation calls twice, this is set to `true` by default. (Only methylation calls of read 1 are used since read 1 has historically higher quality basecalls than read 2). Whilst this option removes a bias towards more methylation calls in the center of sequenced fragments it may de facto remove a sizable proportion of the data. To count methylation data from both reads in overlapping regions, set this to `false`. ", "fa_icon": "fas fa-allergies" }, + "ignore": { + "type": "integer", + "default": 0, + "description": "Ignore methylation in first n bases of 5' end of R1", + "help_text": "Ignore the first bp from the 5' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove e.g. a restriction enzyme site at the start of each read or any other source of bias (such as PBAT-Seq data).", + "fa_icon": "far fa-eye-slash" + }, "ignore_r2": { "type": "integer", "default": 2, @@ -308,6 +315,13 @@ "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", "fa_icon": "far fa-eye-slash" }, + "ignore_3prime": { + "type": "integer", + "default": 0, + "description": "Ignore methylation in last n bases of 3' end of R1", + "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", + "fa_icon": "far fa-eye-slash" + }, "ignore_3prime_r2": { "type": "integer", "default": 2, diff --git a/tests/pipeline/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap index 9daf67f6..736dfb05 100644 --- a/tests/pipeline/bismark/main.nf.test.snap +++ b/tests/pipeline/bismark/main.nf.test.snap @@ -3,57 +3,57 @@ "content": [ "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_software_versions": { "content": [ "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_nomeseq_software_versions": { "content": [ "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_nomeseq": { "content": [ - "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,292c73be0dfaad92ee7d54ec3e5d0dd6", + "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,5ce32b88b28766d6fa0bf91ea9df77d6", "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,d70ee229a29b68db8634fa7d568b7c53", "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_rrbs_software_versions": { "content": [ "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" }, "single_end": { "content": [ - "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,292c73be0dfaad92ee7d54ec3e5d0dd6", + "Ecoli_10K_methylated_1_val_1_bismark_bt2_pe.deduplicated_splitting_report.txt:md5,5ce32b88b28766d6fa0bf91ea9df77d6", "SRR389222_sub1_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,d70ee229a29b68db8634fa7d568b7c53", "SRR389222_sub2_trimmed_bismark_bt2.deduplicated_splitting_report.txt:md5,3038a90a6e513c3b1c0de540439eb101", "genome_results.txt:md5,7886cc1a96ee892e49898b0d69ed66f2", "genome_results.txt:md5,79287f69ff6c8a308f0bb988c97f0e15", "genome_results.txt:md5,29e911b75d7160e964530e9607110164" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_rrbs": { "content": [ - "Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt:md5,9f72ee663677fb74fde3752b08350858", + "Ecoli_10K_methylated_R1_bismark_bt2_pe_splitting_report.txt:md5,fa089f6146df9667862970b6ddc998b0", "SRR389222_sub1_bismark_bt2_splitting_report.txt:md5,f59eac7a30ec7f1f690aac921a5591c0", "SRR389222_sub2.merged_bismark_bt2_splitting_report.txt:md5,43ca6c76e47ce2eb1c8dd6027475b5a6", "genome_results.txt:md5,41becdcecee6ebbdd46f472629731892", "genome_results.txt:md5,7a86634fd1c768d8302e071088641815", "genome_results.txt:md5,9e9b6841f3b3a46a299c61c516aa125e" ], - "timestamp": "2023-05-28T00:41:58+0000" + "timestamp": "2023-06-01T02:27:54+0000" } } \ No newline at end of file diff --git a/tests/pipeline/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap index f4497a3f..4fb90208 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test.snap +++ b/tests/pipeline/bismark_hisat/main.nf.test.snap @@ -3,34 +3,34 @@ "content": [ "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-28T00:47:31+0000" + "timestamp": "2023-06-01T02:43:30+0000" }, "single_end_rrbs_software_versions": { "content": [ "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" ], - "timestamp": "2023-05-28T00:47:31+0000" + "timestamp": "2023-06-01T02:43:30+0000" }, "single_end": { "content": [ - "Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt:md5,4e3ab3792be5eedde05d468a7e7f17e4", + "Ecoli_10K_methylated_1_val_1_bismark_hisat2_pe.deduplicated_splitting_report.txt:md5,bdec8746a4f29c56ddbe37a92f7d35dd", "SRR389222_sub1_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4020063dd8610b58b7b39c3ca62cef17", "SRR389222_sub2_trimmed_bismark_hisat2.deduplicated_splitting_report.txt:md5,4db2155fdfae1cb9c7c758edba256e2e", "genome_results.txt:md5,cd5d3b03821c038eb3bda24beb3ae938", "genome_results.txt:md5,9956ed1dcf51e6975f4a6ab8fe156aba", "genome_results.txt:md5,1133ea5d1376389c6a17def89d2e90ec" ], - "timestamp": "2023-05-28T00:47:31+0000" + "timestamp": "2023-06-01T02:43:30+0000" }, "single_end_rrbs": { "content": [ - "Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt:md5,f59c6bf8089c7dde3225cc577e0260e8", + "Ecoli_10K_methylated_R1_bismark_hisat2_pe_splitting_report.txt:md5,76868c2015864a3f7c2b4cd1738e4765", "SRR389222_sub1_bismark_hisat2_splitting_report.txt:md5,754ed7bf4d75acee466c48c05a11d331", "SRR389222_sub2.merged_bismark_hisat2_splitting_report.txt:md5,0629e1f8b4727efe8fd69500783eec61", "genome_results.txt:md5,3afd1eb7d4dc72a05e4d4de887d791e3", "genome_results.txt:md5,58a2aeeb5eacca4c766bb23fee8f9013", "genome_results.txt:md5,473022a29cf12d97e75451a4eea67c02" ], - "timestamp": "2023-05-28T00:47:31+0000" + "timestamp": "2023-06-01T02:43:30+0000" } } \ No newline at end of file From 8f47a536155f4bfae1033e109eef35c383cc5a76 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Thu, 1 Jun 2023 16:02:26 -0400 Subject: [PATCH 47/58] update-changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb718fa..1460ea66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Pipeline Updates - Updated template to nf-core/tools v2.8 -- Add `--bamqc_regions_file` parameter for targeted methylation sequencing data. +- Add `--bamqc_regions_file` parameter for targeted methylation sequencing data #302 - ✨ Add NF-TEST tests and snapshots for the pipeline test profile ### Bug fixes & refactoring @@ -16,6 +16,7 @@ - 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` - 🤖 GitHub Actions CI - `fail-fast` set to false - 🐛 fix `ignore_3prime_r2` param #299 +- 🐛 removed unused directory (`299e5c5`) ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16 From f72f23e91d9c6cc4a6f402e2f8d8b60acc6acc5c Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Thu, 1 Jun 2023 16:03:39 -0400 Subject: [PATCH 48/58] update-changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1460ea66..b2036803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Updated template to nf-core/tools v2.8 - Add `--bamqc_regions_file` parameter for targeted methylation sequencing data #302 -- ✨ Add NF-TEST tests and snapshots for the pipeline test profile +- ✨ Add NF-TEST tests and snapshots for the pipeline test profile #310 ### Bug fixes & refactoring From b8c7b327175f80127ad3d9e3a5b939ae30fe5583 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Thu, 1 Jun 2023 16:50:05 -0400 Subject: [PATCH 49/58] CI-disable-pr-commenter-ref-dev --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc39e93d..6601fff2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,7 @@ jobs: # Comment on the PR with the test results - name: Comment PR with execution number + if: github.event_name == 'pull_request' && github.ref != 'refs/heads/dev' uses: thollander/actions-comment-pull-request@v2 with: message: | @@ -183,6 +184,7 @@ jobs: # Comment on the PR with the test results - name: Comment PR with execution number + if: github.event_name == 'pull_request' && github.ref != 'refs/heads/dev' uses: thollander/actions-comment-pull-request@v2 with: message: | From 5e4f54091096826a59210e83a515c967b595a109 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Fri, 2 Jun 2023 14:03:14 -0400 Subject: [PATCH 50/58] bump version 2.4.0 --- .github/workflows/ci.yml | 10 ---------- CHANGELOG.md | 15 ++++++++------- nextflow.config | 2 +- tests/pipeline/bismark/main.nf.test.snap | 8 ++++---- tests/pipeline/bismark_hisat/main.nf.test.snap | 4 ++-- tests/pipeline/bwameth/main.nf.test.snap | 4 ++-- 6 files changed, 17 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6601fff2..b173ba13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,13 +181,3 @@ jobs: run: | sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - - # Comment on the PR with the test results - - name: Comment PR with execution number - if: github.event_name == 'pull_request' && github.ref != 'refs/heads/dev' - uses: thollander/actions-comment-pull-request@v2 - with: - message: | - _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ - comment_tag: execution - reactions: eyes diff --git a/CHANGELOG.md b/CHANGELOG.md index b2036803..a95cb801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/methylseq -## v2.4.0dev - [date] +## [v2.4.0](https://github.com/nf-core/methylseq/releases/tag/2.4.0) - 2023-06-02 ### Pipeline Updates @@ -10,13 +10,14 @@ ### Bug fixes & refactoring -- 🛠️ update index file channels to explicit value channels (`877ad9f`) -- 🐛 fix `params.test_data_base` in test and test_full configs (`1105d4f`) -- 🤖 GitHub Actions CI - pull_reqest to `dev` tests with NXF_VER `latest-everything` -- 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` -- 🤖 GitHub Actions CI - `fail-fast` set to false +- 🛠️ update index file channels to explicit value channels #310 +- 🐛 fix `params.test_data_base` in test and test_full configs #310 +- 🤖 GitHub Actions CI - pull_reqest to `dev` tests with NXF_VER `latest-everything` #310 +- 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310 +- 🤖 GitHub Actions CI - `fail-fast` set to false #310 +- 🐛 get to the bottom of index tests #278 - 🐛 fix `ignore_3prime_r2` param #299 -- 🐛 removed unused directory (`299e5c5`) +- 🐛 removed unused directory #297 ## [v2.3.0](https://github.com/nf-core/methylseq/releases/tag/2.3.0) - 2022-12-16 diff --git a/nextflow.config b/nextflow.config index a304abed..a07e8d57 100644 --- a/nextflow.config +++ b/nextflow.config @@ -287,7 +287,7 @@ manifest { description = """Methylation (Bisulfite-Sequencing) Best Practice analysis pipeline, part of the nf-core community.""" mainScript = 'main.nf' nextflowVersion = '!>=22.10.1' - version = '2.4.0dev' + version = '2.4.0' doi = '10.5281/zenodo.1343417' } diff --git a/tests/pipeline/bismark/main.nf.test.snap b/tests/pipeline/bismark/main.nf.test.snap index 736dfb05..27cf8920 100644 --- a/tests/pipeline/bismark/main.nf.test.snap +++ b/tests/pipeline/bismark/main.nf.test.snap @@ -1,19 +1,19 @@ { "clipping_params_software_versions": { "content": [ - "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_software_versions": { "content": [ - "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-06-01T02:27:54+0000" }, "single_end_nomeseq_software_versions": { "content": [ - "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-06-01T02:27:54+0000" }, @@ -30,7 +30,7 @@ }, "single_end_rrbs_software_versions": { "content": [ - "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-06-01T02:27:54+0000" }, diff --git a/tests/pipeline/bismark_hisat/main.nf.test.snap b/tests/pipeline/bismark_hisat/main.nf.test.snap index 4fb90208..c016095d 100644 --- a/tests/pipeline/bismark_hisat/main.nf.test.snap +++ b/tests/pipeline/bismark_hisat/main.nf.test.snap @@ -1,13 +1,13 @@ { "single_end_software_versions": { "content": [ - "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-06-01T02:43:30+0000" }, "single_end_rrbs_software_versions": { "content": [ - "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BISMARK_ALIGN={bismark=0.24.0}, BISMARK_GENOMEPREPARATION={bismark=0.24.0}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-06-01T02:43:30+0000" }, diff --git a/tests/pipeline/bwameth/main.nf.test.snap b/tests/pipeline/bwameth/main.nf.test.snap index 1014e5f0..146f52e1 100644 --- a/tests/pipeline/bwameth/main.nf.test.snap +++ b/tests/pipeline/bwameth/main.nf.test.snap @@ -1,13 +1,13 @@ { "single_end_software_versions": { "content": [ - "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, TRIMGALORE={cutadapt=3.4, trimgalore=0.6.7}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-05-28T00:53:43+0000" }, "single_end_rrbs_software_versions": { "content": [ - "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0dev}}" + "{BWAMETH_ALIGN={bwameth=0.2.2}, BWAMETH_INDEX={bwameth=0.2.2}, CAT_FASTQ={cat=8.30}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FASTQC={fastqc=0.11.9}, QUALIMAP_BAMQC={qualimap=2.2.2-dev}, SAMPLESHEET_CHECK={python=3.8.3}, Workflow={nf-core/methylseq=2.4.0}}" ], "timestamp": "2023-05-28T00:53:43+0000" }, From e9d932b6572a4a576302199d2df41349a9862770 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Fri, 2 Jun 2023 14:09:50 -0400 Subject: [PATCH 51/58] rm-pr-commenter --- .github/workflows/ci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b173ba13..7e714a58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,16 +98,6 @@ jobs: sudo apt install bat > /dev/null batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - # Comment on the PR with the test results - - name: Comment PR with execution number - if: github.event_name == 'pull_request' && github.ref != 'refs/heads/dev' - uses: thollander/actions-comment-pull-request@v2 - with: - message: | - _(execution **${{ github.run_id }}** / attempt **${{ github.run_attempt }}**)_ - comment_tag: execution - reactions: eyes - test-master: # Only run if the pull request is against the master branch if: ${{ (github.base_ref == 'master') }} From 1fb3dee80fa1de4f349c2f803b23f1b3ea571437 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Mon, 5 Jun 2023 08:05:27 -0400 Subject: [PATCH 52/58] rename ignore_r1 and ignore_3prime_r1 params --- .github/workflows/ci.yml | 4 ++-- README.md | 12 +++++------- nextflow.config | 4 ++-- nextflow_schema.json | 12 ++++++------ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e714a58..964b938d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - "bismark" - "bismark_hisat" - "bwameth" - profile: # TODO ["singularity", "conda"] + profile: - "docker" steps: - name: Check out pipeline code @@ -115,7 +115,7 @@ jobs: - "bismark" - "bismark_hisat" - "bwameth" - profile: # TODO ["singularity", "conda"] + profile: - "docker" steps: - name: Check out pipeline code diff --git a/README.md b/README.md index e51aa122..7eb930ce 100644 --- a/README.md +++ b/README.md @@ -47,22 +47,20 @@ Choose between workflows by using `--aligner bismark` (default, uses bowtie2 for > to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) > with `-profile test` before running the workflow on actual data. - - Now, you can run the pipeline using: ```bash diff --git a/nextflow.config b/nextflow.config index a07e8d57..7e1f1b86 100644 --- a/nextflow.config +++ b/nextflow.config @@ -65,9 +65,9 @@ params { // Bismark default is 0.2 (L,0,-0.2), Bowtie2 default is 0.6 (L,0,-0.6) meth_cutoff = null no_overlap = true - ignore = 0 + ignore_r1 = 0 ignore_r2 = 2 - ignore_3prime = 0 + ignore_3prime_r1 = 0 ignore_3prime_r2 = 2 known_splices = null local_alignment = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 8425b549..48bdd078 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -301,32 +301,32 @@ "help_text": "For paired-end reads it is theoretically possible that read_1 and read_2 overlap. To avoid scoring overlapping methylation calls twice, this is set to `true` by default. (Only methylation calls of read 1 are used since read 1 has historically higher quality basecalls than read 2). Whilst this option removes a bias towards more methylation calls in the center of sequenced fragments it may de facto remove a sizable proportion of the data. To count methylation data from both reads in overlapping regions, set this to `false`. ", "fa_icon": "fas fa-allergies" }, - "ignore": { + "ignore_r1": { "type": "integer", "default": 0, "description": "Ignore methylation in first n bases of 5' end of R1", - "help_text": "Ignore the first bp from the 5' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove e.g. a restriction enzyme site at the start of each read or any other source of bias (such as PBAT-Seq data).", + "help_text": "Ignore the first 0 bp from the 5' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove e.g. a restriction enzyme site at the start of each read or any other source of bias (such as PBAT-Seq data).", "fa_icon": "far fa-eye-slash" }, "ignore_r2": { "type": "integer", "default": 2, "description": "Ignore methylation in first n bases of 5' end of R2", - "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", + "help_text": "Ignore the first 2 bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", "fa_icon": "far fa-eye-slash" }, - "ignore_3prime": { + "ignore_3prime_r1": { "type": "integer", "default": 0, "description": "Ignore methylation in last n bases of 3' end of R1", - "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", + "help_text": "Ignore the first 0 bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", "fa_icon": "far fa-eye-slash" }, "ignore_3prime_r2": { "type": "integer", "default": 2, "description": "Ignore methylation in last n bases of 3' end of R2", - "help_text": "Ignore the last bp from the 3' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove unwanted biases from the end of reads.", + "help_text": "Ignore the last 2 bp from the 3' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove unwanted biases from the end of reads.", "fa_icon": "far fa-eye-slash" }, "known_splices": { From 0894a7389b013b421968a0b1a231ffb4f037962a Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Mon, 5 Jun 2023 08:08:50 -0400 Subject: [PATCH 53/58] code review fixes --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a95cb801..f7b672e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,11 @@ - 🛠️ update index file channels to explicit value channels #310 - 🐛 fix `params.test_data_base` in test and test_full configs #310 -- 🤖 GitHub Actions CI - pull_reqest to `dev` tests with NXF_VER `latest-everything` #310 -- 🤖 GitHub Actions CI - pull_reqest to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310 +- 🤖 GitHub Actions CI - pull_request to `dev` tests with NXF_VER `latest-everything` #310 +- 🤖 GitHub Actions CI - pull_request to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310 - 🤖 GitHub Actions CI - `fail-fast` set to false #310 - 🐛 get to the bottom of index tests #278 +- 🛠️ rename `ignore` -> `ignore_r1` and `ignore_3prime` -> `ignore_3prime_r1` params - 🐛 fix `ignore_3prime_r2` param #299 - 🐛 removed unused directory #297 From bc002805d31c25f83d5c3d3656c7a8744fbecb2a Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Wed, 7 Jun 2023 11:17:07 -0400 Subject: [PATCH 54/58] CI-skip-condition-for-dev --- .github/workflows/ci.yml | 101 +++++++-------------------------------- 1 file changed, 17 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 964b938d..707fdf2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,23 +16,15 @@ env: NXF_ANSI_LOG: false NFTEST_VER: "0.7.3" -# cancel-in-progress: true ensures that if a new push is made to the same branch while a previous run is still in progress, the previous run is cancelled concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true jobs: - test-dev: - # Only run if the pull request is against the dev branch or if the push is to the dev branch - if: github.base_ref == 'dev' || (github.event_name == 'push') - name: Run pipeline with test data on dev + test: + name: Run pipeline with test data runs-on: ubuntu-latest - # Add PR write permissions to allow commenting on the PR - permissions: - pull-requests: write - # Set up the matrix of different pipeline versions and profiles to test strategy: - # Do not fail fast if one of the tests fails fail-fast: false matrix: NXF_VER: @@ -44,89 +36,26 @@ jobs: profile: - "docker" steps: - - name: Check out pipeline code - uses: actions/checkout@v3 - - - name: Hash Github Workspace - id: hash_workspace - run: | - echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - - - name: Cache test data - id: cache-testdata - uses: actions/cache@v3 - with: - path: test-datasets/ - key: ${{ steps.hash_workspace.outputs.digest }} - - - name: Check out test data - if: steps.cache-testdata.outputs.cache-hit != 'true' - uses: actions/checkout@v3 - with: - repository: nf-core/test-datasets - ref: methylseq - path: test-datasets/ - - - name: Replace remote paths in samplesheets - run: | - for f in ./test-datasets/samplesheet/*csv; do - sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f - sed -i 's=https://raw.githubusercontent.com/nf-core/test-datasets/methylseq/=./test-datasets/=g' $f - echo "========== $f ============" - cat $f - echo "========================================" - done; - - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Install nf-test - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER - sudo mv nf-test /usr/local/bin/ - - - name: Run nf-test + # Skip if it's a pull_request to dev and NXF_VER is '22.10.1' + - name: Skip condition + id: condition run: | - nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" == "22.10.1" ]]; then + echo "skip=true" >> $GITHUB_OUTPUT + fi - # If the test fails, output the software_versions.yml using the 'batcat' utility - - name: Output log on failure - if: failure() - run: | - sudo apt install bat > /dev/null - batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml - - test-master: - # Only run if the pull request is against the master branch - if: ${{ (github.base_ref == 'master') }} - name: Run pipeline with test data on master - runs-on: ubuntu-latest - # Add PR write permissions to allow commenting on the PR - permissions: - pull-requests: write - # Set up the matrix of different pipeline versions and profiles to test - strategy: - matrix: - NXF_VER: - - "latest-everything" - aligner: - - "bismark" - - "bismark_hisat" - - "bwameth" - profile: - - "docker" - steps: - name: Check out pipeline code + if: steps.condition.outputs.skip != 'true' uses: actions/checkout@v3 - name: Hash Github Workspace + if: steps.condition.outputs.skip != 'true' id: hash_workspace run: | echo "digest=$(echo methylseq2_${{ github.workspace }} | md5sum | cut -c 1-25)" >> $GITHUB_OUTPUT - name: Cache test data + if: steps.condition.outputs.skip != 'true' id: cache-testdata uses: actions/cache@v3 with: @@ -134,7 +63,7 @@ jobs: key: ${{ steps.hash_workspace.outputs.digest }} - name: Check out test data - if: steps.cache-testdata.outputs.cache-hit != 'true' + if: steps.cache-testdata.outputs.cache-hit != 'true' && steps.condition.outputs.skip != 'true' uses: actions/checkout@v3 with: repository: nf-core/test-datasets @@ -142,6 +71,7 @@ jobs: path: test-datasets/ - name: Replace remote paths in samplesheets + if: steps.condition.outputs.skip != 'true' run: | for f in ./test-datasets/samplesheet/*csv; do sed -i 's=https://github.com/nf-core/test-datasets/raw/methylseq/=./test-datasets/=g' $f @@ -152,16 +82,19 @@ jobs: done; - name: Install Nextflow + if: steps.condition.outputs.skip != 'true' uses: nf-core/setup-nextflow@v1 with: version: "${{ matrix.NXF_VER }}" - name: Install nf-test + if: steps.condition.outputs.skip != 'true' run: | wget -qO- https://code.askimed.com/install/nf-test | bash -s $NFTEST_VER sudo mv nf-test /usr/local/bin/ - name: Run nf-test + if: steps.condition.outputs.skip != 'true' run: | nf-test test tests/pipeline/${{ matrix.aligner }}/ --profile "test,${{ matrix.profile }}" --tap=test.tap @@ -170,4 +103,4 @@ jobs: if: failure() run: | sudo apt install bat > /dev/null - batcat --decorations=always --color=always /home/runner/work/methylseq/methylseq/.nf-test/tests/*/output/pipeline_info/software_versions.yml + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml From 75d6d87f0310a738121306dfd5067bb97075252b Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Fri, 9 Jun 2023 14:55:09 +0000 Subject: [PATCH 55/58] Make R1 ignore parameters actually do something --- conf/modules.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/modules.config b/conf/modules.config index 6bf158ba..50017087 100755 --- a/conf/modules.config +++ b/conf/modules.config @@ -209,6 +209,8 @@ process { params.comprehensive ? ' --comprehensive --merge_non_CpG' : '', params.meth_cutoff ? " --cutoff ${params.meth_cutoff}" : '', params.nomeseq ? '--CX' : '', + params.ignore_r1 > 0 ? '--ignore ${params.ignore_r1}' : '', + params.ignore_3prime_r1 > 0 ? '--ignore_3prime ${params.ignore_3prime_r1}' : '', meta.single_end ? '' : (params.no_overlap ? ' --no_overlap' : '--include_overlap'), meta.single_end ? '' : (params.ignore_r2 > 0 ? "--ignore_r2 ${params.ignore_r2}" : ""), meta.single_end ? '' : (params.ignore_3prime_r2 > 0 ? "--ignore_3prime_r2 ${params.ignore_3prime_r2}": "") From a4e831ae20133756cbc8a36fca4fbeb16dddb256 Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Fri, 9 Jun 2023 12:25:17 -0400 Subject: [PATCH 56/58] CI-update-skip-dev-condition and update changelog --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 707fdf2a..e5cc0af0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: - name: Skip condition id: condition run: | - if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" == "22.10.1" ]]; then + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then echo "skip=true" >> $GITHUB_OUTPUT fi diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b672e4..42816150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - 🤖 GitHub Actions CI - pull_request to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310 - 🤖 GitHub Actions CI - `fail-fast` set to false #310 - 🐛 get to the bottom of index tests #278 +- ✨ Support for Bismark methylation extraction `ignore` and `ignore_3prime` parameters when `ignore_r1` or `ignore_3prime_r1` are greater than 0. #322 - 🛠️ rename `ignore` -> `ignore_r1` and `ignore_3prime` -> `ignore_3prime_r1` params - 🐛 fix `ignore_3prime_r2` param #299 - 🐛 removed unused directory #297 From 06c95122f654d5265430e5cea60570263fb2bd7e Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Sat, 10 Jun 2023 09:56:13 -0400 Subject: [PATCH 57/58] CI-update-skip-dev-condition, update README and changelog --- .github/workflows/ci.yml | 3 ++- CHANGELOG.md | 3 ++- README.md | 10 +++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 707fdf2a..f0f49267 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,7 @@ jobs: fail-fast: false matrix: NXF_VER: + - "22.10.1" - "latest-everything" aligner: - "bismark" @@ -40,7 +41,7 @@ jobs: - name: Skip condition id: condition run: | - if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" == "22.10.1" ]]; then + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then echo "skip=true" >> $GITHUB_OUTPUT fi diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b672e4..696d7219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ - 🤖 GitHub Actions CI - pull_request to `master` tests with NXF_VER `22.10.1` & `latest-everything` #310 - 🤖 GitHub Actions CI - `fail-fast` set to false #310 - 🐛 get to the bottom of index tests #278 -- 🛠️ rename `ignore` -> `ignore_r1` and `ignore_3prime` -> `ignore_3prime_r1` params +- ✨ Support for Bismark methylation extraction `ignore` and `ignore_3prime` parameters when `ignore_r1` or `ignore_3prime_r1` are greater than 0. #322 +- 🛠️ rename `ignore` -> `ignore_r1` and `ignore_3prime` -> `ignore_3prime_r1` params #322 - 🐛 fix `ignore_3prime_r2` param #299 - 🐛 removed unused directory #297 diff --git a/README.md b/README.md index 7eb930ce..8e039f1d 100644 --- a/README.md +++ b/README.md @@ -52,11 +52,11 @@ First, prepare a samplesheet with your input data that looks as follows: `samplesheet.csv`: ```csv -sample,fastq_1,fastq_2,genome -SRR389222_sub1,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz,, -SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz,, -SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub3.fastq.gz,, -Ecoli_10K_methylated,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz, +sample,fastq_1,fastq_2 +SRR389222_sub1,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz +SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz +SRR389222_sub2,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub3.fastq.gz +Ecoli_10K_methylated,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R1.fastq.gz,https://github.com/nf-core/test-datasets/raw/methylseq/testdata/Ecoli_10K_methylated_R2.fastq.gz ``` Each row represents a fastq file (single-end) or a pair of fastq files (paired end). From fd81abeb5b0be72dd74ff3a4d7bc4834654121df Mon Sep 17 00:00:00 2001 From: Sateesh Peri Date: Mon, 12 Jun 2023 13:18:02 -0400 Subject: [PATCH 58/58] fix ignore params help text --- nextflow_schema.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 48bdd078..ff4927ef 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -305,28 +305,28 @@ "type": "integer", "default": 0, "description": "Ignore methylation in first n bases of 5' end of R1", - "help_text": "Ignore the first 0 bp from the 5' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove e.g. a restriction enzyme site at the start of each read or any other source of bias (such as PBAT-Seq data).", + "help_text": "Ignore the first bp from the 5' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove e.g. a restriction enzyme site at the start of each read or any other source of bias (such as PBAT-Seq data).", "fa_icon": "far fa-eye-slash" }, "ignore_r2": { "type": "integer", "default": 2, "description": "Ignore methylation in first n bases of 5' end of R2", - "help_text": "Ignore the first 2 bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", + "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", "fa_icon": "far fa-eye-slash" }, "ignore_3prime_r1": { "type": "integer", "default": 0, "description": "Ignore methylation in last n bases of 3' end of R1", - "help_text": "Ignore the first 0 bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", + "help_text": "Ignore the first bp from the 5' end of Read 2 of paired-end sequencing results only. Since the first couple of bases in Read 2 of BS-Seq experiments show a severe bias towards non-methylation as a result of end-repairing sonicated fragments with unmethylated cytosines (see M-bias plot), it is recommended that the first couple of bp of Read 2 are removed before starting downstream analysis. Please see the section on M-bias plots in the Bismark User Guide for more details.", "fa_icon": "far fa-eye-slash" }, "ignore_3prime_r2": { "type": "integer", "default": 2, "description": "Ignore methylation in last n bases of 3' end of R2", - "help_text": "Ignore the last 2 bp from the 3' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove unwanted biases from the end of reads.", + "help_text": "Ignore the last bp from the 3' end of Read 1 (or single-end alignment files) when processing the methylation call string. This can remove unwanted biases from the end of reads.", "fa_icon": "far fa-eye-slash" }, "known_splices": {