From 03a9f356a1a333923c1177c2912fa7bc61bb46f3 Mon Sep 17 00:00:00 2001 From: Evangelos Karatzas <32259775+vagkaratzas@users.noreply.github.com> Date: Thu, 14 Nov 2024 13:51:04 +0200 Subject: [PATCH 01/12] =?UTF-8?q?hmmalign=20ported=20to=20nf-test,=20versi?= =?UTF-8?q?on=20bump,=20wave=20container,=20standard=20ou=E2=80=A6=20(#699?= =?UTF-8?q?2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hmmalign ported to nf-test, version bump, wave container, standard output format, hmmbuild version bump, wave container, subworkflow updated accordingly --- .../nf-core/hmmer/hmmalign/environment.yml | 2 +- modules/nf-core/hmmer/hmmalign/main.nf | 21 ++- modules/nf-core/hmmer/hmmalign/meta.yml | 7 +- .../nf-core/hmmer/hmmalign/tests/main.nf.test | 58 +++++++ .../hmmer/hmmalign/tests/main.nf.test.snap | 68 ++++++++ .../nf-core/hmmer/hmmbuild/environment.yml | 2 +- modules/nf-core/hmmer/hmmbuild/main.nf | 4 +- .../nf-core/hmmer/hmmbuild/tests/main.nf.test | 12 +- .../hmmer/hmmbuild/tests/main.nf.test.snap | 14 +- .../nf-core/fasta_newick_epang_gappa/main.nf | 4 +- .../tests/main.nf.test.snap | 164 +++++++++--------- tests/config/pytest_modules.yml | 3 - tests/modules/nf-core/hmmer/hmmalign/main.nf | 30 ---- .../nf-core/hmmer/hmmalign/nextflow.config | 8 - tests/modules/nf-core/hmmer/hmmalign/test.yml | 16 -- 15 files changed, 247 insertions(+), 166 deletions(-) create mode 100644 modules/nf-core/hmmer/hmmalign/tests/main.nf.test create mode 100644 modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap delete mode 100644 tests/modules/nf-core/hmmer/hmmalign/main.nf delete mode 100644 tests/modules/nf-core/hmmer/hmmalign/nextflow.config delete mode 100644 tests/modules/nf-core/hmmer/hmmalign/test.yml diff --git a/modules/nf-core/hmmer/hmmalign/environment.yml b/modules/nf-core/hmmer/hmmalign/environment.yml index 7c62eac7e3a..c5ddec5d912 100644 --- a/modules/nf-core/hmmer/hmmalign/environment.yml +++ b/modules/nf-core/hmmer/hmmalign/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::hmmer=3.3.2 + - bioconda::hmmer=3.4 diff --git a/modules/nf-core/hmmer/hmmalign/main.nf b/modules/nf-core/hmmer/hmmalign/main.nf index 00ae8da2b95..39b17c787d8 100644 --- a/modules/nf-core/hmmer/hmmalign/main.nf +++ b/modules/nf-core/hmmer/hmmalign/main.nf @@ -4,16 +4,16 @@ process HMMER_HMMALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/hmmer:3.3.2--h1b792b2_1' : - 'biocontainers/hmmer:3.3.2--h1b792b2_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/07/07c4cbd91c4459dc86b13b5cd799cacba96b27d66c276485550d299c7a4c6f8a/data' : + 'community.wave.seqera.io/library/hmmer:3.4--cb5d2dd2e85974ca' }" input: tuple val(meta), path(fasta) path hmm output: - tuple val(meta), path("*.sthlm.gz"), emit: sthlm - path "versions.yml" , emit: versions + tuple val(meta), path("*.sto.gz"), emit: sto + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -25,7 +25,18 @@ process HMMER_HMMALIGN { hmmalign \\ $args \\ $hmm \\ - $fasta | gzip -c > ${prefix}.sthlm.gz + $fasta | gzip -c > ${prefix}.sto.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + hmmer: \$(hmmalign -h | grep -o '^# HMMER [0-9.]*' | sed 's/^# HMMER *//') + END_VERSIONS + """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + echo | gzip > ${prefix}.sto.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/hmmer/hmmalign/meta.yml b/modules/nf-core/hmmer/hmmalign/meta.yml index 366cf4f5a58..fb16ba6822e 100644 --- a/modules/nf-core/hmmer/hmmalign/meta.yml +++ b/modules/nf-core/hmmer/hmmalign/meta.yml @@ -30,16 +30,16 @@ input: description: A gzipped HMM file pattern: "*.hmm.gz" output: - - sthlm: + - sto: - meta: type: map description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - "*.sthlm.gz": + - "*.sto.gz": type: file description: Multiple alignment in gzipped Stockholm format - pattern: "*.sthlm.gz" + pattern: "*.sto.gz" - versions: - versions.yml: type: file @@ -51,3 +51,4 @@ authors: maintainers: - "@erikrikarddaniel" - "@jfy133" + - "@vagkaratzas" diff --git a/modules/nf-core/hmmer/hmmalign/tests/main.nf.test b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test new file mode 100644 index 00000000000..7b0368fc630 --- /dev/null +++ b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test @@ -0,0 +1,58 @@ + +nextflow_process { + + name "Test Process HMMER_HMMALIGN" + script "../main.nf" + process "HMMER_HMMALIGN" + + tag "modules" + tag "modules_nfcore" + tag "hmmer" + tag "hmmer/hmmalign" + + test("test-hmmer-hmmalign") { + + when { + process { + """ + input[0] = [ + [ id: 'test' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/e_coli_k12_16s.fna.gz', checkIfExists: true) + ] + input[1] = file(params.modules_testdata_base_path + 'delete_me/hmmer/bac.16S_rRNA.hmm.gz', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("test-hmmer-hmmalign-stub") { + options '-stub' + + when { + process { + """ + input[0] = [ + [ id: 'test' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/e_coli_k12_16s.fna.gz', checkIfExists: true) + ] + input[1] = file(params.modules_testdata_base_path + 'delete_me/hmmer/bac.16S_rRNA.hmm.gz', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + +} diff --git a/modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap new file mode 100644 index 00000000000..ae65cacb18b --- /dev/null +++ b/modules/nf-core/hmmer/hmmalign/tests/main.nf.test.snap @@ -0,0 +1,68 @@ +{ + "test-hmmer-hmmalign": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,4ae989d5ade2aaae9578cb88ba031e8f" + ] + ], + "1": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ], + "sto": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,4ae989d5ade2aaae9578cb88ba031e8f" + ] + ], + "versions": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-13T12:37:26.551497424" + }, + "test-hmmer-hmmalign-stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ], + "sto": [ + [ + { + "id": "test" + }, + "test.sto.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,601e13e7a1e6057766d862a828d501c4" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-13T12:37:32.244343836" + } +} \ No newline at end of file diff --git a/modules/nf-core/hmmer/hmmbuild/environment.yml b/modules/nf-core/hmmer/hmmbuild/environment.yml index 7c62eac7e3a..c5ddec5d912 100644 --- a/modules/nf-core/hmmer/hmmbuild/environment.yml +++ b/modules/nf-core/hmmer/hmmbuild/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::hmmer=3.3.2 + - bioconda::hmmer=3.4 diff --git a/modules/nf-core/hmmer/hmmbuild/main.nf b/modules/nf-core/hmmer/hmmbuild/main.nf index b4629cb47a6..8eed7fcd7ad 100644 --- a/modules/nf-core/hmmer/hmmbuild/main.nf +++ b/modules/nf-core/hmmer/hmmbuild/main.nf @@ -4,8 +4,8 @@ process HMMER_HMMBUILD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/hmmer:3.3.2--h87f3376_2': - 'biocontainers/hmmer:3.3.2--h1b792b2_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/07/07c4cbd91c4459dc86b13b5cd799cacba96b27d66c276485550d299c7a4c6f8a/data' : + 'community.wave.seqera.io/library/hmmer:3.4--cb5d2dd2e85974ca' }" input: tuple val(meta), path(alignment) diff --git a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test index d7c058255ea..635f5b661f1 100644 --- a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test +++ b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test @@ -16,9 +16,9 @@ nextflow_process { process { """ input[0] = [ - [ id: 'PF14720' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) - ] + [ id: 'PF14720' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) + ] input[1] = [] """ @@ -46,9 +46,9 @@ nextflow_process { process { """ input[0] = [ - [ id: 'PF14720' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) - ] + [ id: 'PF14720' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) + ] input[1] = [] """ diff --git a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap index 8cdff17c142..79317b89ebd 100644 --- a/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap +++ b/modules/nf-core/hmmer/hmmbuild/tests/main.nf.test.snap @@ -14,7 +14,7 @@ "PF14720.hmmbuild.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "2": [ - "versions.yml:md5,6f4796b714cb0d5981cd12d9e89cb599" + "versions.yml:md5,f8a0bffcbbc58404752849403812905b" ], "hmm": [ [ @@ -28,28 +28,28 @@ "PF14720.hmmbuild.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "versions": [ - "versions.yml:md5,6f4796b714cb0d5981cd12d9e89cb599" + "versions.yml:md5,f8a0bffcbbc58404752849403812905b" ] } ], "meta": { "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nextflow": "23.04.5" }, - "timestamp": "2024-09-05T21:38:44.751216" + "timestamp": "2024-11-13T12:52:29.794123574" }, "test-hmmer-hmmbuild": { "content": [ "PF14720.hmm.gz", "PF14720.hmmbuild.txt", [ - "versions.yml:md5,6f4796b714cb0d5981cd12d9e89cb599" + "versions.yml:md5,f8a0bffcbbc58404752849403812905b" ] ], "meta": { "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nextflow": "23.04.5" }, - "timestamp": "2024-09-05T21:49:25.772308" + "timestamp": "2024-11-13T12:52:23.95935055" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf b/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf index 28016014f94..723f4a2d91a 100644 --- a/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf +++ b/subworkflows/nf-core/fasta_newick_epang_gappa/main.nf @@ -82,10 +82,10 @@ workflow FASTA_NEWICK_EPANG_GAPPA { ch_versions = ch_versions.mix(HMMER_HMMALIGNQUERY.out.versions) // 1.d Mask the alignments (Add '--rf-is-mask' ext.args in config for the process.) - HMMER_MASKREF ( HMMER_HMMALIGNREF.out.sthlm.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) + HMMER_MASKREF ( HMMER_HMMALIGNREF.out.sto.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) ch_versions = ch_versions.mix(HMMER_MASKREF.out.versions) - HMMER_MASKQUERY ( HMMER_HMMALIGNQUERY.out.sthlm.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) + HMMER_MASKQUERY ( HMMER_HMMALIGNQUERY.out.sto.map { [ it[0], it[1], [], [], [], [], [], [] ] }, [] ) ch_versions = ch_versions.mix(HMMER_MASKQUERY.out.versions) // 1.e Reformat alignments to "afa" (aligned fasta) diff --git a/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap b/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap index e6627edbe99..207d28e5013 100644 --- a/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fasta_newick_epang_gappa/tests/main.nf.test.snap @@ -4,201 +4,201 @@ "hmmer.taxonomy.per_query.tsv" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:45:46.123551197" + "timestamp": "2024-11-14T10:16:44.874521222" }, "mafft.taxonomy.profile.tsv_lines": { "content": "eb4ae24c330c4b4ab768f7d1c26c7215", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:24:06.138041" + "timestamp": "2024-11-14T10:16:59.978336429" + }, + "hmmer.graft.placement.epa_result.newick_lines": { + "content": "8533b159d543f75b92efcfb4fd330280", + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-14T10:16:44.866852673" }, "hmmer.taxonomy.profile.tsv": { "content": [ "hmmer.taxonomy.profile.tsv" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-04-05T20:45:46.115530176" - }, - "hmmer.graft.placement.epa_result.newick_lines": { - "content": "8533b159d543f75b92efcfb4fd330280", - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T10:55:06.361779" + "timestamp": "2024-11-14T10:16:44.868720482" }, "hmmer.taxonomy.profile.tsv_lines": { "content": "8533b159d543f75b92efcfb4fd330280", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T10:59:31.223864" + "timestamp": "2024-11-14T10:16:44.871385161" }, "mafft.graft.placement.epa_result.newick_lines": { "content": "eb4ae24c330c4b4ab768f7d1c26c7215", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:24:06.117996" + "timestamp": "2024-11-14T10:16:59.973967772" }, "mafft.taxonomy.per_query.tsv_lines": { "content": "eb4ae24c330c4b4ab768f7d1c26c7215", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:24:06.175662" + "timestamp": "2024-11-14T10:16:59.98411855" }, "nucl_hmmer.versions.yml": { "content": [ "versions.yml" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:42.212896" + "timestamp": "2024-11-14T10:17:41.730837855" }, "hmmer.graft.placement.epa_result.newick": { "content": [ "hmmer.graft.hmmer.epa_result.newick" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:45:46.105219765" + "timestamp": "2024-11-14T10:16:44.863462591" }, "hmmer.versions.yml": { "content": [ "versions.yml" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:05.782093" + "timestamp": "2024-11-14T10:16:44.880743216" }, - "nucl_hmmer.graft.placement.epa_result.newick": { + "hmmer.epa_result.jplace.gz": { "content": [ - "nucl_hmmer.graft.nucl_hmmer.epa_result.newick" + "hmmer.epa_result.jplace.gz" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:37:46.19414" + "timestamp": "2024-11-14T10:16:44.857734455" }, - "hmmer.epa_result.jplace.gz": { + "nucl_hmmer.graft.placement.epa_result.newick": { "content": [ - "hmmer.epa_result.jplace.gz" + "nucl_hmmer.graft.nucl_hmmer.epa_result.newick" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:56:49.564967" + "timestamp": "2024-11-14T10:17:41.728034651" }, "hmmer.taxonomy.per_query.tsv_lines": { "content": "8533b159d543f75b92efcfb4fd330280", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T10:59:31.236689" + "timestamp": "2024-11-14T10:16:44.878231148" }, "mafft.taxonomy.per_query.tsv": { "content": [ "mafft.taxonomy.per_query.tsv" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:57:13.505168194" + "timestamp": "2024-11-14T10:16:59.980370338" }, "mafft.graft.placement.epa_result.newick": { "content": [ "mafft.graft.mafft.epa_result.newick" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:57:13.49856233" + "timestamp": "2024-11-14T10:16:59.972095017" }, "nucl_hmmer.epa_result.jplace.gz": { "content": [ "nucl_hmmer.epa_result.jplace.gz" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:44:49.494246" + "timestamp": "2024-11-14T10:17:41.725684052" }, "nucl_hmmer.versions.yml_lines": { - "content": "664dc00d47aecd41a420dc6ba7ef4eea", + "content": "eafb4184937621216ff03df17c4245a6", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:42.217138" + "timestamp": "2024-11-14T10:17:41.733022413" }, - "mafft.taxonomy.profile.tsv": { - "content": [ - "mafft.taxonomy.profile.tsv" - ], + "hmmer.versions.yml_lines": { + "content": "eafb4184937621216ff03df17c4245a6", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-05T20:57:13.501534227" + "timestamp": "2024-11-14T10:16:44.884598465" }, "mafft.epa_result.jplace.gz": { "content": [ "mafft.epa_result.jplace.gz" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T11:57:06.815658" + "timestamp": "2024-11-14T10:16:59.970209461" }, - "hmmer.versions.yml_lines": { - "content": "664dc00d47aecd41a420dc6ba7ef4eea", + "mafft.taxonomy.profile.tsv": { + "content": [ + "mafft.taxonomy.profile.tsv" + ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:05.849215" + "timestamp": "2024-11-14T10:16:59.975738553" }, "mafft.versions.yml": { "content": [ "versions.yml" ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:20.88609" + "timestamp": "2024-11-14T10:16:59.986610634" }, "mafft.versions.yml_lines": { "content": "f5957093def0191f32bb294000fb9242", "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.0", + "nextflow": "23.04.5" }, - "timestamp": "2024-04-08T13:37:20.890963" + "timestamp": "2024-11-14T10:16:59.988948939" } } \ No newline at end of file diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 55984e2de9e..2e0e6420c33 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -299,9 +299,6 @@ hmmer/eslalimask: hmmer/eslreformat: - modules/nf-core/hmmer/eslreformat/** - tests/modules/nf-core/hmmer/eslreformat/** -hmmer/hmmalign: - - modules/nf-core/hmmer/hmmalign/** - - tests/modules/nf-core/hmmer/hmmalign/** hmtnote/annotate: - modules/nf-core/hmtnote/annotate/** - tests/modules/nf-core/hmtnote/annotate/** diff --git a/tests/modules/nf-core/hmmer/hmmalign/main.nf b/tests/modules/nf-core/hmmer/hmmalign/main.nf deleted file mode 100644 index 64814641c3d..00000000000 --- a/tests/modules/nf-core/hmmer/hmmalign/main.nf +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { HMMER_HMMALIGN as HMMER_HMMALIGN } from '../../../../../modules/nf-core/hmmer/hmmalign/main.nf' -include { HMMER_HMMALIGN as HMMER_HMMALIGN_PREFIX } from '../../../../../modules/nf-core/hmmer/hmmalign/main.nf' - -workflow test_hmmer_hmmalign { - - input = [ - [ id:'test' ], // meta map - file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/e_coli_k12_16s.fna.gz') // Change to params.test_data syntax after the data is included in ./tests/config/test_data.config - ] - - hmm = file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/bac.16S_rRNA.hmm.gz') - - HMMER_HMMALIGN ( input, hmm ) -} - -workflow test_hmmer_hmmalign_prefix { - - input = [ - [ id:'test' ], // meta map - file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/e_coli_k12_16s.fna.gz') // Change to params.test_data syntax after the data is included in ./tests/config/test_data.config - ] - - hmm = file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/hmmer/bac.16S_rRNA.hmm.gz') - - HMMER_HMMALIGN_PREFIX ( input, hmm ) -} diff --git a/tests/modules/nf-core/hmmer/hmmalign/nextflow.config b/tests/modules/nf-core/hmmer/hmmalign/nextflow.config deleted file mode 100644 index b2115af55d9..00000000000 --- a/tests/modules/nf-core/hmmer/hmmalign/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: HMMER_HMMALIGN_PREFIX { - ext.prefix = { "${meta.id}.prefix" } - } -} diff --git a/tests/modules/nf-core/hmmer/hmmalign/test.yml b/tests/modules/nf-core/hmmer/hmmalign/test.yml deleted file mode 100644 index 2f483956c98..00000000000 --- a/tests/modules/nf-core/hmmer/hmmalign/test.yml +++ /dev/null @@ -1,16 +0,0 @@ -- name: hmmer hmmalign test_hmmer_hmmalign - command: nextflow run ./tests/modules/nf-core/hmmer/hmmalign -entry test_hmmer_hmmalign -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/hmmer/hmmalign/nextflow.config - tags: - - hmmer - - hmmer/hmmalign - files: - - path: output/hmmer/test.sthlm.gz - md5sum: ddaa8b96291edf4e1a929a224329161b -- name: hmmer hmmalign test_hmmer_hmmalign_prefix - command: nextflow run ./tests/modules/nf-core/hmmer/hmmalign -entry test_hmmer_hmmalign_prefix -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/hmmer/hmmalign/nextflow.config - tags: - - hmmer - - hmmer/hmmalign - files: - - path: output/hmmer/test.prefix.sthlm.gz - md5sum: ddaa8b96291edf4e1a929a224329161b From 920895254d42e874c035d07d783d40da7395c2da Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Thu, 14 Nov 2024 18:12:33 +0530 Subject: [PATCH 02/12] Fix/biscuit apps (#6979) * init update biscuit apps * update index module + test; use seqera container * update align module + test; use seqera container * update blaster container + test * update container + test * update container + test * update mergecg container + test + rm tags.yml * update pileup container + test * update index container * update qc container + test * update vcf2bed container + test * fix biscuit, samtools cpu determination --- modules/nf-core/biscuit/align/environment.yml | 4 +- modules/nf-core/biscuit/align/main.nf | 33 ++++-- modules/nf-core/biscuit/align/meta.yml | 21 +++- .../nf-core/biscuit/align/tests/main.nf.test | 70 ++++++++++-- .../biscuit/align/tests/main.nf.test.snap | 87 ++++++++++++-- modules/nf-core/biscuit/align/tests/tags.yml | 3 - .../biscuit/biscuitblaster/environment.yml | 4 +- .../nf-core/biscuit/biscuitblaster/main.nf | 32 +++--- .../nf-core/biscuit/biscuitblaster/meta.yml | 24 +++- .../biscuit/biscuitblaster/tests/main.nf.test | 58 ++++++---- .../biscuitblaster/tests/main.nf.test.snap | 53 ++++----- .../biscuit/biscuitblaster/tests/tags.yml | 2 - .../nf-core/biscuit/bsconv/environment.yml | 2 +- modules/nf-core/biscuit/bsconv/main.nf | 19 ++- modules/nf-core/biscuit/bsconv/meta.yml | 26 ++++- .../nf-core/biscuit/bsconv/tests/main.nf.test | 49 +++++--- .../biscuit/bsconv/tests/main.nf.test.snap | 63 ++-------- modules/nf-core/biscuit/bsconv/tests/tags.yml | 2 - .../nf-core/biscuit/epiread/environment.yml | 4 +- modules/nf-core/biscuit/epiread/main.nf | 31 +++-- modules/nf-core/biscuit/epiread/meta.yml | 35 +++++- .../biscuit/epiread/tests/main.nf.test | 92 +++++++++++---- .../biscuit/epiread/tests/main.nf.test.snap | 92 +++++++-------- modules/nf-core/biscuit/index/environment.yml | 2 +- modules/nf-core/biscuit/index/main.nf | 11 +- modules/nf-core/biscuit/index/meta.yml | 18 ++- .../nf-core/biscuit/index/tests/main.nf.test | 17 +-- .../biscuit/index/tests/main.nf.test.snap | 108 +++++++++++------- .../nf-core/biscuit/mergecg/environment.yml | 4 +- modules/nf-core/biscuit/mergecg/main.nf | 21 ++-- modules/nf-core/biscuit/mergecg/meta.yml | 21 +++- .../biscuit/mergecg/tests/main.nf.test | 47 ++++---- .../biscuit/mergecg/tests/main.nf.test.snap | 36 +++--- .../nf-core/biscuit/mergecg/tests/tags.yml | 2 - .../nf-core/biscuit/pileup/environment.yml | 4 +- modules/nf-core/biscuit/pileup/main.nf | 15 +-- modules/nf-core/biscuit/pileup/meta.yml | 19 ++- .../nf-core/biscuit/pileup/tests/main.nf.test | 36 ++++-- .../biscuit/pileup/tests/main.nf.test.snap | 64 +++++------ modules/nf-core/biscuit/qc/environment.yml | 2 +- modules/nf-core/biscuit/qc/main.nf | 13 ++- modules/nf-core/biscuit/qc/meta.yml | 20 +++- modules/nf-core/biscuit/qc/tests/main.nf.test | 21 +++- .../biscuit/qc/tests/main.nf.test.snap | 28 ++--- .../nf-core/biscuit/vcf2bed/environment.yml | 4 +- modules/nf-core/biscuit/vcf2bed/main.nf | 12 +- modules/nf-core/biscuit/vcf2bed/meta.yml | 1 + .../biscuit/vcf2bed/tests/main.nf.test | 14 +-- .../biscuit/vcf2bed/tests/main.nf.test.snap | 24 ++-- 49 files changed, 839 insertions(+), 531 deletions(-) delete mode 100644 modules/nf-core/biscuit/align/tests/tags.yml delete mode 100644 modules/nf-core/biscuit/biscuitblaster/tests/tags.yml delete mode 100644 modules/nf-core/biscuit/bsconv/tests/tags.yml delete mode 100644 modules/nf-core/biscuit/mergecg/tests/tags.yml diff --git a/modules/nf-core/biscuit/align/environment.yml b/modules/nf-core/biscuit/align/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/align/environment.yml +++ b/modules/nf-core/biscuit/align/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/align/main.nf b/modules/nf-core/biscuit/align/main.nf index 5154748725d..0d43dc05183 100644 --- a/modules/nf-core/biscuit/align/main.nf +++ b/modules/nf-core/biscuit/align/main.nf @@ -4,12 +4,13 @@ process BISCUIT_ALIGN { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(reads) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: tuple val(meta), path("*.bam"), emit: bam @@ -23,18 +24,34 @@ process BISCUIT_ALIGN { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1) - def samtools_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit align \\ $args \\ -@ $biscuit_cpus \\ - \$INDEX \\ + $index/$fasta \\ $reads \\ - | samtools sort $args2 --threads $samtools_cpus --write-index -o ${prefix}.bam##idx##${prefix}.bam.bai - + | samtools sort $args2 --threads $samtools_cpus --write-index -o ${prefix}.bam -O BAM - + samtools index ${prefix}.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + biscuit: \$( biscuit version |& sed '1!d; s/^.*BISCUIT Version: //' ) + samtools: \$( samtools --version |& sed '1!d; s/^.*samtools //' ) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.bam + touch ${prefix}.bam.bai cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/align/meta.yml b/modules/nf-core/biscuit/align/meta.yml index 645fc2c98a3..464e435bfa9 100644 --- a/modules/nf-core/biscuit/align/meta.yml +++ b/modules/nf-core/biscuit/align/meta.yml @@ -27,12 +27,24 @@ input: - reads: type: file description: | - List of input fastq files of size 1 and 2 for single-end and paired-end data, + List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - bam: - meta: @@ -63,3 +75,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/align/tests/main.nf.test b/modules/nf-core/biscuit/align/tests/main.nf.test index 1dc3eb3faca..84feb7a2249 100644 --- a/modules/nf-core/biscuit/align/tests/main.nf.test +++ b/modules/nf-core/biscuit/align/tests/main.nf.test @@ -12,16 +12,19 @@ nextflow_process { setup { run("BISCUIT_INDEX") { - script "../../index/main.nf" + script "../../../biscuit/index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("Single-End sarscov2 test_methylated_1 [fastq_gz]") { + test("sarscov2 single-end [fastq_gz]") { when { process { @@ -30,7 +33,11 @@ nextflow_process { [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz', checkIfExists: true) ] ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -38,14 +45,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert file(process.out.bam[0][1]).name == "test.bam" }, - { assert file(process.out.bai[0][1]).name == "test.bam.bai" }, - { assert snapshot(process.out.versions).match("single-end-versions") } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } ) } } - test("Paired-End sarscov2 test_methylated_1 [fastq_gz]") { + test("sarscov2 paired-end [fastq_gz]") { when { process { @@ -55,7 +65,11 @@ nextflow_process { [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz', checkIfExists: true) ] ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -63,9 +77,41 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert file(process.out.bam[0][1]).name == "test.bam" }, - { assert file(process.out.bai[0][1]).name == "test.bam.bai" }, - { assert snapshot(process.out.versions).match("paired-end-versions") } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } + ) + } + } + + test("sarscov2 paired-end [fastq_gz] - stub") { + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz', checkIfExists: true) ] + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() + } ) } } diff --git a/modules/nf-core/biscuit/align/tests/main.nf.test.snap b/modules/nf-core/biscuit/align/tests/main.nf.test.snap index 5776f87c62f..b6aeda38f7b 100644 --- a/modules/nf-core/biscuit/align/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/align/tests/main.nf.test.snap @@ -1,26 +1,93 @@ { - "single-end-versions": { + "sarscov2 single-end [fastq_gz]": { "content": [ + "499055250804bb3674c724bd9adf9d02", [ - "versions.yml:md5,0b2a8c4c438a785a890110a458399bfd" + [ + { + "id": "test" + }, + "test.bam.bai:md5,a748afc3cec6c27906ae80638e2024f5" + ] + ], + [ + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-22T20:06:02.295706212" + "timestamp": "2024-11-14T10:03:51.461022384" }, - "paired-end-versions": { + "sarscov2 paired-end [fastq_gz]": { "content": [ + "6b5afc62da5533472e1479950bb5f09a", + [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,ba60b802f583496e330c555bca9f0b97" + ] + ], [ - "versions.yml:md5,0b2a8c4c438a785a890110a458399bfd" + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T10:04:02.88266119" + }, + "sarscov2 paired-end [fastq_gz] - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" + ], + "bai": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,18c276eafa69dc43fa4d5b83b8392016" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-22T20:06:17.879689395" + "timestamp": "2024-11-13T06:48:19.558887499" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/align/tests/tags.yml b/modules/nf-core/biscuit/align/tests/tags.yml deleted file mode 100644 index d73b8183f7f..00000000000 --- a/modules/nf-core/biscuit/align/tests/tags.yml +++ /dev/null @@ -1,3 +0,0 @@ -biscuit/align: - - modules/nf-core/biscuit/index/** - - modules/nf-core/biscuit/align/** diff --git a/modules/nf-core/biscuit/biscuitblaster/environment.yml b/modules/nf-core/biscuit/biscuitblaster/environment.yml index ba50e31d11c..74a7b2fb908 100644 --- a/modules/nf-core/biscuit/biscuitblaster/environment.yml +++ b/modules/nf-core/biscuit/biscuitblaster/environment.yml @@ -2,6 +2,6 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 - bioconda::samblaster=0.1.26 - - bioconda::samtools=1.16.1 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/biscuitblaster/main.nf b/modules/nf-core/biscuit/biscuitblaster/main.nf index c4f7e1a52c5..f7876cb2fb5 100644 --- a/modules/nf-core/biscuit/biscuitblaster/main.nf +++ b/modules/nf-core/biscuit/biscuitblaster/main.nf @@ -4,12 +4,13 @@ process BISCUIT_BLASTER { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/22/22f141a92e12f98040054b9a47a51fb4de9c544174bbfa6d251a110f5e26fddc/data': + 'community.wave.seqera.io/library/biscuit_samblaster_samtools:a683d0887e7a91bf' }" input: tuple val(meta), path(reads) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: tuple val(meta), path("*.bam"), emit: bam @@ -24,23 +25,24 @@ process BISCUIT_BLASTER { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def args3 = task.ext.args3 ?: '' - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.95),1) - def samtools_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit align \\ -@ $biscuit_cpus \\ $args \\ - \$INDEX \\ - $reads | \\ - samblaster \\ - $args2 | \\ - samtools sort \\ - -@ $samtools_cpus \\ - $args3 \\ - --write-index \\ - -o ${prefix}.bam##idx##${prefix}.bam.bai + $index/$fasta\\ + $reads \\ + | samblaster $args2 \\ + | samtools sort \\ + -@ $samtools_cpus \\ + $args3 \\ + --write-index \\ + -o ${prefix}.bam -O BAM - + + samtools index ${prefix}.bam cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/biscuitblaster/meta.yml b/modules/nf-core/biscuit/biscuitblaster/meta.yml index d64cc1c2694..f2bed154053 100644 --- a/modules/nf-core/biscuit/biscuitblaster/meta.yml +++ b/modules/nf-core/biscuit/biscuitblaster/meta.yml @@ -1,6 +1,5 @@ name: biscuit_blaster -description: A fast, compact one-liner to produce duplicate-marked, sorted, and indexed - BAM files using Biscuit +description: A fast, compact one-liner to produce duplicate-marked, sorted, and indexed BAM files using Biscuit keywords: - biscuit - DNA methylation @@ -48,12 +47,24 @@ input: - reads: type: file description: | - List of input fastq files of size 1 and 2 for single-end and paired-end data, + List of input FastQ files of size 1 and 2 for single-end and paired-end data, respectively. - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - bam: - meta: @@ -84,3 +95,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test index ccea7f5b85e..9b7f7562ed8 100644 --- a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test +++ b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test @@ -14,21 +14,27 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.fasta", checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2 - single") { + test("sarscov2 single-end [fastq_gz]") { when { process { """ input[0] = [ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz", checkIfExists: true) ] - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -36,25 +42,30 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(file(process.out.bai[0][1]).name, - process.out.bam.collect { bam(it[1]).getHeaderMD5() }, - process.out.bam.collect { bam(it[1]).getReadsMD5() }, - process.out.versions).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } ) } } - test("sarscov2 - paired-end") { + test("sarscov2 paired-end [fastq_gz]") { when { process { """ input[0] = [ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz", checkIfExists: true), file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz", checkIfExists: true) ] - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -62,15 +73,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.bai, - process.out.bam.collect { bam(it[1]).getHeaderMD5() }, - process.out.bam.collect { bam(it[1]).getReadsMD5() }, - process.out.versions).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.bai, + process.out.versions + ).match() + } ) } } - test("sarscov2 - paired-end -stub") { + test("sarscov2 paired-end [fastq_gz] - stub") { options "-stub" when { process { @@ -78,9 +91,12 @@ nextflow_process { input[0] = [ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_1.fastq.gz", checkIfExists: true), file(params.modules_testdata_base_path + "genomics/sarscov2/illumina/fastq/test.methylated_2.fastq.gz", checkIfExists: true) ] - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap index e537de7d8ca..72f3eea613f 100644 --- a/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/biscuitblaster/tests/main.nf.test.snap @@ -1,50 +1,47 @@ { - "sarscov2 - single": { + "sarscov2 single-end [fastq_gz]": { "content": [ - "test.bam.bai", + "499055250804bb3674c724bd9adf9d02", [ - "a3b85097046b61879ab72293c38daea0" - ], - [ - "499055250804bb3674c724bd9adf9d02" + [ + { + "id": "test" + }, + "test.bam.bai:md5,56efb7d64af887266e8490a4afd6171f" + ] ], [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-10T11:20:35.533081588" + "timestamp": "2024-11-14T06:39:47.556767238" }, - "sarscov2 - paired-end": { + "sarscov2 paired-end [fastq_gz]": { "content": [ + "6b5afc62da5533472e1479950bb5f09a", [ [ { "id": "test" }, - "test.bam.bai:md5,bc235b43786886957bd5bb5b6623d4dd" + "test.bam.bai:md5,128302694afc1cf9e04b45c41860785f" ] ], [ - "d66c238598e02630bba7595baa2a6e71" - ], - [ - "6b5afc62da5533472e1479950bb5f09a" - ], - [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-10T10:57:59.954913657" + "timestamp": "2024-11-14T06:40:01.376002655" }, - "sarscov2 - paired-end -stub": { + "sarscov2 paired-end [fastq_gz] - stub": { "content": [ { "0": [ @@ -64,7 +61,7 @@ ] ], "2": [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ], "bai": [ [ @@ -83,14 +80,14 @@ ] ], "versions": [ - "versions.yml:md5,3f08154389b908fdfee5d4a328781e96" + "versions.yml:md5,92fe349c3b2cd12866273c14f89d79b9" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-10T10:58:19.26950429" + "timestamp": "2024-11-14T06:40:14.345664344" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/biscuitblaster/tests/tags.yml b/modules/nf-core/biscuit/biscuitblaster/tests/tags.yml deleted file mode 100644 index 13adde05efb..00000000000 --- a/modules/nf-core/biscuit/biscuitblaster/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -biscuit/biscuitblaster: - - "modules/nf-core/biscuit/biscuitblaster/**" diff --git a/modules/nf-core/biscuit/bsconv/environment.yml b/modules/nf-core/biscuit/bsconv/environment.yml index 56b7e967693..123215dc20f 100644 --- a/modules/nf-core/biscuit/bsconv/environment.yml +++ b/modules/nf-core/biscuit/bsconv/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 diff --git a/modules/nf-core/biscuit/bsconv/main.nf b/modules/nf-core/biscuit/bsconv/main.nf index 881fa0be813..5fb47439d1c 100644 --- a/modules/nf-core/biscuit/bsconv/main.nf +++ b/modules/nf-core/biscuit/bsconv/main.nf @@ -2,18 +2,19 @@ process BISCUIT_BSCONV { tag "$meta.id" label 'process_long' - conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/biscuit:1.1.0.20220707--he272189_1': - 'biocontainers/biscuit:1.1.0.20220707--he272189_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/33/33a9ca30b4154f11253c8d91a75382065dcb8282ba99b74dbee59ed8faceabd7/data': + 'community.wave.seqera.io/library/biscuit:1.5.0.20240506--ca92d9d0a37b5fa8' }" input: - tuple val(meta), path(bam), path(bai) - path(index) + tuple val(meta), path(bam) + tuple val(meta2), path(bai) + tuple val(meta3), path(fasta) + tuple val(meta4), path(index) output: - tuple val(meta), path("*.bam"), emit: bsconv_bam + tuple val(meta), path("*.bam"), emit: bam path "versions.yml" , emit: versions when: @@ -24,11 +25,11 @@ process BISCUIT_BSCONV { def prefix = task.ext.prefix ?: "${meta.id}" if ("$bam" == "${prefix}.bam") error "Input and output names are the same, set prefix in module configuration to disambiguate!" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit bsconv \\ $args \\ - \$INDEX \\ + $index/$fasta \\ $bam \\ ${prefix}.bam @@ -41,8 +42,6 @@ process BISCUIT_BSCONV { stub: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - if ("$bam" == "${prefix}.bam") error "Input and output names are the same, set prefix in module configuration to disambiguate!" - """ touch ${prefix}.bam diff --git a/modules/nf-core/biscuit/bsconv/meta.yml b/modules/nf-core/biscuit/bsconv/meta.yml index 75d9b2b7939..e162e175af0 100644 --- a/modules/nf-core/biscuit/bsconv/meta.yml +++ b/modules/nf-core/biscuit/bsconv/meta.yml @@ -27,15 +27,32 @@ input: - bam: type: file description: BAM file contained mapped reads + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - bai: type: file description: BAM file index - - - index: + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta4: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - - bsconv_bam: + - bam: - meta: type: map description: | @@ -54,3 +71,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/bsconv/tests/main.nf.test b/modules/nf-core/biscuit/bsconv/tests/main.nf.test index 1cbf306a88b..331c3df01f8 100644 --- a/modules/nf-core/biscuit/bsconv/tests/main.nf.test +++ b/modules/nf-core/biscuit/bsconv/tests/main.nf.test @@ -9,29 +9,36 @@ nextflow_process { tag "biscuit/bsconv" tag "biscuit/index" - setup { run("BISCUIT_INDEX") { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2-pe - bam") { + test("sarscov2 paired-end [bam]") { when { process { """ input[0] = [ [id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ [id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[2] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[3] = BISCUIT_INDEX.out.index """ } } @@ -39,14 +46,17 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.bsconv_bam).match("bsconv_bam") }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.versions + ).match() + } ) } } - test("sarscov2-pe - bam - stub") { + test("sarscov2 paired-end [bam] - stub") { options "-stub" @@ -54,11 +64,16 @@ nextflow_process { process { """ input[0] = [ [id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ [id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ] + input[2] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[3] = BISCUIT_INDEX.out.index """ } } @@ -66,7 +81,11 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + process.out.versions + ).match() + } ) } diff --git a/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap b/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap index 96dc77c097b..7796ebbb5ad 100644 --- a/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/bsconv/tests/main.nf.test.snap @@ -1,67 +1,28 @@ { - "versions": { + "sarscov2 paired-end [bam]": { "content": [ + "1608a01cabbbaba635e2905b7049f487", [ - "versions.yml:md5,45455cc9522a3235fa482ad065a242b3" + "versions.yml:md5,f8bba061ed5d28794abb4ffc2898aa02" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-07T15:28:04.601734997" + "timestamp": "2024-11-14T08:18:08.129627771" }, - "sarscov2-pe - bam - stub": { - "content": [ - { - "0": [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "1": [ - "versions.yml:md5,45455cc9522a3235fa482ad065a242b3" - ], - "bsconv_bam": [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "versions": [ - "versions.yml:md5,45455cc9522a3235fa482ad065a242b3" - ] - } - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" - }, - "timestamp": "2024-05-07T15:28:14.729945289" - }, - "bsconv_bam": { + "sarscov2 paired-end [bam] - stub": { "content": [ + "d41d8cd98f00b204e9800998ecf8427e", [ - [ - { - "id": "test", - "single_end": false - }, - "test.bam:md5,0d425851bd18963ebf1b6d727a7d02c7" - ] + "versions.yml:md5,f8bba061ed5d28794abb4ffc2898aa02" ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-05-07T15:28:04.580771449" + "timestamp": "2024-11-14T08:18:18.658974235" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/bsconv/tests/tags.yml b/modules/nf-core/biscuit/bsconv/tests/tags.yml deleted file mode 100644 index 5b890ac3597..00000000000 --- a/modules/nf-core/biscuit/bsconv/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -biscuit/bsconv: - - "modules/nf-core/biscuit/bsconv/**" diff --git a/modules/nf-core/biscuit/epiread/environment.yml b/modules/nf-core/biscuit/epiread/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/epiread/environment.yml +++ b/modules/nf-core/biscuit/epiread/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/epiread/main.nf b/modules/nf-core/biscuit/epiread/main.nf index 8eeb675c09c..110177c000c 100644 --- a/modules/nf-core/biscuit/epiread/main.nf +++ b/modules/nf-core/biscuit/epiread/main.nf @@ -4,15 +4,18 @@ process BISCUIT_EPIREAD { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: - tuple val(meta), path(bam), path(bai), path(snp_bed) - path(index) + tuple val(meta), path(bam) + tuple val(meta2), path(bai) + tuple val(meta3), path(snp_bed) + tuple val(meta4), path(fasta) + tuple val(meta5), path(index) output: - tuple val(meta), path("*.bed.gz"), emit: epiread_bed + tuple val(meta), path("*.bed.gz"), emit: bed path "versions.yml" , emit: versions when: @@ -22,8 +25,8 @@ process BISCUIT_EPIREAD { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1) - def samtools_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def samtools_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) // As of 2/25/22, epiread does not support reading a gzipped SNP BED file. // This is a bit hacky but allows the user to supply a gzipped OR uncompressed bed file def unzip_snp_bed = snp_bed && (snp_bed.toString() =~ /\.gz$/) ? "bgzip -d ${snp_bed}" : "" @@ -32,7 +35,7 @@ process BISCUIT_EPIREAD { def options_snp_bed = snp_bed ? "-B ${unzipped_snp_bed}" : "" if ("$options_snp_bed" == "${prefix}.bed.gz") error "Input and output names for biscuit epiread are the same, set prefix in module configuration to disambiguate!" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta $unzip_snp_bed @@ -40,11 +43,10 @@ process BISCUIT_EPIREAD { -@ $biscuit_cpus \\ $args \\ $options_snp_bed \\ - \$INDEX \\ - $bam | \\ - LC_ALL=C sort -k1,1 -k2,2n | \\ - bgzip \\ - -@ $samtools_cpus \\ + $index/$fasta \\ + $bam \\ + | LC_ALL=C sort -k1,1 -k2,2n \\ + | bgzip -@ $samtools_cpus \\ $args2 \\ -c > ${prefix}.bed.gz @@ -57,9 +59,6 @@ process BISCUIT_EPIREAD { stub: def prefix = task.ext.prefix ?: "${meta.id}" - def unzipped_snp_bed = snp_bed ? snp_bed.toString() - ~/\.gz$/: "" - def options_snp_bed = snp_bed ? "-B ${unzipped_snp_bed}" : "" - if ("$options_snp_bed" == "${prefix}.bed.gz") error "Input and output names for biscuit epiread are the same, set prefix in module configuration to disambiguate!" """ echo | gzip > ${prefix}.bed.gz diff --git a/modules/nf-core/biscuit/epiread/meta.yml b/modules/nf-core/biscuit/epiread/meta.yml index 8974f1d952a..b695be1cd8f 100644 --- a/modules/nf-core/biscuit/epiread/meta.yml +++ b/modules/nf-core/biscuit/epiread/meta.yml @@ -27,19 +27,41 @@ input: e.g. [ id:'test', single_end:false ] - bam: type: file - description: Biscuit BAM file + description: BAM file contained mapped reads + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - bai: type: file - description: BAM index + description: BAM file index + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - snp_bed: type: file description: BED file containing SNP information (optional) - - - index: + - - meta4: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta5: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - - epiread_bed: + - bed: - meta: type: map description: | @@ -58,3 +80,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/epiread/tests/main.nf.test b/modules/nf-core/biscuit/epiread/tests/main.nf.test index 5a151008afa..ceb1bff5b6f 100644 --- a/modules/nf-core/biscuit/epiread/tests/main.nf.test +++ b/modules/nf-core/biscuit/epiread/tests/main.nf.test @@ -15,24 +15,36 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2 - test_paired_end_methylated_sorted_bam") { + test("sarscov2 paired-end [bam]") { when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - [] //SNP BED file + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], [] //SNP BED file + ] + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } @@ -46,18 +58,28 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - snp") { + test("sarscov2 paired-end [bam] [snp]") { when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed') + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], + file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } @@ -71,18 +93,28 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz") { + test("sarscov2 paired-end [bam] [snp_gz]") { when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz') + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], + file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz', checkIfExists: true) + ] + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } @@ -96,19 +128,29 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz - stub") { + test("sarscov2 paired-end [bam] [snp_gz] - stub") { options '-stub' when { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true), - file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz') + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'test' ], + file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test-snp.bed.gz', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[3] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[4] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap b/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap index 1a64679d465..19a1e069912 100644 --- a/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/epiread/tests/main.nf.test.snap @@ -1,142 +1,134 @@ { - "sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz": { + "sarscov2 paired-end [bam] [snp_gz]": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:24.162952" + "timestamp": "2024-11-14T08:58:54.313858378" }, - "sarscov2 - test_paired_end_methylated_sorted_bam - snp": { + "sarscov2 paired-end [bam]": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,f75d8fe928fe2dc1a3ce6178d8963602" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,f2c701470965f9a93c45524e9e460509" + "test.bed.gz:md5,f75d8fe928fe2dc1a3ce6178d8963602" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:17.32731" + "timestamp": "2024-11-14T08:58:32.745357698" }, - "sarscov2 - test_paired_end_methylated_sorted_bam": { + "sarscov2 paired-end [bam] [snp]": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,3f60061464bd1c282bce0dc6478f8738" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, - "test.bed.gz:md5,3f60061464bd1c282bce0dc6478f8738" + "test.bed.gz:md5,709eeae76ae45a08983e8b45e183dad8" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:09.919651" + "timestamp": "2024-11-14T08:58:43.662473332" }, - "sarscov2 - test_paired_end_methylated_sorted_bam - snp - gz - stub": { + "sarscov2 paired-end [bam] [snp_gz] - stub": { "content": [ { "0": [ [ { - "id": "test", - "single_end": false + "id": "test" }, "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "1": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ], - "epiread_bed": [ + "bed": [ [ { - "id": "test", - "single_end": false + "id": "test" }, "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "versions": [ - "versions.yml:md5,aba0dae715c3208d621eff05bfff02d3" + "versions.yml:md5,e8b0e104428b24831e25ae3d6844aaae" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:55:29.932067" + "timestamp": "2024-11-14T08:59:04.687765786" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/index/environment.yml b/modules/nf-core/biscuit/index/environment.yml index 56b7e967693..123215dc20f 100644 --- a/modules/nf-core/biscuit/index/environment.yml +++ b/modules/nf-core/biscuit/index/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 diff --git a/modules/nf-core/biscuit/index/main.nf b/modules/nf-core/biscuit/index/main.nf index 43566fc4b50..fb5ab0d40c5 100644 --- a/modules/nf-core/biscuit/index/main.nf +++ b/modules/nf-core/biscuit/index/main.nf @@ -4,15 +4,15 @@ process BISCUIT_INDEX { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/biscuit:1.1.0.20220707--he272189_1': - 'biocontainers/biscuit:1.1.0.20220707--he272189_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/33/33a9ca30b4154f11253c8d91a75382065dcb8282ba99b74dbee59ed8faceabd7/data': + 'community.wave.seqera.io/library/biscuit:1.5.0.20240506--ca92d9d0a37b5fa8' }" input: - path fasta, stageAs: "BiscuitIndex/*" + tuple val(meta), path(fasta, name:"BiscuitIndex/") output: - path "BiscuitIndex/*.fa*", emit: index, includeInputs: true - path "versions.yml" , emit: versions + tuple val(meta), path("BiscuitIndex"), emit: index + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when @@ -34,7 +34,6 @@ process BISCUIT_INDEX { stub: def args = task.ext.args ?: '' """ - touch ${fasta}.bis.amb touch ${fasta}.bis.ann touch ${fasta}.bis.pac diff --git a/modules/nf-core/biscuit/index/meta.yml b/modules/nf-core/biscuit/index/meta.yml index 39a0bcf8d94..cdf51a8b4d6 100644 --- a/modules/nf-core/biscuit/index/meta.yml +++ b/modules/nf-core/biscuit/index/meta.yml @@ -19,15 +19,24 @@ tools: licence: ["MIT"] identifier: biotools:biscuit input: - - - fasta: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: type: file description: Input genome fasta file output: - index: - - BiscuitIndex/*.fa*: + - meta: type: directory - description: Biscuit genome index directory - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index + pattern: "index" + - BiscuitIndex: + type: directory + description: Directory containing biscuit genome index + pattern: "index" - versions: - versions.yml: type: file @@ -37,3 +46,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/index/tests/main.nf.test b/modules/nf-core/biscuit/index/tests/main.nf.test index 0402a262200..3c99b27bb08 100644 --- a/modules/nf-core/biscuit/index/tests/main.nf.test +++ b/modules/nf-core/biscuit/index/tests/main.nf.test @@ -1,4 +1,3 @@ - nextflow_process { name "Test Process BISCUIT_INDEX" @@ -10,13 +9,15 @@ nextflow_process { tag "biscuit" tag "biscuit/index" - test("test-biscuit-index") { + test("sarscov2 genome [fasta]") { when { process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } @@ -29,14 +30,16 @@ nextflow_process { } } - test("test-biscuit-index-stub") { + test("sarscov2 genome [fasta] - stub") { options '-stub' when { process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } diff --git a/modules/nf-core/biscuit/index/tests/main.nf.test.snap b/modules/nf-core/biscuit/index/tests/main.nf.test.snap index 698fc7e4bd8..715875012bd 100644 --- a/modules/nf-core/biscuit/index/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/index/tests/main.nf.test.snap @@ -1,84 +1,104 @@ { - "test-biscuit-index": { + "sarscov2 genome [fasta] - stub": { "content": [ { "0": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", - "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", - "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", - "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", - "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ] ], "1": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ], "index": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", - "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", - "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", - "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", - "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", - "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", - "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", + "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ] ], "versions": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-28T10:33:50.146795" + "timestamp": "2024-11-13T06:48:54.473235719" }, - "test-biscuit-index-stub": { + "sarscov2 genome [fasta]": { "content": [ { "0": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", + "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", + "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", + "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", + "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + ] ] ], "1": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ], "index": [ [ - "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", - "genome.fasta.bis.amb:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.ann:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.bis.pac:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.dau.sa:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.bwt:md5,d41d8cd98f00b204e9800998ecf8427e", - "genome.fasta.par.sa:md5,d41d8cd98f00b204e9800998ecf8427e" + { + "id": "test" + }, + [ + "genome.fasta:md5,6e9fe4042a72f2345f644f239272b7e6", + "genome.fasta.bis.amb:md5,3a68b8b2287e07dd3f5f95f4344ba76e", + "genome.fasta.bis.ann:md5,c32e11f6c859f166c7525a9c1d583567", + "genome.fasta.bis.pac:md5,983e3d2cd6f36e2546e6d25a0da78d66", + "genome.fasta.dau.bwt:md5,a11bc31775f7b7a4f9cd3bc4f981661a", + "genome.fasta.dau.sa:md5,9c9e07fa1c75ef32d764274579c89b08", + "genome.fasta.par.bwt:md5,62eb83cd557a47b59589713d98024fc2", + "genome.fasta.par.sa:md5,55bcd97d7059bf73dc0d221e36e8e901" + ] ] ], "versions": [ - "versions.yml:md5,067af0790b982d7524b24ba17e1dece7" + "versions.yml:md5,c451c038f5d41cd7237c5b0f07926ad6" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T10:49:18.069711" + "timestamp": "2024-11-13T06:48:44.596263127" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/mergecg/environment.yml b/modules/nf-core/biscuit/mergecg/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/mergecg/environment.yml +++ b/modules/nf-core/biscuit/mergecg/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/mergecg/main.nf b/modules/nf-core/biscuit/mergecg/main.nf index 93094b2a6cd..14efa0b7d81 100644 --- a/modules/nf-core/biscuit/mergecg/main.nf +++ b/modules/nf-core/biscuit/mergecg/main.nf @@ -4,16 +4,17 @@ process BISCUIT_MERGECG { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(bed) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: - tuple val(meta), path("*.bed.gz"), emit: mergecg_bed + tuple val(meta), path("*.bed.gz"), emit: bed path "versions.yml" , emit: versions when: @@ -24,16 +25,14 @@ process BISCUIT_MERGECG { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit mergecg \\ $args \\ - \$INDEX \\ - $bed | \\ - LC_ALL=C sort -k1,1 -k2,2n | \\ - bgzip \\ - $args2 \\ - -c > ${prefix}.bed.gz + $index/$fasta\\ + $bed \\ + | LC_ALL=C sort -k1,1 -k2,2n \\ + | bgzip $args2 -c > ${prefix}.bed.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/mergecg/meta.yml b/modules/nf-core/biscuit/mergecg/meta.yml index 2084f43b9d6..29b0889a4a1 100644 --- a/modules/nf-core/biscuit/mergecg/meta.yml +++ b/modules/nf-core/biscuit/mergecg/meta.yml @@ -27,12 +27,24 @@ input: type: file description: | Biscuit BED file (output of biscuit vcf2bed) - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - - mergecg_bed: + - bed: - meta: type: map description: | @@ -51,3 +63,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/mergecg/tests/main.nf.test b/modules/nf-core/biscuit/mergecg/tests/main.nf.test index 0b593f76842..cfa29b719d9 100644 --- a/modules/nf-core/biscuit/mergecg/tests/main.nf.test +++ b/modules/nf-core/biscuit/mergecg/tests/main.nf.test @@ -9,27 +9,34 @@ nextflow_process { tag "biscuit/index" tag "biscuit/mergecg" - test("sars_cov_bed") { - - setup { + setup { run("BISCUIT_INDEX") { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.fasta", checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } + + test("sarscov2 cov [bed]") { + when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + "delete_me/biscuit/test-cg.bed.gz", checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -43,27 +50,21 @@ nextflow_process { } - test("sars_cov_bed -stub") { + test("sarscov2 cov [bed] - stub") { options "-stub" - setup { - run("BISCUIT_INDEX") { - script "../../index/main.nf" - process { - """ - input[0] = file(params.modules_testdata_base_path + "genomics/sarscov2/genome/genome.fasta", checkIfExists: true) - """ - } - } - } + when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + "delete_me/biscuit/test-cg.bed.gz", checkIfExists: true) - ] - - input[1] = BISCUIT_INDEX.out.index + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap b/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap index 148a1f947bd..f003dee7b8d 100644 --- a/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/mergecg/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sars_cov_bed": { + "sarscov2 cov [bed] - stub": { "content": [ { "0": [ @@ -8,33 +8,33 @@ "id": "test", "single_end": false }, - "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" + "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "1": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ], - "mergecg_bed": [ + "bed": [ [ { "id": "test", "single_end": false }, - "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" + "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], "versions": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-11T17:31:48.698330905" + "timestamp": "2024-11-14T09:09:45.257367761" }, - "sars_cov_bed -stub": { + "sarscov2 cov [bed]": { "content": [ { "0": [ @@ -43,30 +43,30 @@ "id": "test", "single_end": false }, - "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" ] ], "1": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ], - "mergecg_bed": [ + "bed": [ [ { "id": "test", "single_end": false }, - "test.bed.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + "test.bed.gz:md5,8136632b0c17c61857a302a57ae302ef" ] ], "versions": [ - "versions.yml:md5,caa3e8096399d35c725d7e2d430605c9" + "versions.yml:md5,4b2f3877d68071eafa0b0ec3e37c58a3" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-06-12T09:59:25.605969084" + "timestamp": "2024-11-14T09:09:30.544664794" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/mergecg/tests/tags.yml b/modules/nf-core/biscuit/mergecg/tests/tags.yml deleted file mode 100644 index d7140b07550..00000000000 --- a/modules/nf-core/biscuit/mergecg/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -biscuit/mergecg: - - "modules/nf-core/biscuit/mergecg/**" diff --git a/modules/nf-core/biscuit/pileup/environment.yml b/modules/nf-core/biscuit/pileup/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/pileup/environment.yml +++ b/modules/nf-core/biscuit/pileup/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/pileup/main.nf b/modules/nf-core/biscuit/pileup/main.nf index dbbd6ffa2d5..db51aae2c09 100644 --- a/modules/nf-core/biscuit/pileup/main.nf +++ b/modules/nf-core/biscuit/pileup/main.nf @@ -4,12 +4,13 @@ process BISCUIT_PILEUP { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(normal_bams), path(normal_bais), path(tumor_bam), path(tumor_bai) - path index + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: tuple val(meta), path("*.vcf.gz"), emit: vcf @@ -22,18 +23,18 @@ process BISCUIT_PILEUP { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def biscuit_cpus = (int) Math.max(Math.floor(task.cpus*0.9),1) - def bgzip_cpus = task.cpus-biscuit_cpus + def biscuit_cpus = [(task.cpus * 0.9) as int, 1].max() + def bgzip_cpus = (task.cpus - biscuit_cpus < 1) ? biscuit_cpus : (task.cpus - biscuit_cpus) if ( tumor_bam != [] && normal_bams.toList().size() > 1 ) error "[BISCUIT_PILEUP] error: Tumor BAM provided with more than one normal BAM" if ( tumor_bam.toList().size() > 1 ) error "[BISCUIT_PILEUP] error: more than one tumor BAM provided" input = ( tumor_bam==[] ) ? "${normal_bams}" : "-S -T ${tumor_bam} -I ${normal_bams}" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit pileup \\ -@ $biscuit_cpus \\ $args \\ - \$INDEX \\ + $index/$fasta \\ $input \\ | bgzip -@ $bgzip_cpus $args2 > ${prefix}.vcf.gz diff --git a/modules/nf-core/biscuit/pileup/meta.yml b/modules/nf-core/biscuit/pileup/meta.yml index 9f532797e43..0ab31cc549b 100644 --- a/modules/nf-core/biscuit/pileup/meta.yml +++ b/modules/nf-core/biscuit/pileup/meta.yml @@ -47,10 +47,22 @@ input: type: file description: Optional. BAM index file corresponding to provided tumor_bam pattern: "*.{bai}" - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: + type: file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: type: directory - description: Biscuit genome index directory (generated with 'biscuit index') - pattern: "BiscuitIndex" + description: Directory containing biscuit genome index output: - vcf: - meta: @@ -71,3 +83,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/pileup/tests/main.nf.test b/modules/nf-core/biscuit/pileup/tests/main.nf.test index b188103a06f..74ebff4d271 100644 --- a/modules/nf-core/biscuit/pileup/tests/main.nf.test +++ b/modules/nf-core/biscuit/pileup/tests/main.nf.test @@ -15,18 +15,21 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("homo_sapiens - test_paired_end_sorted_bam") { + test("homo_sapiens - paired-end [bam]") { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test' ], // meta map [ file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), @@ -38,9 +41,12 @@ nextflow_process { ], [], // tumor bam [] // tumor bai - ] - - input[1] = BISCUIT_INDEX.out.index + ]) + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -58,7 +64,7 @@ nextflow_process { } - test("homo_sapiens - test_paired_end_sorted_bam - somatic") { + test("homo_sapiens - paired-end [bam] - somatic") { when { process { @@ -70,8 +76,11 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai', checkIfExists: true) ] - - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -89,7 +98,7 @@ nextflow_process { } - test("homo_sapiens - test_paired_end_sorted_bam - somatic - stub") { + test("homo_sapiens - paired-end [bam] - somatic - stub") { options '-stub' when { @@ -102,8 +111,11 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam', checkIfExists: true), file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.sorted.bam.bai', checkIfExists: true) ] - - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap b/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap index ca99df65997..14c40aa1a4d 100644 --- a/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/pileup/tests/main.nf.test.snap @@ -1,31 +1,5 @@ { - "homo_sapiens - test_paired_end_sorted_bam - somatic": { - "content": [ - "12357b78693a704403196644b34d69a6", - [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" - }, - "timestamp": "2024-08-29T11:43:13.955407" - }, - "homo_sapiens - test_paired_end_sorted_bam": { - "content": [ - "4ae9e1f5d3dbe07aca279680840bbe20", - [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" - }, - "timestamp": "2024-08-29T11:41:00.666921" - }, - "homo_sapiens - test_paired_end_sorted_bam - somatic - stub": { + "homo_sapiens - paired-end [bam] - somatic - stub": { "content": [ { "0": [ @@ -37,7 +11,7 @@ ] ], "1": [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" ], "vcf": [ [ @@ -48,14 +22,40 @@ ] ], "versions": [ - "versions.yml:md5,d73c753b5bbc7439f06fba6c1794f719" + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T09:24:21.352967241" + }, + "homo_sapiens - paired-end [bam] - somatic": { + "content": [ + "5e1157703410c1a6b7cc7eded1436425", + [ + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T09:27:58.885274336" + }, + "homo_sapiens - paired-end [bam]": { + "content": [ + "6bea182d0b89d10246cc0ac711f6507", + [ + "versions.yml:md5,beff27f24fc9327d67f5968cf1cef6cd" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:43:19.616012" + "timestamp": "2024-11-14T09:27:47.811858126" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/qc/environment.yml b/modules/nf-core/biscuit/qc/environment.yml index 56b7e967693..123215dc20f 100644 --- a/modules/nf-core/biscuit/qc/environment.yml +++ b/modules/nf-core/biscuit/qc/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 + - bioconda::biscuit=1.5.0.20240506 diff --git a/modules/nf-core/biscuit/qc/main.nf b/modules/nf-core/biscuit/qc/main.nf index 0ac7bd9eeab..4e63292ebd9 100644 --- a/modules/nf-core/biscuit/qc/main.nf +++ b/modules/nf-core/biscuit/qc/main.nf @@ -4,15 +4,16 @@ process BISCUIT_QC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/biscuit:1.1.0.20220707--he272189_1': - 'biocontainers/biscuit:1.1.0.20220707--he272189_1' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/33/33a9ca30b4154f11253c8d91a75382065dcb8282ba99b74dbee59ed8faceabd7/data': + 'community.wave.seqera.io/library/biscuit:1.5.0.20240506--ca92d9d0a37b5fa8' }" input: tuple val(meta), path(bam) - path(index) + tuple val(meta2), path(fasta) + tuple val(meta3), path(index) output: - tuple val(meta), path("*.txt"), emit: biscuit_qc_reports + tuple val(meta), path("*.txt"), emit: reports path "versions.yml" , emit: versions when: @@ -23,12 +24,12 @@ process BISCUIT_QC { def prefix = task.ext.prefix ?: "${meta.id}" def se = meta.single_end ? "-s" : "" """ - INDEX=`find -L ./ -name "*.bis.amb" | sed 's/\\.bis.amb\$//'` + ln -sf \$(readlink $fasta) $index/$fasta biscuit qc \\ $args \\ $se \\ - \$INDEX \\ + $index/$fasta \\ $bam \\ $prefix diff --git a/modules/nf-core/biscuit/qc/meta.yml b/modules/nf-core/biscuit/qc/meta.yml index ab7500ba823..379f46bbf3e 100644 --- a/modules/nf-core/biscuit/qc/meta.yml +++ b/modules/nf-core/biscuit/qc/meta.yml @@ -27,11 +27,24 @@ input: - bam: type: file description: BAM file produced using Biscuit - - - index: + - - meta2: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fasta: type: file - description: index file + description: Input genome fasta file + - - meta3: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - index: + type: directory + description: Directory containing biscuit genome index output: - - biscuit_qc_reports: + - reports: - meta: type: file description: | @@ -65,3 +78,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/qc/tests/main.nf.test b/modules/nf-core/biscuit/qc/tests/main.nf.test index d4b4e57ad47..24d0dc23729 100644 --- a/modules/nf-core/biscuit/qc/tests/main.nf.test +++ b/modules/nf-core/biscuit/qc/tests/main.nf.test @@ -15,13 +15,16 @@ nextflow_process { script "../../index/main.nf" process { """ - input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + input[0] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) """ } } } - test("sarscov2 - test_paired_end_methylated_sorted_bam") { + test("sarscov2 paired-end [bam]") { when { process { @@ -30,7 +33,11 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } @@ -44,7 +51,7 @@ nextflow_process { } - test("sarscov2 - test_paired_end_methylated_sorted_bam - stub") { + test("sarscov2 paired-end [bam] - stub") { options '-stub' when { @@ -54,7 +61,11 @@ nextflow_process { [ id:'test', single_end:false ], // meta map file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.methylated.sorted.bam', checkIfExists: true) ] - input[1] = BISCUIT_INDEX.out.index + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BISCUIT_INDEX.out.index """ } } diff --git a/modules/nf-core/biscuit/qc/tests/main.nf.test.snap b/modules/nf-core/biscuit/qc/tests/main.nf.test.snap index 51c995d9e5b..4c5deb3fd8d 100644 --- a/modules/nf-core/biscuit/qc/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/qc/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "sarscov2 - test_paired_end_methylated_sorted_bam": { + "sarscov2 paired-end [bam]": { "content": [ { "0": [ @@ -20,9 +20,9 @@ ] ], "1": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ], - "biscuit_qc_reports": [ + "reports": [ [ { "id": "test", @@ -40,17 +40,17 @@ ] ], "versions": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:27:13.208945" + "timestamp": "2024-11-14T09:49:31.146030396" }, - "sarscov2 - test_paired_end_methylated_sorted_bam - stub": { + "sarscov2 paired-end [bam] - stub": { "content": [ { "0": [ @@ -71,9 +71,9 @@ ] ], "1": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ], - "biscuit_qc_reports": [ + "reports": [ [ { "id": "test", @@ -91,14 +91,14 @@ ] ], "versions": [ - "versions.yml:md5,c9bca700bf29bd9d02fc254fe070e279" + "versions.yml:md5,15148723e4f43101b9e07a02566dbbd4" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:30:49.37928" + "timestamp": "2024-11-14T09:49:41.278929227" } } \ No newline at end of file diff --git a/modules/nf-core/biscuit/vcf2bed/environment.yml b/modules/nf-core/biscuit/vcf2bed/environment.yml index b4bab59944c..222c156ed9a 100644 --- a/modules/nf-core/biscuit/vcf2bed/environment.yml +++ b/modules/nf-core/biscuit/vcf2bed/environment.yml @@ -2,5 +2,5 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::biscuit=1.1.0.20220707 - - bioconda::samtools=1.16.1 + - bioconda::biscuit=1.5.0.20240506 + - bioconda::samtools=1.21 diff --git a/modules/nf-core/biscuit/vcf2bed/main.nf b/modules/nf-core/biscuit/vcf2bed/main.nf index f2ba512ef63..d53c8ee342b 100644 --- a/modules/nf-core/biscuit/vcf2bed/main.nf +++ b/modules/nf-core/biscuit/vcf2bed/main.nf @@ -4,8 +4,8 @@ process BISCUIT_VCF2BED { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0': - 'biocontainers/mulled-v2-d94f582b04a3edcede1215189c0d881506640fd9:6519548ea4f3d6a526c78ad0350c58f867f28574-0' }" + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/5b/5b542bbe1f99afd494ef07423ea8b52f2b8a081b85f92db2726c283c78da3cf0/data': + 'community.wave.seqera.io/library/biscuit_samtools:84373c8a97fa63b8' }" input: tuple val(meta), path(vcf) @@ -24,11 +24,9 @@ process BISCUIT_VCF2BED { """ biscuit vcf2bed \\ $args \\ - $vcf | \\ - LC_ALL=C sort -k1,1 -k2,2n | \\ - bgzip \\ - $args2 \\ - -c > ${prefix}.bed.gz + $vcf \\ + | LC_ALL=C sort -k1,1 -k2,2n \\ + | bgzip $args2 -c > ${prefix}.bed.gz cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/biscuit/vcf2bed/meta.yml b/modules/nf-core/biscuit/vcf2bed/meta.yml index 76ee1cad11c..b50fa56d44a 100644 --- a/modules/nf-core/biscuit/vcf2bed/meta.yml +++ b/modules/nf-core/biscuit/vcf2bed/meta.yml @@ -48,3 +48,4 @@ authors: - "@njspix" maintainers: - "@njspix" + - "@sateeshperi" diff --git a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test index 527f0c0ff4f..b9097a07e02 100644 --- a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test +++ b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test @@ -10,16 +10,15 @@ nextflow_process { tag "biscuit" tag "biscuit/vcf2bed" - test("test-biscuit-vcf2bed") { + test("test [vcf]") { when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test.vcf.gz', checkIfExists: true) - ] - + ]) """ } } @@ -32,17 +31,16 @@ nextflow_process { } } - test("test-biscuit-vcf2bed-stub") { + test("test [vcf] - stub") { options '-stub' when { process { """ - input[0] = [ + input[0] = Channel.of([ [ id:'test', single_end:false ], // meta map file('https://github.com/nf-core/test-datasets/raw/modules/data/delete_me/biscuit/test.vcf.gz', checkIfExists: true) - ] - + ]) """ } } diff --git a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap index fd232298b85..816bd1cceaa 100644 --- a/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap +++ b/modules/nf-core/biscuit/vcf2bed/tests/main.nf.test.snap @@ -1,5 +1,5 @@ { - "test-biscuit-vcf2bed": { + "test [vcf]": { "content": [ { "0": [ @@ -12,7 +12,7 @@ ] ], "1": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ], "bed": [ [ @@ -24,17 +24,17 @@ ] ], "versions": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:02:38.571195" + "timestamp": "2024-11-14T09:55:25.366750014" }, - "test-biscuit-vcf2bed-stub": { + "test [vcf] - stub": { "content": [ { "0": [ @@ -47,7 +47,7 @@ ] ], "1": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ], "bed": [ [ @@ -59,14 +59,14 @@ ] ], "versions": [ - "versions.yml:md5,8626f51de333380e99cc607aaebb2cce" + "versions.yml:md5,54a906013b2703ac5c5250af2a996259" ] } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-29T11:02:43.49219" + "timestamp": "2024-11-14T09:55:34.060320551" } } \ No newline at end of file From 3ef36024bb95d306c63c7bf9014132e62c7b4755 Mon Sep 17 00:00:00 2001 From: Evangelos Karatzas <32259775+vagkaratzas@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:03:39 +0200 Subject: [PATCH 03/12] clipkit - new module (#6993) * clipkit - new module * standard prefix.clipkit output + better meta description for the alignment file --- modules/nf-core/clipkit/environment.yml | 7 ++ modules/nf-core/clipkit/main.nf | 46 +++++++++++++ modules/nf-core/clipkit/meta.yml | 50 ++++++++++++++ modules/nf-core/clipkit/tests/main.nf.test | 66 ++++++++++++++++++ .../nf-core/clipkit/tests/main.nf.test.snap | 68 +++++++++++++++++++ 5 files changed, 237 insertions(+) create mode 100644 modules/nf-core/clipkit/environment.yml create mode 100644 modules/nf-core/clipkit/main.nf create mode 100644 modules/nf-core/clipkit/meta.yml create mode 100644 modules/nf-core/clipkit/tests/main.nf.test create mode 100644 modules/nf-core/clipkit/tests/main.nf.test.snap diff --git a/modules/nf-core/clipkit/environment.yml b/modules/nf-core/clipkit/environment.yml new file mode 100644 index 00000000000..65c451ffb1f --- /dev/null +++ b/modules/nf-core/clipkit/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::clipkit=2.3.0" diff --git a/modules/nf-core/clipkit/main.nf b/modules/nf-core/clipkit/main.nf new file mode 100644 index 00000000000..2a9ebe3f851 --- /dev/null +++ b/modules/nf-core/clipkit/main.nf @@ -0,0 +1,46 @@ +process CLIPKIT { + tag "$meta.id" + label 'process_single' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/clipkit:2.3.0--pyhdfd78af_0': + 'biocontainers/clipkit:2.3.0--pyhdfd78af_0' }" + + input: + tuple val(meta), path(aln) + + output: + tuple val(meta), path("*.clipkit"), emit: clipkit + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + clipkit \\ + $args \\ + $aln \\ + -o ${prefix}.clipkit + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + clipkit: \$(clipkit --version |& sed '1!d ; s/clipkit //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.clipkit + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + clipkit: \$(clipkit --version |& sed '1!d ; s/clipkit //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/clipkit/meta.yml b/modules/nf-core/clipkit/meta.yml new file mode 100644 index 00000000000..5347ad5923a --- /dev/null +++ b/modules/nf-core/clipkit/meta.yml @@ -0,0 +1,50 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "clipkit" +description: ClipKIT is a fast and flexible alignment trimming tool that keeps phylogenetically informative sites and removes those that display characteristics poor phylogenetic signal. +keywords: + - sort + - example + - genomics +tools: + - "clipkit": + description: "Alignment trimming software for phylogenetics." + homepage: "https://jlsteenwyk.com/ClipKIT/" + documentation: "https://jlsteenwyk.com/ClipKIT/" + tool_dev_url: "https://github.com/JLSteenwyk/ClipKIT" + doi: "10.1371/journal.pbio.3001007" + licence: ["MIT"] + identifier: "" + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - aln: + type: file + description: Multiple sequence alignment file in various supported formats. + pattern: "*.{fa,fasta,fa,fna,faa,alnfaa,aln,sto,stk,mauve,alignment,clustal}" + +output: + - clipkit: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.clipkit": + type: file + description: Trimmed multiple sequence alignment file + pattern: "*.clipkit" + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@vagkaratzas" +maintainers: + - "@vagkaratzas" diff --git a/modules/nf-core/clipkit/tests/main.nf.test b/modules/nf-core/clipkit/tests/main.nf.test new file mode 100644 index 00000000000..46cc8a00716 --- /dev/null +++ b/modules/nf-core/clipkit/tests/main.nf.test @@ -0,0 +1,66 @@ +nextflow_process { + + name "Test Process CLIPKIT" + script "../main.nf" + process "CLIPKIT" + + tag "modules" + tag "modules_nfcore" + tag "clipkit" + tag "gunzip" + + setup { + run("GUNZIP") { + script "../../gunzip/main.nf" + process { + """ + input[0] = [ + [ id:'PF14720' ], // meta map + file(params.modules_testdata_base_path + 'delete_me/hmmer/PF14720_seed.alnfaa.gz', checkIfExists: true) + ] + """ + } + } + } + + test("PF14720 - aln") { + + when { + process { + """ + input[0] = GUNZIP.out.gunzip + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("PF14720 - aln - stub") { + + options "-stub" + + when { + process { + """ + input[0] = GUNZIP.out.gunzip + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/clipkit/tests/main.nf.test.snap b/modules/nf-core/clipkit/tests/main.nf.test.snap new file mode 100644 index 00000000000..55f837d9e52 --- /dev/null +++ b/modules/nf-core/clipkit/tests/main.nf.test.snap @@ -0,0 +1,68 @@ +{ + "PF14720 - aln - stub": { + "content": [ + { + "0": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ], + "clipkit": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-14T13:11:21.109400968" + }, + "PF14720 - aln": { + "content": [ + { + "0": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,97e4328b5990f5e5a6241c225ed6d922" + ] + ], + "1": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ], + "clipkit": [ + [ + { + "id": "PF14720" + }, + "PF14720.clipkit:md5,97e4328b5990f5e5a6241c225ed6d922" + ] + ], + "versions": [ + "versions.yml:md5,1eaf2dc589bc08c1e37cbbddbedfcc80" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "23.04.5" + }, + "timestamp": "2024-11-14T13:11:14.278212183" + } +} \ No newline at end of file From 0e2d0c6d94f048eb1ba88924a0c87edc2c1bd6ef Mon Sep 17 00:00:00 2001 From: Edmund Miller <20095261+edmundmiller@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:25:15 -0600 Subject: [PATCH 04/12] Wave with Dockerfiles (#4940) * ci(renovate): Add Dockerfile building * build: Bump quarto to 24.04 * ci: Get clever with singularity flag * build: Try a better Dockerfile * ci(wave): Better names * ci(wave): Properly skip environment.yml and Dockerfiles * style(wittyer): Remove environment.yml * ci(wave): Let Dockerfiles fail and request @edmundmiller and @maxulysse --- .github/CODEOWNERS | 1 + .github/workflows/wave.yml | 101 ++++++++++++++-------- modules/nf-core/quartonotebook/Dockerfile | 2 +- modules/nf-core/wittyer/Dockerfile | 2 + modules/nf-core/wittyer/environment.yml | 5 -- 5 files changed, 71 insertions(+), 40 deletions(-) delete mode 100644 modules/nf-core/wittyer/environment.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3893bf75737..9552f7c99ad 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,3 +12,4 @@ modules/nf-core/pints/caller/** @edmundmiller modules/nf-core/preseq/ccurve/** @edmundmiller modules/nf-core/preseq/lcextrap/** @edmundmiller subworkflows/nf-core/homer/** @edmundmiller +modules/nf-core/**/Dockerfile @edmundmiller @maxulysse diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index 4f36575b6d1..a86fd9bb10a 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - "**/environment.yml" + - "**/Dockerfile" # TODO On complete call testing CI # TODO Skip testing CI if any changes to environment.yml @@ -18,38 +19,44 @@ jobs: runs-on: ubuntu-latest # Only run on Pull Requests within the same repository, and not from forks if: github.event.pull_request.head.repo.full_name == github.repository - + outputs: + conda-matrix: ${{ steps.conda-diff.outputs.all_changed_files }} + docker-matrix: ${{ steps.docker-diff.outputs.all_changed_files }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Calculate file differences - id: diff + - name: Find conda differences + id: conda-diff uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45 with: json: true - quotepath: false - # TODO Add Dockerfiles + escape_json: false files: | modules/**/environment.yml - - name: Debug - run: echo ${{ steps.diff.outputs.all_changed_files }} - - id: set-matrix - run: echo "matrix={\"profile\":[\"docker\", \"singularity\"],\"files\":${{ steps.diff.outputs.all_changed_files }} }" >> "$GITHUB_OUTPUT" - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + - name: Find Dockerfile differences + id: docker-diff + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45 + with: + json: true + escape_json: false + files: | + modules/**/Dockerfile - build: + conda-wave: # NOTE This should get skipped because generate-matrix won't run # if: github.repository == 'nf-core/modules' - if: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}" + if: ${{ needs.generate-matrix.outputs.conda-matrix != '[]' }} needs: generate-matrix - name: build + name: Build Conda-based ${{ matrix.profile }} Container runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: "${{ fromJson(needs.generate-matrix.outputs.matrix) }}" + max-parallel: 4 + matrix: + files: "${{ fromJson(needs.generate-matrix.outputs.conda-matrix) }}" + profile: [docker, singularity] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -59,38 +66,64 @@ jobs: sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave chmod +x /usr/local/bin/wave - - name: Create a registry name - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: registry-name - with: - result-encoding: string - script: | - return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/environment.yml', '').replace('/', '_'); - - - name: Build container + - name: Build ${{ matrix.profile }} container # FIXME Hack while iron out the CI continue-on-error: true - if: matrix.profile == 'docker' + env: + PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} run: | wave --conda-file "${{ matrix.files }}" \ + $PROFILE \ --freeze \ --await \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} - - name: Build Singularity - # FIXME Hack while iron out the CI - continue-on-error: true - if: matrix.profile == 'singularity' + docker-wave: + # NOTE This should get skipped because generate-matrix won't run + # if: github.repository == 'nf-core/modules' + if: ${{ needs.generate-matrix.outputs.docker-matrix != '[]' }} + needs: generate-matrix + name: Build Dockerfile-based ${{ matrix.profile }} Container + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + files: "${{ fromJson(needs.generate-matrix.outputs.docker-matrix) }}" + profile: [docker, singularity] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Install wave-cli run: | - wave --conda-file "${{ matrix.files }}" \ + wget -q https://github.com/seqeralabs/wave-cli/releases/download/v${WAVE_VER}/wave-${WAVE_VER}-linux-x86_64 + sudo mv wave-${WAVE_VER}-linux-x86_64 /usr/local/bin/wave + chmod +x /usr/local/bin/wave + + - name: Create a registry name + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: registry-name + with: + result-encoding: string + script: | + return '${{ matrix.files }}'.replace('modules/nf-core/', '').replace('/Dockerfile', '').replace('/', '_'); + + - name: Build ${{ matrix.profile }} container + # NOTE If you're changing a Dockerfile and this is running, try to update the Dockerfile to build with wave + continue-on-error: false + env: + PROFILE: ${{ (contains(matrix.profile, 'singularity') && '--singularity') || '' }} + run: | + wave -f "${{ matrix.files }}" \ + $PROFILE \ --freeze \ --await \ + --build-repo quay.io/nf-core/modules/${{steps.registry-name.outputs.result}} \ + --cache-repository quay.io/nf-core/wave-cache \ --tower-token ${{ secrets.TOWER_ACCESS_TOKEN }} \ - --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} \ - --singularity - - # TODO Build from Dockerfiles + --tower-workspace-id ${{ secrets.TOWER_WORKSPACE_ID }} # bump-versions: # needs: generate-matrix diff --git a/modules/nf-core/quartonotebook/Dockerfile b/modules/nf-core/quartonotebook/Dockerfile index 78d2ab20c23..26624b7ea6d 100644 --- a/modules/nf-core/quartonotebook/Dockerfile +++ b/modules/nf-core/quartonotebook/Dockerfile @@ -1,7 +1,7 @@ # # First stage: Quarto installation # -FROM ubuntu:20.04 as quarto +FROM ubuntu:24.04 as quarto ARG QUARTO_VERSION=1.3.433 ARG TARGETARCH RUN apt-get update \ diff --git a/modules/nf-core/wittyer/Dockerfile b/modules/nf-core/wittyer/Dockerfile index fae697ab9dc..fbf732f32bd 100644 --- a/modules/nf-core/wittyer/Dockerfile +++ b/modules/nf-core/wittyer/Dockerfile @@ -1,4 +1,5 @@ FROM mcr.microsoft.com/dotnet/sdk:6.0@sha256:6df1177e48b55272316d08f19cb383483af82aca5cdc67a76c414bc200847624 as builder + WORKDIR /src COPY . /src RUN cd Wittyer \ @@ -6,6 +7,7 @@ RUN cd Wittyer \ && chmod +x /output/Wittyer FROM mcr.microsoft.com/dotnet/runtime:6.0@sha256:437cda84bdce26ce074d88b63abeec567c7226d73e8b911605077468e1d5c8d5 + LABEL git_repository=https://git.illumina.com/DASTE/Ilmn.Das.App.Wittyer.git WORKDIR /opt/Wittyer RUN apt-get -y update && apt-get -y install tabix libunwind8 openssl procps diff --git a/modules/nf-core/wittyer/environment.yml b/modules/nf-core/wittyer/environment.yml deleted file mode 100644 index c90cb8dccbf..00000000000 --- a/modules/nf-core/wittyer/environment.yml +++ /dev/null @@ -1,5 +0,0 @@ -channels: - - conda-forge - - bioconda -dependencies: - - tabix From 754825005d1c009e0221aca276aeabf1972ca31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= <86408271+kbestak@users.noreply.github.com> Date: Thu, 14 Nov 2024 23:54:32 +0100 Subject: [PATCH 05/12] nf-test for ashlar (#6931) * nf-test for ashlar * Update modules/nf-core/ashlar/main.nf Co-authored-by: Simon Pearce <24893913+SPPearce@users.noreply.github.com> * added multiple tests * pin tifffile version --------- Co-authored-by: Simon Pearce <24893913+SPPearce@users.noreply.github.com> --- modules/nf-core/ashlar/environment.yml | 1 + modules/nf-core/ashlar/main.nf | 20 +++ modules/nf-core/ashlar/tests/main.nf.test | 155 ++++++++++++++++ .../nf-core/ashlar/tests/main.nf.test.snap | 167 ++++++++++++++++++ modules/nf-core/ashlar/tests/nextflow.config | 5 + tests/config/pytest_modules.yml | 3 - tests/modules/nf-core/ashlar/main.nf | 90 ---------- tests/modules/nf-core/ashlar/nextflow.config | 9 - tests/modules/nf-core/ashlar/test.yml | 44 ----- 9 files changed, 348 insertions(+), 146 deletions(-) create mode 100644 modules/nf-core/ashlar/tests/main.nf.test create mode 100644 modules/nf-core/ashlar/tests/main.nf.test.snap create mode 100644 modules/nf-core/ashlar/tests/nextflow.config delete mode 100644 tests/modules/nf-core/ashlar/main.nf delete mode 100644 tests/modules/nf-core/ashlar/nextflow.config delete mode 100644 tests/modules/nf-core/ashlar/test.yml diff --git a/modules/nf-core/ashlar/environment.yml b/modules/nf-core/ashlar/environment.yml index 8b3e771ac01..502f7177299 100644 --- a/modules/nf-core/ashlar/environment.yml +++ b/modules/nf-core/ashlar/environment.yml @@ -3,3 +3,4 @@ channels: - bioconda dependencies: - bioconda::ashlar=1.18.0 + - tifffile=2024.1.30 diff --git a/modules/nf-core/ashlar/main.nf b/modules/nf-core/ashlar/main.nf index 2ddc710feb5..86120b8de58 100644 --- a/modules/nf-core/ashlar/main.nf +++ b/modules/nf-core/ashlar/main.nf @@ -49,4 +49,24 @@ process ASHLAR { ashlar: \$(ashlar --version | sed 's/^.*ashlar //' ) END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + def num_files = images instanceof List ? images.size() : 1 + def opt_dfp_size = opt_dfp instanceof List ? opt_dfp.size() : 1 + def opt_ffp_size = opt_ffp instanceof List ? opt_ffp.size() : 1 + def dfp_validated = (opt_dfp_size == 0 || opt_dfp_size == 1 || opt_dfp_size == num_files) ? true : false + def ffp_validated = (opt_ffp_size == 0 || opt_ffp_size == 1 || opt_ffp_size == num_files) ? true : false + + if ( !dfp_validated ) { error "Please input only zero, one, or N dfp files, where N is the number of input images" } + if ( !ffp_validated ) { error "Please input only zero, one, or N ffp files, where N is the number of input images" } + + """ + touch ${prefix}.ome.tif + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + ashlar: \$(ashlar --version | sed 's/^.*ashlar //' ) + END_VERSIONS + """ } diff --git a/modules/nf-core/ashlar/tests/main.nf.test b/modules/nf-core/ashlar/tests/main.nf.test new file mode 100644 index 00000000000..fd69ebd500d --- /dev/null +++ b/modules/nf-core/ashlar/tests/main.nf.test @@ -0,0 +1,155 @@ +nextflow_process { + + name "Test Process ASHLAR" + script "../main.nf" + process "ASHLAR" + + tag "modules" + tag "modules_nfcore" + tag "ashlar" + + test("ashlar - with fields") { + + when { + process { + """ + input[0] = [ + [ id:'test_all' ], + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif', checkIfExists: true) + ] + input[1] = [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-dfp.ome.tif', checkIfExists: true) + ] + input[2] = [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-ffp.ome.tif', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("ashlar - 1 cycle - ome-tif") { + + when { + process { + """ + input[0] = [ + [ id:'test_1' ], + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif', checkIfExists: true) + ] + input[1] = [] + input[2] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("ashlar - all cycles - ome-tif") { + + when { + process { + """ + input[0] = [ + [ id:'test_1' ], + [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif', checkIfExists: true), + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle2.ome.tif', checkIfExists: true), + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle3.ome.tif', checkIfExists: true) + + ] + ] + input[1] = [] + input[2] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("ashlar - all cycles - with fields - ome-tif") { + + when { + process { + """ + input[0] = [ + [ id:'test_1' ], + [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif', checkIfExists: true), + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle2.ome.tif', checkIfExists: true), + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle3.ome.tif', checkIfExists: true) + + ] + ] + input[1] = [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-dfp.ome.tif', checkIfExists: true) + ] + input[2] = [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-ffp.ome.tif', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("ashlar - with fields - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test_all' ], + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif', checkIfExists: true) + ] + input[1] = [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-dfp.ome.tif', checkIfExists: true) + ] + input[2] = [ + file(params.modules_testdata_base_path + 'imaging/ome-tiff/cycif-tonsil-ffp.ome.tif', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/ashlar/tests/main.nf.test.snap b/modules/nf-core/ashlar/tests/main.nf.test.snap new file mode 100644 index 00000000000..7c41f5b6d3e --- /dev/null +++ b/modules/nf-core/ashlar/tests/main.nf.test.snap @@ -0,0 +1,167 @@ +{ + "ashlar - with fields - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test_all" + }, + "test_all.ome.tif:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ], + "tif": [ + [ + { + "id": "test_all" + }, + "test_all.ome.tif:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-04T13:32:44.241620025" + }, + "ashlar - 1 cycle - ome-tif": { + "content": [ + { + "0": [ + [ + { + "id": "test_1" + }, + "test_1.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5" + ] + ], + "1": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ], + "tif": [ + [ + { + "id": "test_1" + }, + "test_1.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5" + ] + ], + "versions": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T11:13:49.614970275" + }, + "ashlar - with fields": { + "content": [ + { + "0": [ + [ + { + "id": "test_all" + }, + "test_all.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5" + ] + ], + "1": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ], + "tif": [ + [ + { + "id": "test_all" + }, + "test_all.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5" + ] + ], + "versions": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-04T13:28:26.923837754" + }, + "ashlar - all cycles - ome-tif": { + "content": [ + { + "0": [ + [ + { + "id": "test_1" + }, + "test_1.ome.tif:md5,c01dda923325588ea34a4fe341b17e6e" + ] + ], + "1": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ], + "tif": [ + [ + { + "id": "test_1" + }, + "test_1.ome.tif:md5,c01dda923325588ea34a4fe341b17e6e" + ] + ], + "versions": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T11:14:14.530220621" + }, + "ashlar - all cycles - with fields - ome-tif": { + "content": [ + { + "0": [ + [ + { + "id": "test_1" + }, + "test_1.ome.tif:md5,c01dda923325588ea34a4fe341b17e6e" + ] + ], + "1": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ], + "tif": [ + [ + { + "id": "test_1" + }, + "test_1.ome.tif:md5,c01dda923325588ea34a4fe341b17e6e" + ] + ], + "versions": [ + "versions.yml:md5,d5b1baeee01093ff65d37e155c9ce462" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T11:14:56.268009236" + } +} \ No newline at end of file diff --git a/modules/nf-core/ashlar/tests/nextflow.config b/modules/nf-core/ashlar/tests/nextflow.config new file mode 100644 index 00000000000..f7f07fcb6fb --- /dev/null +++ b/modules/nf-core/ashlar/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: ASHLAR_TILE { + ext.args = "--tile-size 512" + } +} diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 2e0e6420c33..887e215cbe8 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -22,9 +22,6 @@ artic/minion: ascat: - modules/nf-core/ascat/** - tests/modules/nf-core/ascat/** -ashlar: - - modules/nf-core/ashlar/** - - tests/modules/nf-core/ashlar/** ataqv/ataqv: - modules/nf-core/ataqv/ataqv/** - tests/modules/nf-core/ataqv/ataqv/** diff --git a/tests/modules/nf-core/ashlar/main.nf b/tests/modules/nf-core/ashlar/main.nf deleted file mode 100644 index 278bc3195c7..00000000000 --- a/tests/modules/nf-core/ashlar/main.nf +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { ASHLAR } from '../../../../modules/nf-core/ashlar/main.nf' -include { ASHLAR as ASHLAR_TILE } from '../../../../modules/nf-core/ashlar/main.nf' - -workflow test_ashlar_1_file { - - input_list = [ [ id:'test_all' ], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle1'], checkIfExists: true)] ] - - ASHLAR ( input_list, [], [] ) - -} - -workflow test_ashlar_all_files { - - input_list = [ [ id:'test_all' ], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle1'], checkIfExists: true), - file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle2'], checkIfExists: true), - file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle3'], checkIfExists: true)] ] - - ASHLAR ( input_list, [], [] ) - -} - -workflow test_ashlar_all_files_tile_size { - - input_list = [ [ id:'test_all' ], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle1'], checkIfExists: true), - file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle2'], checkIfExists: true), - file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle3'], checkIfExists: true)] ] - - ASHLAR_TILE ( input_list, [], [] ) - -} - -workflow test_ashlar_all_files_dfp_ffp { - - input_list = [ [ id:'test_all' ], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle1'], checkIfExists: true), - file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle2'], checkIfExists: true), - file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle3'], checkIfExists: true)] ] - - ASHLAR ( input_list, [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_dfp'], checkIfExists: true)], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_ffp'], checkIfExists: true)] ) - -} - -workflow test_ashlar_1_file_dfp_ffp { - - input_list = [ [ id:'test_all' ], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_cycle1'], checkIfExists: true)] ] - - ASHLAR ( input_list, [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_dfp'], checkIfExists: true)], - [file(params.test_data['imaging'] - ['ome-tiff'] - ['cycif_tonsil_ffp'], checkIfExists: true)] ) - -} - diff --git a/tests/modules/nf-core/ashlar/nextflow.config b/tests/modules/nf-core/ashlar/nextflow.config deleted file mode 100644 index 9b566dd3953..00000000000 --- a/tests/modules/nf-core/ashlar/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: ASHLAR_TILE { - ext.args = "--tile-size 512" - } - -} diff --git a/tests/modules/nf-core/ashlar/test.yml b/tests/modules/nf-core/ashlar/test.yml deleted file mode 100644 index 0dad0fdd990..00000000000 --- a/tests/modules/nf-core/ashlar/test.yml +++ /dev/null @@ -1,44 +0,0 @@ -- name: ashlar test_ashlar_1_file - command: nextflow run ./tests/modules/nf-core/ashlar -entry test_ashlar_1_file -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/ashlar/nextflow.config - tags: - - ashlar - files: - - path: output/ashlar/test_all.ome.tif - md5sum: 988bbb2c74d47dc22a9f3ac348f53ef5 - - path: output/ashlar/versions.yml - -- name: ashlar test_ashlar_all_files - command: nextflow run ./tests/modules/nf-core/ashlar -entry test_ashlar_all_files -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/ashlar/nextflow.config - tags: - - ashlar - files: - - path: output/ashlar/test_all.ome.tif - md5sum: c01dda923325588ea34a4fe341b17e6e - - path: output/ashlar/versions.yml - -- name: ashlar test_ashlar_all_files_tile_size - command: nextflow run ./tests/modules/nf-core/ashlar -entry test_ashlar_all_files_tile_size -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/ashlar/nextflow.config - tags: - - ashlar - files: - - path: output/ashlar/test_all.ome.tif - md5sum: 6d9573d40a93e38356ebc8b257f8dfb9 - - path: output/ashlar/versions.yml - -- name: ashlar test_ashlar_all_files_dfp_ffp - command: nextflow run ./tests/modules/nf-core/ashlar -entry test_ashlar_all_files_dfp_ffp -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/ashlar/nextflow.config - tags: - - ashlar - files: - - path: output/ashlar/test_all.ome.tif - md5sum: c01dda923325588ea34a4fe341b17e6e - - path: output/ashlar/versions.yml - -- name: ashlar test_ashlar_1_file_dfp_ffp - command: nextflow run ./tests/modules/nf-core/ashlar -entry test_ashlar_1_file_dfp_ffp -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/ashlar/nextflow.config - tags: - - ashlar - files: - - path: output/ashlar/test_all.ome.tif - md5sum: 988bbb2c74d47dc22a9f3ac348f53ef5 - - path: output/ashlar/versions.yml From 4a76db98b110df9f802cb6436019427b2d336368 Mon Sep 17 00:00:00 2001 From: Takeshi Gitano Date: Fri, 15 Nov 2024 15:08:25 +0100 Subject: [PATCH 06/12] parabricks/applybqsr version updated and tests migrated to nf-test (#6881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * parabricks/applybqsr version updated and tests migrated to pytest * parabricks/applybqsr version updated and tests migrated to pytest * modules update & possibly anything more * modules update & possibly anything more * modules update & possibly anything more * modules update & possibly anything more * parabricks/applybqsr version updated and tests migrated to pytest * update snaps * parabricks/applybqsr version updated and tests migrated to pytest * parabricks/applybqsr version updated and tests migrated to pytest * new snap * parabricks/applybqsr version updated and tests migrated to pytest * parabricks/applybqsr version updated and tests migrated to pytest * parabricks/applybqsr version updated and tests migrated to pytest * update snap * parabricks/applybqsr version updated and tests migrated to pytest * parabricks/applybqsr version updated and tests migrated to pytest * update snap * parabricks/applybqsr version updated and tests migrated to pytest * parabricks/applybqsr version updated and tests migrated to pytest * add other parabricks modules * add gpu label * remove old txt file * remove parabricks from normal nf-test * exclude parabricks from conda * fix excluding * simplify excluding * add docker debugging step * more debbuging * more debugging * add ssh step * move ssh debugging * remove ssh debugging * fix debugging step * add ssh-tunnel back * remove debugging * add gpu profile to test * modify test inclusion * update fq2bam snap * update snap fq2bammeth * remove fq2bammeth --------- Co-authored-by: orodrig3 Co-authored-by: famosab Co-authored-by: Famke Bäuerle <45968370+famosab@users.noreply.github.com> Co-authored-by: Matthias Hörtenhuber Co-authored-by: Matthias Hörtenhuber --- .github/workflows/test.yml | 144 ++++++++++-- .../parabricks/applybqsr/environment.yml | 3 - modules/nf-core/parabricks/applybqsr/main.nf | 13 +- modules/nf-core/parabricks/applybqsr/meta.yml | 17 +- .../parabricks/applybqsr/tests/main.nf.test | 213 ++++++++++++++++++ .../applybqsr/tests/main.nf.test.snap | 142 ++++++++++++ modules/nf-core/parabricks/fq2bam/main.nf | 1 + .../parabricks/fq2bam/tests/main.nf.test.snap | 20 +- .../fq2bammeth/tests/main.nf.test.snap | 36 ++- tests/config/nextflow.config | 5 + tests/config/nf-test.config | 6 + tests/config/pytest_modules.yml | 3 - .../nf-core/parabricks/applybqsr/main.nf | 39 ---- .../parabricks/applybqsr/nextflow.config | 5 - .../nf-core/parabricks/applybqsr/test.yml | 21 -- .../parabricks/applybqsr/test_GPU_config.txt | 9 - 16 files changed, 544 insertions(+), 133 deletions(-) delete mode 100644 modules/nf-core/parabricks/applybqsr/environment.yml create mode 100644 modules/nf-core/parabricks/applybqsr/tests/main.nf.test create mode 100644 modules/nf-core/parabricks/applybqsr/tests/main.nf.test.snap delete mode 100644 tests/modules/nf-core/parabricks/applybqsr/main.nf delete mode 100644 tests/modules/nf-core/parabricks/applybqsr/nextflow.config delete mode 100644 tests/modules/nf-core/parabricks/applybqsr/test.yml delete mode 100644 tests/modules/nf-core/parabricks/applybqsr/test_GPU_config.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43fe9733d3a..4c22ed82bed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -344,14 +344,10 @@ jobs: tags: nanoplot - profile: conda tags: ncbitools/vecscreen - - profile: conda - tags: parabricks/applybqsr - profile: conda tags: parabricks/dbsnp - profile: conda tags: parabricks/deepvariant - - profile: conda - tags: parabricks/fq2bam - profile: conda tags: parabricks/genotypegvcf - profile: conda @@ -648,18 +644,6 @@ jobs: path: modules/nf-core/islandpath - profile: conda path: modules/nf-core/scimap/mcmicro - - profile: conda - path: modules/nf-core/parabricks/fq2bammeth - - profile: docker_self_hosted - path: modules/nf-core/parabricks/fq2bammeth - - profile: singularity - path: modules/nf-core/parabricks/fq2bammeth - - profile: conda - path: modules/nf-core/parabricks/fq2bam - - profile: docker_self_hosted - path: modules/nf-core/parabricks/fq2bam - - profile: singularity - path: modules/nf-core/parabricks/fq2bam - profile: conda path: subworkflows/nf-core/vcf_annotate_ensemblvep - profile: conda @@ -672,6 +656,9 @@ jobs: path: subworkflows/nf-core/fastq_align_bwa - profile: conda path: subworkflows/nf-core/fasta_newick_epang_gappa + - path: modules/nf-core/parabricks/fq2bammeth + - path: modules/nf-core/parabricks/fq2bam + - path: modules/nf-core/parabricks/applybqsr - profile: conda path: modules/nf-core/xeniumranger/relabel - profile: conda @@ -786,6 +773,131 @@ jobs: run: | sudo rm -rf /home/ubuntu/tests/ + nf-test-gpu: + runs-on: ${{ github.event.inputs.runners || 'gpu' }} + name: nf-test with GPUs + needs: [nf-test-changes] + if: ( needs.nf-test-changes.outputs.paths != '[]' ) + strategy: + fail-fast: false + matrix: + path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] + profile: [conda, docker_self_hosted, singularity] + include: + - path: modules/nf-core/parabricks/applybqsr + - path: modules/nf-core/parabricks/fq2bam + # - path: modules/nf-core/parabricks/fq2bammeth excluded - tackle in different PR + exclude: + - profile: conda + path: modules/nf-core/parabricks/fq2bam + - profile: conda + path: modules/nf-core/parabricks/fq2bammeth + - profile: conda + path: modules/nf-core/parabricks/applybqsr + env: + NXF_ANSI_LOG: false + NFTEST_VER: "0.9.0" + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFTEST_VER }} + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + id: cache-pip-pdiff + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-pdiff + restore-keys: | + ${{ runner.os }}-pip-pdiff + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pdiff cryptography + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + auto-update-conda: true + channels: conda-forge,bioconda + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Set up secrets + - name: Set up nextflow secrets + # TODO Only run if the tag includes `sentieon` + if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null + run: | + nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") + # Test the module + - name: Run nf-test + if: ${{ matrix.path != '' }} + env: + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + run: | + # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker + if [ "${{ matrix.profile }}" == "docker" ]; then + PROFILE="docker_self_hosted" + else + PROFILE=${{ matrix.profile }} + fi + + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ matrix.profile }},gpu \ + --tap=test.tap \ + --verbose \ + ${{ matrix.path }} + + - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 + if: ${{ matrix.path != '' }} + with: + path: >- + test.tap + + - name: Clean up + if: always() + run: | + sudo rm -rf /home/ubuntu/tests/ + confirm-pass: runs-on: ubuntu-latest needs: diff --git a/modules/nf-core/parabricks/applybqsr/environment.yml b/modules/nf-core/parabricks/applybqsr/environment.yml deleted file mode 100644 index 3cebeff05bf..00000000000 --- a/modules/nf-core/parabricks/applybqsr/environment.yml +++ /dev/null @@ -1,3 +0,0 @@ -channels: - - conda-forge - - bioconda diff --git a/modules/nf-core/parabricks/applybqsr/main.nf b/modules/nf-core/parabricks/applybqsr/main.nf index b545297d89a..38b2af002b8 100644 --- a/modules/nf-core/parabricks/applybqsr/main.nf +++ b/modules/nf-core/parabricks/applybqsr/main.nf @@ -1,12 +1,16 @@ process PARABRICKS_APPLYBQSR { tag "$meta.id" label 'process_high' + label 'process_gpu' - container "nvcr.io/nvidia/clara/clara-parabricks:4.3.0-1" + container "nvcr.io/nvidia/clara/clara-parabricks:4.3.2-1" input: - tuple val(meta), path(bam), path(bam_index), path(bqsr_table), path(intervals) - tuple val(meta2), path(fasta) + tuple val(meta), path(bam) + tuple val(meta2), path(bam_index) + tuple val(meta3), path(bqsr_table) + tuple val(meta4), path(intervals) + tuple val(meta5), path(fasta) output: tuple val(meta), path("*.bam"), emit: bam @@ -25,6 +29,7 @@ process PARABRICKS_APPLYBQSR { def prefix = task.ext.prefix ?: "${meta.id}" def interval_command = intervals ? intervals.collect{"--interval-file $it"}.join(' ') : "" def copy_index_command = bam_index ? "cp -L $bam_index `readlink -f $bam`.bai" : "" + def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' """ # parabricks complains when index is not a regular file in the same directory as the bam # copy the index to this path. @@ -38,7 +43,7 @@ process PARABRICKS_APPLYBQSR { $interval_command \\ --out-bam ${prefix}.bam \\ --num-threads $task.cpus \\ - --num-gpus $task.accelerator.request \\ + $num_gpus \\ $args cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/parabricks/applybqsr/meta.yml b/modules/nf-core/parabricks/applybqsr/meta.yml index 09fdbacab1b..22f0c4c2067 100644 --- a/modules/nf-core/parabricks/applybqsr/meta.yml +++ b/modules/nf-core/parabricks/applybqsr/meta.yml @@ -23,18 +23,33 @@ input: type: file description: BAM file pattern: "*.bam" + - - meta2: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test' ] - bam_index: type: file description: BAM index file pattern: "*.bai" + - - meta3: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test' ] - bqsr_table: type: file description: Table from calculating BQSR. Output from parabricks/fq2bam or gatk4/baserecalibrator. pattern: "*.table" + - - meta4: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test' ] - intervals: type: file description: intervals - - - meta2: + - - meta5: type: map description: | Groovy Map containing sample information diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test new file mode 100644 index 00000000000..c85fd447b3d --- /dev/null +++ b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test @@ -0,0 +1,213 @@ +nextflow_process { + + name "Test Process PARABRICKS_APPLYBQSR" + script "../main.nf" + process "PARABRICKS_APPLYBQSR" + + tag "modules" + tag "modules_nfcore" + tag "parabricks" + tag "parabricks/applybqsr" + + test("sarscov2 - bam - pe") { + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [], [] // index not needed unless using intervals + ] + input[2] = [ + [ id:'bqsr_table'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/gatk/test.baserecalibrator.table', checkIfExists: true) + ] + input[3] = [ + [], [] // no intervals + ] + input[4] = [ + [ id:'reference'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + file(process.out.bai[0][1]).name, + process.out.versions + ).match() } + ) + } + + } + + test("sarscov2 - bam - pe - intervals") { + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [ id:'test-index'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'bqsr_table'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/gatk/test.baserecalibrator.table', checkIfExists: true) + ] + input[3] = [ + [ id:'intervals'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/bed/test.bed', checkIfExists: true) + ] + input[4] = [ + [ id:'reference'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + file(process.out.bai[0][1]).name, + process.out.versions + ).match() } + ) + } + } + + test("human - cram - pe") { + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true) + ] + input[1] = [ + [], [] // index not needed unless using intervals + ] + input[2] = [ + [ id:'bqsr_table'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gatk/test.baserecalibrator.table', checkIfExists: true) + ] + input[3] = [ + [], [] // no intervals + ] + input[4] = [ + [ id:'reference'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + bam(process.out.bam[0][1]).getReadsMD5(), + file(process.out.bai[0][1]).name, + process.out.versions + ).match() } + ) + } + + } + + test("human - cram - pe - intervals - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram', checkIfExists: true) + ] + input[1] = [ + [ id:'test-index'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/cram/test.paired_end.sorted.cram.crai', checkIfExists: true) + ] + input[2] = [ + [ id:'bqsr_table'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/gatk/test.baserecalibrator.table', checkIfExists: true) + ] + input[3] = [ + [ id:'intervals'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.bed', checkIfExists: true) + ] + input[4] = [ + [ id:'reference'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("sarscov2 - bam - pe - intervals - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ] + input[1] = [ + [ id:'test-index'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam.bai', checkIfExists: true) + ] + input[2] = [ + [ id:'bqsr_table'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/gatk/test.baserecalibrator.table', checkIfExists: true) + ] + input[3] = [ + [ id:'intervals'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/bed/test.bed', checkIfExists: true) + ] + input[4] = [ + [ id:'reference'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + +} diff --git a/modules/nf-core/parabricks/applybqsr/tests/main.nf.test.snap b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test.snap new file mode 100644 index 00000000000..c7a55002ac5 --- /dev/null +++ b/modules/nf-core/parabricks/applybqsr/tests/main.nf.test.snap @@ -0,0 +1,142 @@ +{ + "human - cram - pe": { + "content": [ + "2f11e4fe3390b8ad0a1852616fd1da04", + "test.bam.bai", + [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-10-31T08:40:58.540517299" + }, + "sarscov2 - bam - pe": { + "content": [ + "894549ee3ced6b5ca2eed2563a985217", + "test.bam.bai", + [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-10-30T10:55:02.759938278" + }, + "sarscov2 - bam - pe - intervals": { + "content": [ + "b194021b699cc5cf4b2b1f715e0b7b4c", + "test.bam.bai", + [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-10-30T11:20:43.476752722" + }, + "sarscov2 - bam - pe - intervals - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ], + "bai": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-10-30T11:21:02.658767469" + }, + "human - cram - pe - intervals - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ], + "bai": [ + [ + { + "id": "test" + }, + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "bam": [ + [ + { + "id": "test" + }, + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,139b8f2a0926a26cf843ac23daace6d0" + ] + } + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-10-30T11:20:56.908929981" + } +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index c1877bdf468..e6bf840c1f1 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -1,6 +1,7 @@ process PARABRICKS_FQ2BAM { tag "$meta.id" label 'process_high' + label 'process_gpu' container "nvcr.io/nvidia/clara/clara-parabricks:4.3.2-1" diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 50d96e78ffb..8d455f0efde 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -21,10 +21,10 @@ ] ], "2": [ - "versions.yml:md5,c054e8254095319b660a92bdc07d990f" + ], "3": [ - + "versions.yml:md5,c054e8254095319b660a92bdc07d990f" ], "4": [ @@ -65,10 +65,10 @@ } ], "meta": { - "nf-test": "0.9.1", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-10-30T00:20:56.208782" + "timestamp": "2024-11-12T11:09:05.670189769" }, "sarscov2 - fastq - pe - stub": { "content": [ @@ -92,10 +92,10 @@ ] ], "2": [ - "versions.yml:md5,c054e8254095319b660a92bdc07d990f" + ], "3": [ - + "versions.yml:md5,c054e8254095319b660a92bdc07d990f" ], "4": [ @@ -136,10 +136,10 @@ } ], "meta": { - "nf-test": "0.9.1", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-10-30T00:35:07.267293" + "timestamp": "2024-11-12T11:09:56.252425453" }, "sarscov2 - fastq - pe": { "content": [ diff --git a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap index 265bb4bf917..cf06f28c7b3 100644 --- a/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bammeth/tests/main.nf.test.snap @@ -21,15 +21,13 @@ ] ], "2": [ - [ - "alignment.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "3": [ ], "4": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "5": [ "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" @@ -56,12 +54,10 @@ ], "duplicate_metrics": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "qc_metrics": [ - [ - "alignment.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "versions": [ "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" @@ -69,10 +65,10 @@ } ], "meta": { - "nf-test": "0.9.1", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-10-26T21:29:35.546741" + "timestamp": "2024-11-12T11:20:10.809580487" }, "E. coli - fastq - pe - stub": { "content": [ @@ -96,15 +92,13 @@ ] ], "2": [ - [ - "alignment.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "3": [ ], "4": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "5": [ "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" @@ -131,12 +125,10 @@ ], "duplicate_metrics": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "qc_metrics": [ - [ - "alignment.txt:md5,d41d8cd98f00b204e9800998ecf8427e" - ] + ], "versions": [ "versions.yml:md5,882d996a64ad457fcaf85e3a75514baf" @@ -144,10 +136,10 @@ } ], "meta": { - "nf-test": "0.9.1", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.0" }, - "timestamp": "2024-10-26T21:33:35.78287" + "timestamp": "2024-11-12T11:20:18.927598721" }, "SRR389222 - fastq - se": { "content": [ diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index c86fb134d75..de10acc9fb2 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -25,6 +25,11 @@ if ("$PROFILE" == "singularity") { podman.enabled = true podman.userEmulation = true podman.runOptions = "--runtime crun --platform linux/x86_64 --systemd=always" +} else if ("$PROFILE" == "gpu") { + docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' + apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' + singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' + use_gpu = true } else { docker.enabled = true docker.userEmulation = false diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 0bbe5e5b0c0..1bdf57d23c1 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -32,6 +32,12 @@ profiles { docker.enabled = true docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' + apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' + singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' + use_gpu = true + } docker_self_hosted { docker.enabled = true docker.fixOwnership = true diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 887e215cbe8..8a781b0c78c 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -449,9 +449,6 @@ ncbitools/vecscreen: oncocnv: - modules/nf-core/oncocnv/** - tests/modules/nf-core/oncocnv/** -parabricks/applybqsr: - - modules/nf-core/parabricks/applybqsr/** - - tests/modules/nf-core/parabricks/applybqsr/** parabricks/dbsnp: - modules/nf-core/parabricks/dbsnp/** - tests/modules/nf-core/parabricks/dbsnp/** diff --git a/tests/modules/nf-core/parabricks/applybqsr/main.nf b/tests/modules/nf-core/parabricks/applybqsr/main.nf deleted file mode 100644 index 8da248632f8..00000000000 --- a/tests/modules/nf-core/parabricks/applybqsr/main.nf +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { PARABRICKS_APPLYBQSR } from '../../../../../modules/nf-core/parabricks/applybqsr/main.nf' - -workflow test_parabricks_applybqsr { - - input = [ - [ id:'test'], - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - [], // index not needed unless using intervals - file(params.test_data['sarscov2']['illumina']['test_baserecalibrator_table'], checkIfExists: true), - [] - ] - fasta = [ - [ id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] - - PARABRICKS_APPLYBQSR ( input, fasta ) -} - -workflow test_parabricks_applybqsr_intervals { - - input = [ - [ id:'test'], - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_baserecalibrator_table'], checkIfExists: true), - file(params.test_data['sarscov2']['genome']['test_bed'], checkIfExists: true) - ] - fasta = [ - [ id:'test'], - file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - ] - - PARABRICKS_APPLYBQSR ( input, fasta ) -} diff --git a/tests/modules/nf-core/parabricks/applybqsr/nextflow.config b/tests/modules/nf-core/parabricks/applybqsr/nextflow.config deleted file mode 100644 index 8730f1c4b93..00000000000 --- a/tests/modules/nf-core/parabricks/applybqsr/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} diff --git a/tests/modules/nf-core/parabricks/applybqsr/test.yml b/tests/modules/nf-core/parabricks/applybqsr/test.yml deleted file mode 100644 index 11c821cae01..00000000000 --- a/tests/modules/nf-core/parabricks/applybqsr/test.yml +++ /dev/null @@ -1,21 +0,0 @@ -- name: parabricks applybqsr test_parabricks_applybqsr - command: nextflow run ./tests/modules/nf-core/parabricks/applybqsr/ -entry test_parabricks_applybqsr -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/applybqsr/nextflow.config -stub-run - tags: - - parabricks/applybqsr - - parabricks - files: - - path: output/parabricks/test.bam - should_exist: true - - path: output/parabricks/test.bam.bai - should_exist: true - -- name: parabricks applybqsr test_parabricks_applybqsr_intervals - command: nextflow run ./tests/modules/nf-core/parabricks/applybqsr/ -entry test_parabricks_applybqsr_intervals -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/applybqsr/nextflow.config -stub-run - tags: - - parabricks/applybqsr - - parabricks - files: - - path: output/parabricks/test.bam - should_exist: true - - path: output/parabricks/test.bam.bai - should_exist: true diff --git a/tests/modules/nf-core/parabricks/applybqsr/test_GPU_config.txt b/tests/modules/nf-core/parabricks/applybqsr/test_GPU_config.txt deleted file mode 100644 index 093a8b1afd1..00000000000 --- a/tests/modules/nf-core/parabricks/applybqsr/test_GPU_config.txt +++ /dev/null @@ -1,9 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - memory = "15 GB" - cpus = 4 - accelerator = 1 -} -docker.runOptions = "--gpus all" -singularity.runOptions = "--nv" \ No newline at end of file From 1c23afc032d7e340a7e4a17140b3847bf1cc0b18 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:23:45 -0600 Subject: [PATCH 07/12] chore(deps): update dotnet monorepo to v9 (major) (#5846) * chore(deps): update dotnet monorepo to v9 * build: Swap COPY for ADD * build: Bump container version --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Edmund Miller --- modules/nf-core/wittyer/Dockerfile | 6 +++--- modules/nf-core/wittyer/main.nf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/wittyer/Dockerfile b/modules/nf-core/wittyer/Dockerfile index fbf732f32bd..5d1236d5985 100644 --- a/modules/nf-core/wittyer/Dockerfile +++ b/modules/nf-core/wittyer/Dockerfile @@ -1,12 +1,12 @@ -FROM mcr.microsoft.com/dotnet/sdk:6.0@sha256:6df1177e48b55272316d08f19cb383483af82aca5cdc67a76c414bc200847624 as builder +FROM mcr.microsoft.com/dotnet/sdk:9.0@sha256:7d24e90a392e88eb56093e4eb325ff883ad609382a55d42f17fd557b997022ca as builder +ADD https://github.com/Illumina/witty.er.git#v0.5.2 /src WORKDIR /src -COPY . /src RUN cd Wittyer \ && dotnet publish -f net6.0 -r linux-x64 -c Release -o /output \ && chmod +x /output/Wittyer -FROM mcr.microsoft.com/dotnet/runtime:6.0@sha256:437cda84bdce26ce074d88b63abeec567c7226d73e8b911605077468e1d5c8d5 +FROM mcr.microsoft.com/dotnet/runtime:9.0@sha256:1e5eb0ed94ca96a34a914456db80e48bd1bb7bc3e3c8eda5e2c3d89c153c3081 LABEL git_repository=https://git.illumina.com/DASTE/Ilmn.Das.App.Wittyer.git WORKDIR /opt/Wittyer diff --git a/modules/nf-core/wittyer/main.nf b/modules/nf-core/wittyer/main.nf index 5cee35c8fd8..da0f7efbbd9 100644 --- a/modules/nf-core/wittyer/main.nf +++ b/modules/nf-core/wittyer/main.nf @@ -2,7 +2,7 @@ process WITTYER { tag "$meta.id" label 'process_single' - container "nf-core/wittyer:0.5.2.0" + container "nf-core/modules/wittyer:4c55c27c711b558f" // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { From 21f230b8cca43755bf73470e6fd0290832a98aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Sat, 16 Nov 2024 05:24:44 +0100 Subject: [PATCH 08/12] Add GPU-based tests (#6953) * split nf-test section into its own action.yml and add gpu test cases * add gpu profile * Update test.yml * add checkout action before running nf-test action * add miniconda-setup step * Update tests/config/nextflow.config Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> * try to fix composite action * add profile info to include step, to avoid empty profile values * touch fastqc to kick of some tests * add gpu profile to nf-test.config * fix miniconda setup step * remove miniconda version pin * try to exclude all non-gpu tests * remove changed paths * add check for changes in parabricks * fix pip error * add setup-python to main script * remove conda defaults channels --------- Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> --- .github/actions/nf-test-action/action.yml | 83 ++++++++ .github/workflows/test.yml | 234 +++------------------- .pre-commit-config.yaml | 2 +- modules/nf-core/fastqc/meta.yml | 1 + tests/config/nextflow.config | 5 - tests/config/nf-test.config | 6 + 6 files changed, 117 insertions(+), 214 deletions(-) create mode 100644 .github/actions/nf-test-action/action.yml diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml new file mode 100644 index 00000000000..4dd6df48919 --- /dev/null +++ b/.github/actions/nf-test-action/action.yml @@ -0,0 +1,83 @@ +name: "nf-test Action" +description: "Runs nf-test with common setup steps" +inputs: + path: + description: "Path to test" + required: true + profile: + description: "Profile to use" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: "0.9.2" + install-pdiff: true + + - name: Setup apptainer + if: contains(inputs.profile, 'singularity') + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: contains(inputs.profile, 'singularity') + shell: bash + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Conda setup + if: ${{inputs.profile == 'conda'}} + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + auto-update-conda: true + conda-solver: libmamba + conda-remove-defaults: true + + # Set up secrets + - name: Set up nextflow secrets + if: env.SENTIEON_ENCRYPTION_KEY != '' && env.SENTIEON_LICENSE_MESSAGE != '' + shell: bash + run: | + nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE") + + - name: Run nf-test + shell: bash + env: + SENTIEON_LICSRVR_IP: ${{ env.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + run: | + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ inputs.profile }} \ + --tap=test.tap \ + --verbose \ + ${{ inputs.path }} + - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 + if: ${{ inputs.path != '' }} + with: + path: >- + test.tap + + - name: Clean up + if: always() + shell: bash + run: | + sudo rm -rf /home/ubuntu/tests/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c22ed82bed..0ff73656067 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 # FIXME Flip this off once we get to less than a couple hundred. Adding # this so it will only run against changed files. It'll make it much @@ -86,8 +88,7 @@ jobs: with: fetch-depth: 2 # To retrieve the preceding commit. - # TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented - - uses: mirpedrol/paths-filter@main + - uses: dorny/paths-filter@v3 id: filter with: filters: "tests/config/pytest_modules.yml" @@ -135,6 +136,7 @@ jobs: - name: debug run: | + echo ${{ steps.filter.outputs.components }} echo ${{ steps.outputs.outputs.modules }} echo ${{ steps.outputs.outputs.subworkflows }} @@ -441,6 +443,7 @@ jobs: miniconda-version: "latest" channels: conda-forge,bioconda python-version: ${{ matrix.python-version }} + conda-remove-defaults: true - name: Conda setup run: | @@ -485,7 +488,7 @@ jobs: !${{ github.workspace }}/.singularity nf-test: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: nf-test needs: [nf-test-changes] if: ( needs.nf-test-changes.outputs.paths != '[]' ) @@ -670,233 +673,47 @@ jobs: env: NXF_ANSI_LOG: false - NFTEST_VER: "0.9.0" - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pdiff - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pdiff - restore-keys: | - ${{ runner.os }}-pip-pdiff - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pdiff cryptography - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,bioconda - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Set up secrets - - name: Set up nextflow secrets - # TODO Only run if the tag includes `sentieon` - if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null - run: | - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") - - # Test the module - - name: Run nf-test - if: ${{ matrix.path != '' }} - env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }} \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} - - - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 - if: ${{ matrix.path != '' }} + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action with: - path: >- - test.tap - - - name: Clean up - if: always() - run: | - sudo rm -rf /home/ubuntu/tests/ + path: ${{ matrix.path }} + profile: ${{ matrix.profile }} nf-test-gpu: - runs-on: ${{ github.event.inputs.runners || 'gpu' }} - name: nf-test with GPUs + runs-on: "gpu" + name: nf-test-gpu needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) + if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) strategy: fail-fast: false matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [conda, docker_self_hosted, singularity] include: - path: modules/nf-core/parabricks/applybqsr + profile: [docker_self_hosted, singularity] - path: modules/nf-core/parabricks/fq2bam - # - path: modules/nf-core/parabricks/fq2bammeth excluded - tackle in different PR - exclude: - - profile: conda - path: modules/nf-core/parabricks/fq2bam - - profile: conda - path: modules/nf-core/parabricks/fq2bammeth - - profile: conda - path: modules/nf-core/parabricks/applybqsr + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/fq2bammeth + profile: [docker_self_hosted, singularity] env: NXF_ANSI_LOG: false - NFTEST_VER: "0.9.0" - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-test - uses: nf-core/setup-nf-test@v1 - with: - version: ${{ env.NFTEST_VER }} - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pdiff - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pdiff - restore-keys: | - ${{ runner.os }}-pip-pdiff - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pdiff cryptography - - - name: Set up miniconda - if: matrix.profile == 'conda' - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,bioconda - - - name: Conda setup - if: matrix.profile == 'conda' - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Set up secrets - - name: Set up nextflow secrets - # TODO Only run if the tag includes `sentieon` - if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null - run: | - nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}") - # Test the module - - name: Run nf-test - if: ${{ matrix.path != '' }} + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} SENTIEON_AUTH_MECH: "GitHub Actions - token" - run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }},gpu \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} - - - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1 - if: ${{ matrix.path != '' }} with: - path: >- - test.tap - - - name: Clean up - if: always() - run: | - sudo rm -rf /home/ubuntu/tests/ + path: ${{ matrix.path }} + profile: ${{ matrix.profile }},gpu confirm-pass: runs-on: ubuntu-latest @@ -910,6 +727,7 @@ jobs: pytest, nf-test-changes, nf-test, + nf-test-gpu, ] if: always() steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d9bf144771..b97fd79f3a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: prettier additional_dependencies: - - prettier@3.2.5 + - prettier@3.3.3 - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.29.4 hooks: diff --git a/modules/nf-core/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml index 4827da7af24..2b2e62b8aea 100644 --- a/modules/nf-core/fastqc/meta.yml +++ b/modules/nf-core/fastqc/meta.yml @@ -11,6 +11,7 @@ tools: FastQC gives general quality metrics about your reads. It provides information about the quality score distribution across your reads, the per base sequence content (%A/C/G/T). + You get information about adapter contamination and other overrepresented sequences. homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index de10acc9fb2..c86fb134d75 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -25,11 +25,6 @@ if ("$PROFILE" == "singularity") { podman.enabled = true podman.userEmulation = true podman.runOptions = "--runtime crun --platform linux/x86_64 --systemd=always" -} else if ("$PROFILE" == "gpu") { - docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' - apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' - singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' - use_gpu = true } else { docker.enabled = true docker.userEmulation = false diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 1bdf57d23c1..f651ff4d89d 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -43,6 +43,12 @@ profiles { docker.fixOwnership = true docker.runOptions = '--platform=linux/amd64' } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus all' + apptainer.runOptions = '--nv' + singularity.runOptions = '--nv' + use_gpu = true + } } docker.registry = 'quay.io' From 484afd16770cf3c466a6c385e33746c877656663 Mon Sep 17 00:00:00 2001 From: Daniel Schmitz Date: Sat, 16 Nov 2024 06:22:19 +0100 Subject: [PATCH 09/12] Added module trgt/genotype (#7002) * Imported genotype module from trgt branch * Fixed test issues --------- Co-authored-by: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/nf-core/trgt/genotype/environment.yml | 7 + modules/nf-core/trgt/genotype/main.nf | 56 +++++ modules/nf-core/trgt/genotype/meta.yml | 98 +++++++++ .../nf-core/trgt/genotype/tests/main.nf.test | 154 ++++++++++++++ .../trgt/genotype/tests/main.nf.test.snap | 198 ++++++++++++++++++ 5 files changed, 513 insertions(+) create mode 100644 modules/nf-core/trgt/genotype/environment.yml create mode 100644 modules/nf-core/trgt/genotype/main.nf create mode 100644 modules/nf-core/trgt/genotype/meta.yml create mode 100644 modules/nf-core/trgt/genotype/tests/main.nf.test create mode 100644 modules/nf-core/trgt/genotype/tests/main.nf.test.snap diff --git a/modules/nf-core/trgt/genotype/environment.yml b/modules/nf-core/trgt/genotype/environment.yml new file mode 100644 index 00000000000..721e0df3d5f --- /dev/null +++ b/modules/nf-core/trgt/genotype/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::trgt=1.2.0" diff --git a/modules/nf-core/trgt/genotype/main.nf b/modules/nf-core/trgt/genotype/main.nf new file mode 100644 index 00000000000..3fa0b7b4e59 --- /dev/null +++ b/modules/nf-core/trgt/genotype/main.nf @@ -0,0 +1,56 @@ +process TRGT_GENOTYPE { + tag "$meta.id" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/trgt:1.2.0--h9ee0642_0': + 'biocontainers/trgt:1.2.0--h9ee0642_0' }" + + input: + tuple val(meta) , path(bam), path(bai), val(karyotype) + tuple val(meta2), path(fasta) + tuple val(meta3), path(fai) + tuple val(meta4), path(repeats) + + output: + tuple val(meta), path("*.vcf.gz") , emit: vcf + tuple val(meta), path("*.spanning.bam"), emit: bam , optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def karyo = karyotype ? "--karyotype ${karyotype}" : "" + """ + trgt genotype \\ + $args \\ + --genome ${fasta} \\ + --reads ${bam} \\ + --repeats ${repeats} \\ + ${karyo} \\ + --threads ${task.cpus} \\ + --output-prefix ${prefix} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + trgt: \$(trgt --version |& sed '1!d ; s/trgt //') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.spanning.bam + echo "" | gzip > ${prefix}.vcf.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + trgt: \$(trgt --version |& sed '1!d ; s/trgt //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/trgt/genotype/meta.yml b/modules/nf-core/trgt/genotype/meta.yml new file mode 100644 index 00000000000..bd17e4cae82 --- /dev/null +++ b/modules/nf-core/trgt/genotype/meta.yml @@ -0,0 +1,98 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "trgt_genotype" +description: Tandem repeat genotyping from PacBio HiFi data +keywords: + - repeat expansion + - pacbio + - genomics +tools: + - "trgt": + description: "Tandem repeat genotyping and visualization from PacBio HiFi data" + homepage: "https://github.com/PacificBiosciences/trgt" + documentation: "https://github.com/PacificBiosciences/trgt/blob/main/docs/tutorial.md" + tool_dev_url: "https://github.com/PacificBiosciences/trgt" + doi: "10.1038/s41587-023-02057-3" + licence: + ["Pacific Biosciences Software License (https://github.com/PacificBiosciences/trgt/blob/main/LICENSE.md)"] + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - bam: + type: file + description: Sorted BAM file + pattern: "*.bam" + - bai: + type: file + description: "Index of the BAM file" + pattern: "*.bam" + - karyotype: + type: string + description: "Karyotype of the sample. Either XX or XY. Defaults to XX if not given" + enum: + - XX + - XY + - - meta2: + type: map + description: | + Groovy map containing reference information + e.g. `[ id: 'genome' ]` + - fasta: + type: file + description: "FASTA reference file" + pattern: "*.{fasta,fa,fna}" + - - meta3: + type: map + description: | + Groovy map containing reference information + e.g. `[ id: 'genome' ]` + - fai: + type: file + description: "Index for FASTA file" + pattern: "*.fai" + - - meta4: + type: map + description: | + Groovy map containing repeat information + e.g. `[ id: 'repeats' ]` + - repeats: + type: file + description: "BED file with repeat coordinates" + pattern: "*.bed" + +output: + - vcf: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.vcf.gz": + type: file + description: "VCF file with repeat genotypes" + pattern: "*.vcf.gz" + - bam: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*.spanning.bam": + type: file + description: BAM file with pieces of reads aligning to repeats + pattern: "*.spanning.bam" + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@Schmytzi" + - "@fellen31" +maintainers: + - "@Schmytzi" diff --git a/modules/nf-core/trgt/genotype/tests/main.nf.test b/modules/nf-core/trgt/genotype/tests/main.nf.test new file mode 100644 index 00000000000..d93a2e9603c --- /dev/null +++ b/modules/nf-core/trgt/genotype/tests/main.nf.test @@ -0,0 +1,154 @@ +// nf-core modules test trgt +nextflow_process { + + name "Test Process TRGT_GENOTYPE" + script "../main.nf" + process "TRGT_GENOTYPE" + + tag "modules" + tag "modules_nfcore" + tag "trgt" + tag "trgt/genotype" + tag "samtools/faidx" + tag "gunzip" + + setup { + run("GUNZIP"){ + script "../../../gunzip/main.nf" + process { + """ + input[0] = [ + [ id : 'chr22' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr22_chr22_KI270734v1_random/sequence/genome.fa.gz', checkIfExists: true) + ] + """ + } + } + run("SAMTOOLS_FAIDX"){ + script "../../../samtools/faidx/main.nf" + process { + """ + input[0] = GUNZIP.out.gunzip + input[1] = [[],[]] + """ + } + } + } + + test("homo sapiens - [bam,bai,XX], [fa,fai], bed") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + 'XX' + ] + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("homo sapiens - [bam,bai,[]], [fa,fai], bed") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + [] + ] + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("homo sapiens - [bam,bai,XX], [fa,fai], bed - stub") { + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + 'XX' + ] + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("homo sapiens - [bam,bai,[]], [fa,fai], bed - stub") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/pacbio/bam/test.sorted.bam.bai', checkIfExists: true), + [] + ] + input[1] = GUNZIP.out.gunzip + input[2] = SAMTOOLS_FAIDX.out.fai + input[3] = Channel.of('chr22\t18890357\t18890451\tID=TEST;MOTIFS=AT;STRUC=(AT)n') + .collectFile(name : 'repeats.bed', newLine: false) + .map { file -> [ [ id : 'chr22' ], file ] } + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } +} diff --git a/modules/nf-core/trgt/genotype/tests/main.nf.test.snap b/modules/nf-core/trgt/genotype/tests/main.nf.test.snap new file mode 100644 index 00000000000..9e50fb81eb3 --- /dev/null +++ b/modules/nf-core/trgt/genotype/tests/main.nf.test.snap @@ -0,0 +1,198 @@ +{ + "homo sapiens - [bam,bai,XX], [fa,fai], bed": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,4e5b6028e3ea3d06829ca18af1cc492e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,645a2c199d04aaf733ce84247f0b0f37" + ] + ], + "2": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ], + "bam": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,645a2c199d04aaf733ce84247f0b0f37" + ] + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,4e5b6028e3ea3d06829ca18af1cc492e" + ] + ], + "versions": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T16:07:55.029938504" + }, + "homo sapiens - [bam,bai,[]], [fa,fai], bed - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,0337e3287dcc71728c36c185dae7738f" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,8d6d1bf813109870692d2f20d59399e7" + ] + ], + "2": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ], + "bam": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,8d6d1bf813109870692d2f20d59399e7" + ] + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,0337e3287dcc71728c36c185dae7738f" + ] + ], + "versions": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T16:08:33.937952067" + }, + "homo sapiens - [bam,bai,XX], [fa,fai], bed - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ], + "bam": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T16:16:39.197756426" + }, + "homo sapiens - [bam,bai,[]], [fa,fai], bed": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,0337e3287dcc71728c36c185dae7738f" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,8d6d1bf813109870692d2f20d59399e7" + ] + ], + "2": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ], + "bam": [ + [ + { + "id": "test" + }, + "test.spanning.bam:md5,8d6d1bf813109870692d2f20d59399e7" + ] + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf.gz:md5,0337e3287dcc71728c36c185dae7738f" + ] + ], + "versions": [ + "versions.yml:md5,b773615cf974b094032306822d32017b" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-11-15T16:08:07.631383773" + } +} \ No newline at end of file From 48f98fa1f47094df9e0847a0d0f45d8448eaee1e Mon Sep 17 00:00:00 2001 From: Sateesh_Peri <33637490+sateeshperi@users.noreply.github.com> Date: Sat, 16 Nov 2024 11:12:05 +0530 Subject: [PATCH 10/12] Fastq align dedup bismark (#6998) * init subworkflow * bismark + bismark_hisat tests * update coverage2cytosine outputs * update meta, tests * rm index from snapshot * Apply suggestions from code review Co-authored-by: Mahesh Binzer-Panchal * apply suggestions from code review and improve tests * rm extra params from test config --------- Co-authored-by: Mahesh Binzer-Panchal --- .../nf-core/fastq_align_dedup_bismark/main.nf | 153 +++++ .../fastq_align_dedup_bismark/meta.yml | 136 ++++ .../tests/main.nf.test | 336 ++++++++++ .../tests/main.nf.test.snap | 612 ++++++++++++++++++ .../tests/nextflow.config | 15 + 5 files changed, 1252 insertions(+) create mode 100644 subworkflows/nf-core/fastq_align_dedup_bismark/main.nf create mode 100644 subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml create mode 100644 subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test create mode 100644 subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap create mode 100644 subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf b/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf new file mode 100644 index 00000000000..1497e5adda8 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/main.nf @@ -0,0 +1,153 @@ +include { BISMARK_ALIGN } from '../../../modules/nf-core/bismark/align/main' +include { BISMARK_DEDUPLICATE } from '../../../modules/nf-core/bismark/deduplicate/main' +include { BISMARK_METHYLATIONEXTRACTOR } from '../../../modules/nf-core/bismark/methylationextractor/main' +include { BISMARK_COVERAGE2CYTOSINE } from '../../../modules/nf-core/bismark/coverage2cytosine/main' +include { BISMARK_REPORT } from '../../../modules/nf-core/bismark/report/main' +include { BISMARK_SUMMARY } from '../../../modules/nf-core/bismark/summary/main' +include { SAMTOOLS_SORT as SAMTOOLS_SORT_DEDUPLICATED } from '../../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX as SAMTOOLS_INDEX_DEDUPLICATED } from '../../../modules/nf-core/samtools/index/main' + +workflow FASTQ_ALIGN_DEDUP_BISMARK { + + take: + ch_reads // channel: [ val(meta), [ reads ] ] + ch_fasta // channel: [ val(meta), [ fasta ] ] + ch_bismark_index // channel: [ val(meta), [ bismark index ] ] + skip_deduplication // boolean: whether to deduplicate alignments + cytosine_report // boolean: whether the run coverage2cytosine + + main: + ch_versions = Channel.empty() + ch_coverage2cytosine_coverage = Channel.empty() + ch_coverage2cytosine_report = Channel.empty() + ch_coverage2cytosine_summary = Channel.empty() + ch_methylation_bedgraph = Channel.empty() + ch_methylation_calls = Channel.empty() + ch_methylation_coverage = Channel.empty() + ch_methylation_report = Channel.empty() + ch_methylation_mbias = Channel.empty() + ch_bismark_report = Channel.empty() + ch_bismark_summary = Channel.empty() + ch_multiqc_files = Channel.empty() + + /* + * Align with bismark + */ + BISMARK_ALIGN ( + ch_reads, + ch_fasta, + ch_bismark_index + ) + ch_versions = ch_versions.mix(BISMARK_ALIGN.out.versions) + + if (skip_deduplication) { + alignments = BISMARK_ALIGN.out.bam + alignment_reports = BISMARK_ALIGN.out.report.map{ meta, report -> [ meta, report, [] ] } + } else { + /* + * Run deduplicate_bismark + */ + BISMARK_DEDUPLICATE ( + BISMARK_ALIGN.out.bam + ) + alignments = BISMARK_DEDUPLICATE.out.bam + alignment_reports = BISMARK_ALIGN.out.report.join(BISMARK_DEDUPLICATE.out.report) + ch_versions = ch_versions.mix(BISMARK_DEDUPLICATE.out.versions) + } + + /* + * Run bismark_methylation_extractor + */ + BISMARK_METHYLATIONEXTRACTOR ( + alignments, + ch_bismark_index + ) + ch_methylation_bedgraph = BISMARK_METHYLATIONEXTRACTOR.out.bedgraph + ch_methylation_calls = BISMARK_METHYLATIONEXTRACTOR.out.methylation_calls + ch_methylation_coverage = BISMARK_METHYLATIONEXTRACTOR.out.coverage + ch_methylation_report = BISMARK_METHYLATIONEXTRACTOR.out.report + ch_methylation_mbias = BISMARK_METHYLATIONEXTRACTOR.out.mbias + ch_versions = ch_versions.mix(BISMARK_METHYLATIONEXTRACTOR.out.versions) + + /* + * Run bismark coverage2cytosine + */ + if (cytosine_report) { + BISMARK_COVERAGE2CYTOSINE ( + BISMARK_METHYLATIONEXTRACTOR.out.coverage, + ch_fasta, + ch_bismark_index + ) + ch_coverage2cytosine_coverage = BISMARK_COVERAGE2CYTOSINE.out.coverage + ch_coverage2cytosine_report = BISMARK_COVERAGE2CYTOSINE.out.report + ch_coverage2cytosine_summary = BISMARK_COVERAGE2CYTOSINE.out.summary + ch_versions = ch_versions.mix(BISMARK_COVERAGE2CYTOSINE.out.versions) + } + + /* + * Generate bismark sample reports + */ + BISMARK_REPORT ( + alignment_reports + .join(ch_methylation_report) + .join(ch_methylation_mbias) + ) + ch_bismark_report = BISMARK_REPORT.out.report + ch_versions = ch_versions.mix(BISMARK_REPORT.out.versions) + + /* + * Generate bismark summary report + */ + BISMARK_SUMMARY ( + BISMARK_ALIGN.out.bam.collect{ meta, bam -> bam.name }, + alignment_reports.collect{ meta, align_report, dedup_report -> align_report }, + alignment_reports.collect{ meta, align_report, dedup_report -> dedup_report }.ifEmpty([]), + BISMARK_METHYLATIONEXTRACTOR.out.report.collect{ meta, report -> report }, + BISMARK_METHYLATIONEXTRACTOR.out.mbias.collect{ meta, mbias -> mbias } + ) + ch_bismark_summary = BISMARK_SUMMARY.out.summary + ch_versions = ch_versions.mix(BISMARK_SUMMARY.out.versions) + + /* + * MODULE: Run samtools sort on dedup bam + */ + SAMTOOLS_SORT_DEDUPLICATED ( + alignments, + [[:],[]] // [ [meta], [fasta]] + ) + ch_versions = ch_versions.mix(SAMTOOLS_SORT_DEDUPLICATED.out.versions) + + /* + * MODULE: Run samtools index on dedup bam + */ + SAMTOOLS_INDEX_DEDUPLICATED ( + SAMTOOLS_SORT_DEDUPLICATED.out.bam + ) + ch_versions = ch_versions.mix(SAMTOOLS_INDEX_DEDUPLICATED.out.versions) + + /* + * Collect MultiQC inputs + */ + ch_multiqc_files = BISMARK_SUMMARY.out.summary + .mix(alignment_reports.collect{ meta, align_report, dedup_report -> align_report }) + .mix(alignment_reports.collect{ meta, align_report, dedup_report -> dedup_report }) + .mix(BISMARK_METHYLATIONEXTRACTOR.out.report.collect{ meta, report -> report }) + .mix(BISMARK_METHYLATIONEXTRACTOR.out.mbias.collect{ meta, mbias -> mbias }) + .mix(BISMARK_REPORT.out.report.collect{ meta, report -> report }) + + emit: + dedup_bam = SAMTOOLS_SORT_DEDUPLICATED.out.bam // channel: [ val(meta), [ bam ] ] + dedup_bam_index = SAMTOOLS_INDEX_DEDUPLICATED.out.bai // channel: [ val(meta), [ bai ] ] + coverage2cytosine_coverage = ch_coverage2cytosine_coverage // channel: [ val(meta), [ coverage ] ] + coverage2cytosine_report = ch_coverage2cytosine_report // channel: [ val(meta), [ report ] ] + coverage2cytosine_summary = ch_coverage2cytosine_summary // channel: [ val(meta), [ summary ] ] + methylation_bedgraph = ch_methylation_bedgraph // channel: [ val(meta), [ bedgraph ] ] + methylation_calls = ch_methylation_calls // channel: [ val(meta), [ methylation_calls ] ] + methylation_coverage = ch_methylation_coverage // channel: [ val(meta), [ coverage ] ] + methylation_report = ch_methylation_report // channel: [ val(meta), [ report ] ] + methylation_mbias = ch_methylation_mbias // channel: [ val(meta), [ mbias ] ] + bismark_report = ch_bismark_report // channel: [ val(meta), [ report ] ] + bismark_summary = ch_bismark_summary // channel: [ val(meta), [ summary ] ] + multiqc = ch_multiqc_files // path: *{html,txt} + versions = ch_versions // path: *.version.txt +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml b/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml new file mode 100644 index 00000000000..4f3aa80b946 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/meta.yml @@ -0,0 +1,136 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "fastq_align_dedup_bismark" +description: Align BS-Seq reads to a reference genome using bismark, deduplicate and sort +keywords: + - bismark + - 3-letter genome + - map + - methylation + - 5mC + - methylseq + - bisulphite + - bisulfite + - bam +components: + - bismark/align + - samtools/sort + - samtools/index + - bismark/deduplicate + - bismark/methylationextractor + - bismark/coverage2cytosine + - bismark/report + - bismark/summary +input: + - ch_reads: + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. + Structure: [ val(meta), [ path(reads) ] ] + pattern: "*.{fastq,fastq.gz}" + - ch_fasta: + type: file + description: | + Structure: [ val(meta), path(fasta) ] + pattern: "*.{fa,fa.gz}" + - ch_index: + description: | + Bismark genome index files + Structure: [ val(meta), path(index) ] + pattern: "BismarkIndex" + - skip_deduplication: + type: boolean + description: | + Skip deduplication of aligned reads + - cytosine_report: + type: boolean + description: | + Produce coverage2cytosine reports that relates methylation calls back to genomic cytosine contexts +output: + - dedup_bam: + type: file + description: | + Channel containing deduplicated BAM files. + Structure: [ val(meta), path(bam) ] + pattern: "*.bam" + - dedup_bam_index: + type: file + description: | + Channel containing deduplicated BAM index files. + Structure: [ val(meta), path(bam.bai) ] + pattern: "*.bai" + - coverage2cytosine_coverage: + type: file + description: | + Channel containing coverage information from coverage2cytosine. + Structure: [ val(meta), path(coverage.txt) ] + pattern: "*.cov.gz" + - coverage2cytosine_report: + type: file + description: | + Channel containing report from coverage2cytosine summarizing cytosine methylation coverage. + Structure: [ val(meta), path(report.txt) ] + pattern: "*report.txt.gz" + - coverage2cytosine_summary: + type: file + description: | + Channel containing summary information from coverage2cytosine. + Structure: [ val(meta), path(summary.txt) ] + pattern: "*cytosine_context_summary.txt" + - methylation_bedgraph: + type: file + description: | + Channel containing methylation data in bedGraph format. + Structure: [ val(meta), path(methylation.bedgraph) ] + pattern: "*.bedGraph.gz" + - methylation_calls: + type: file + description: | + Channel containing methylation call data. + Structure: [ val(meta), path(calls.txt) ] + pattern: "*.txt.gz" + - methylation_coverage: + type: file + description: | + Channel containing methylation coverage data. + Structure: [ val(meta), path(coverage.txt) ] + pattern: "*.cov.gz" + - methylation_report: + type: file + description: | + Channel containing methylation report detailing methylation patterns. + Structure: [ val(meta), path(report.txt) ] + pattern: "*_splitting_report.txt" + - methylation_mbias: + type: file + description: | + Channel containing M-bias report showing methylation bias across read positions. + Structure: [ val(meta), path(mbias.txt) ] + pattern: "*.M-bias.txt" + - bismark_report: + type: file + description: | + Channel containing Bismark report with mapping and methylation statistics. + Structure: [ val(meta), path(bismark_report.txt) ] + pattern: "*report.{html,txt}" + - bismark_summary: + type: file + description: | + Channel containing Bismark summary report. + Structure: [ val(meta), path(bismark_summary.txt) ] + pattern: "*report.{html,txt}" + - multiqc: + type: file + description: | + Channel containing MultiQC report aggregating results across samples. + Structure: [ val(meta), path(multiqc_report.html) ] + pattern: "*.html" + - versions: + type: file + description: | + File containing software versions. + Structure: [ path(versions.yml) ] + pattern: "versions.yml" +authors: + - "@sateeshperi" +maintainers: + - "@sateeshperi" diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test new file mode 100644 index 00000000000..f30c4344449 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test @@ -0,0 +1,336 @@ +nextflow_workflow { + + name "Test Subworkflow FASTQ_ALIGN_DEDUP_BISMARK" + script "../main.nf" + workflow "FASTQ_ALIGN_DEDUP_BISMARK" + config "./nextflow.config" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/fastq_align_dedup_bismark" + tag "bismark/align" + tag "samtools/sort" + tag "samtools/index" + tag "bismark/deduplicate" + tag "bismark/methylationextractor" + tag "bismark/coverage2cytosine" + tag "bismark/report" + tag "bismark/summary" + tag "untar" + + setup { + run("UNTAR", alias: "BOWTIE2") { + script "../../../../modules/nf-core/untar/main.nf" + process { + """ + input[0] = [ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/Bowtie2_Index.tar.gz', checkIfExists: true) + ] + """ + } + } + + run("UNTAR", alias: "HISAT2") { + script "../../../../modules/nf-core/untar/main.nf" + process { + """ + input[0] = [ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/Hisat2_Index.tar.gz', checkIfExists: true) + ] + """ + } + } + } + + test("Params: bismark | default") { + + when { + params { + aligner = "bismark" + cytosine_report = false + skip_deduplication = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BOWTIE2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, + workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark | skip_deduplication") { + + when { + params { + aligner = "bismark" + skip_deduplication = true + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BOWTIE2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, + workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark | cytosine_report") { + + when { + params { + aligner = "bismark" + cytosine_report = true + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = BOWTIE2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, + workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark_hisat | default") { + + when { + params { + aligner = "bismark_hisat" + cytosine_report = false + skip_deduplication = false + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = HISAT2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, + workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark_hisat | skip_deduplication") { + + when { + params { + aligner = "bismark_hisat" + skip_deduplication = true + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = HISAT2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, + workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } + + test("Params: bismark_hisat | cytosine_report") { + + when { + params { + aligner = "bismark_hisat" + cytosine_report = true + } + + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:true ], + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ]) + input[1] = Channel.of([ + [:], + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + ]) + input[2] = HISAT2.out.untar + input[3] = params.skip_deduplication + input[4] = params.cytosine_report + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot( + workflow.out.dedup.collect { bam(it[1]).getReadsMD5() }, + workflow.out.dedup_bam_index.collect { meta, bai -> file(bai).name }, + workflow.out.coverage2cytosine_coverage, + workflow.out.coverage2cytosine_report, + workflow.out.coverage2cytosine_summary, + workflow.out.methylation_bedgraph, + workflow.out.methylation_calls, + workflow.out.methylation_coverage, + workflow.out.methylation_report, + workflow.out.methylation_mbias, + workflow.out.bismark_report.collect { meta, report -> file(report).name }, + workflow.out.bismark_summary[0][1], + workflow.out.multiqc.flatten().collect { path -> file(path).name }, + workflow.out.versions + ).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap new file mode 100644 index 00000000000..b8f199fb126 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/main.nf.test.snap @@ -0,0 +1,612 @@ +{ + "Params: bismark | skip_deduplication": { + "content": [ + [ + + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.bedGraph.gz:md5,6896f135c70e2f4944cb43260a687c6c" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_SRR389222_sub1_bismark_bt2.txt.gz:md5,11ae58ebe6513375468ebe63a521d6e3", + "CHG_OT_SRR389222_sub1_bismark_bt2.txt.gz:md5,d0754aef3f7066881bdee1adf72b4bd1", + "CHH_OB_SRR389222_sub1_bismark_bt2.txt.gz:md5,12e9e4ebf59e6b23fb52f4872f534251", + "CHH_OT_SRR389222_sub1_bismark_bt2.txt.gz:md5,b800b59b0987650ac6eb4d2eb25451ad", + "CpG_OB_SRR389222_sub1_bismark_bt2.txt.gz:md5,21d614f45ecf703117550cb2eebfa3d5", + "CpG_OT_SRR389222_sub1_bismark_bt2.txt.gz:md5,8bbe9103623f502908e81cc573754524" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.bismark.cov.gz:md5,4e31b9a432731c75a604c89cb9d471ad" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2_splitting_report.txt:md5,d1bc9da2811d0f20981d01c3b1f4c925" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.M-bias.txt:md5,d1083f305a444487fe977362a0174159" + ] + ], + [ + "SRR389222_sub1_bismark_bt2_SE_report.html" + ], + "bismark_summary_report.txt:md5,13f7305cf7003f54e0eba22f3590889a", + [ + "SRR389222_sub1_bismark_bt2.M-bias.txt", + "SRR389222_sub1_bismark_bt2_SE_report.html", + "SRR389222_sub1_bismark_bt2_SE_report.txt", + "SRR389222_sub1_bismark_bt2_splitting_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-15T15:19:32.095795684" + }, + "Params: bismark | default": { + "content": [ + [ + + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated.bedGraph.gz:md5,5d7312489e51ca88e1ce1f689db93a64" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,5c177e6f00dfc6e2335a55c6b70efa0c", + "CHG_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,5362d479358f644fe2c17cfe02f310f8", + "CHH_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,9469ac28e4369032b855ce1a794a56a4", + "CHH_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,d7628ac941d3fea97297a8dfcfb7aed1", + "CpG_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,f0404fc1fc18c7ff58e8766b405239ed", + "CpG_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,67d04d843e9e1abc8643bc25b27a9ff5" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated.bismark.cov.gz:md5,eb0d3f96ed97c5d12afeeb49c234849d" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated_splitting_report.txt:md5,ec268cc572b8bcfea0959b47c7510f05" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated.M-bias.txt:md5,290ebfeca4da7809cf9adfe60bab1b29" + ] + ], + [ + "SRR389222_sub1_bismark_bt2_SE_report.html" + ], + "bismark_summary_report.txt:md5,46f54eec58516c027e20bc9bdcaaae14", + [ + "SRR389222_sub1_bismark_bt2.deduplicated.M-bias.txt", + "SRR389222_sub1_bismark_bt2.deduplicated_splitting_report.txt", + "SRR389222_sub1_bismark_bt2.deduplication_report.txt", + "SRR389222_sub1_bismark_bt2_SE_report.html", + "SRR389222_sub1_bismark_bt2_SE_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-15T15:18:51.250132036" + }, + "Params: bismark | cytosine_report": { + "content": [ + [ + + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.CpG_report.txt.gz:md5,33af7ef6af01da4f1e12457ff1d782c6" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.cytosine_context_summary.txt:md5,0e8c65559470bc64c42dbd0278a531de" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated.bedGraph.gz:md5,5d7312489e51ca88e1ce1f689db93a64" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,5c177e6f00dfc6e2335a55c6b70efa0c", + "CHG_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,5362d479358f644fe2c17cfe02f310f8", + "CHH_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,9469ac28e4369032b855ce1a794a56a4", + "CHH_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,d7628ac941d3fea97297a8dfcfb7aed1", + "CpG_OB_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,f0404fc1fc18c7ff58e8766b405239ed", + "CpG_OT_SRR389222_sub1_bismark_bt2.deduplicated.txt.gz:md5,67d04d843e9e1abc8643bc25b27a9ff5" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated.bismark.cov.gz:md5,eb0d3f96ed97c5d12afeeb49c234849d" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated_splitting_report.txt:md5,ec268cc572b8bcfea0959b47c7510f05" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_bt2.deduplicated.M-bias.txt:md5,290ebfeca4da7809cf9adfe60bab1b29" + ] + ], + [ + "SRR389222_sub1_bismark_bt2_SE_report.html" + ], + "bismark_summary_report.txt:md5,46f54eec58516c027e20bc9bdcaaae14", + [ + "SRR389222_sub1_bismark_bt2.deduplicated.M-bias.txt", + "SRR389222_sub1_bismark_bt2.deduplicated_splitting_report.txt", + "SRR389222_sub1_bismark_bt2.deduplication_report.txt", + "SRR389222_sub1_bismark_bt2_SE_report.html", + "SRR389222_sub1_bismark_bt2_SE_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272", + "versions.yml:md5,eb23ca81cb3427b8c5a8a9a93def4882" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-15T15:20:14.052464546" + }, + "Params: bismark_hisat | cytosine_report": { + "content": [ + [ + + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.CpG_report.txt.gz:md5,d05837f07e4866f007d1f25166376b10" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "test.cytosine_context_summary.txt:md5,c2e99c47feb351c6666990bec7d1a9bd" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated.bedGraph.gz:md5,1f1b14f7be535e6098ece13ea1406bfc" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,5c177e6f00dfc6e2335a55c6b70efa0c", + "CHG_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,138bccceb9ae71ecdae6828fd8fc327f", + "CHH_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,9469ac28e4369032b855ce1a794a56a4", + "CHH_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,f1acfc10c9f9ffb80f2a7a947a210ad7", + "CpG_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,f0404fc1fc18c7ff58e8766b405239ed", + "CpG_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,75b9acaf689411154f906bcfa7264abf" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated.bismark.cov.gz:md5,ec4a2575b47822ef478b3bda35c70590" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated_splitting_report.txt:md5,eaee004ea3fffb9b8aaf968f41f9f903" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated.M-bias.txt:md5,82b760a6824f55f6732297374d77a6ec" + ] + ], + [ + "SRR389222_sub1_bismark_hisat2_SE_report.html" + ], + "bismark_summary_report.txt:md5,762c303acee29bfffef35b3d9e04d153", + [ + "SRR389222_sub1_bismark_hisat2.deduplicated.M-bias.txt", + "SRR389222_sub1_bismark_hisat2.deduplicated_splitting_report.txt", + "SRR389222_sub1_bismark_hisat2.deduplication_report.txt", + "SRR389222_sub1_bismark_hisat2_SE_report.html", + "SRR389222_sub1_bismark_hisat2_SE_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272", + "versions.yml:md5,eb23ca81cb3427b8c5a8a9a93def4882" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-15T15:22:14.454171707" + }, + "Params: bismark_hisat | default": { + "content": [ + [ + + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated.bedGraph.gz:md5,1f1b14f7be535e6098ece13ea1406bfc" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,5c177e6f00dfc6e2335a55c6b70efa0c", + "CHG_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,138bccceb9ae71ecdae6828fd8fc327f", + "CHH_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,9469ac28e4369032b855ce1a794a56a4", + "CHH_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,f1acfc10c9f9ffb80f2a7a947a210ad7", + "CpG_OB_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,f0404fc1fc18c7ff58e8766b405239ed", + "CpG_OT_SRR389222_sub1_bismark_hisat2.deduplicated.txt.gz:md5,75b9acaf689411154f906bcfa7264abf" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated.bismark.cov.gz:md5,ec4a2575b47822ef478b3bda35c70590" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated_splitting_report.txt:md5,eaee004ea3fffb9b8aaf968f41f9f903" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.deduplicated.M-bias.txt:md5,82b760a6824f55f6732297374d77a6ec" + ] + ], + [ + "SRR389222_sub1_bismark_hisat2_SE_report.html" + ], + "bismark_summary_report.txt:md5,762c303acee29bfffef35b3d9e04d153", + [ + "SRR389222_sub1_bismark_hisat2.deduplicated.M-bias.txt", + "SRR389222_sub1_bismark_hisat2.deduplicated_splitting_report.txt", + "SRR389222_sub1_bismark_hisat2.deduplication_report.txt", + "SRR389222_sub1_bismark_hisat2_SE_report.html", + "SRR389222_sub1_bismark_hisat2_SE_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,a675051417ba65700b5db32d98aa65b6", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-15T15:20:53.556708293" + }, + "Params: bismark_hisat | skip_deduplication": { + "content": [ + [ + + ], + [ + "test.sorted.bam.bai" + ], + [ + + ], + [ + + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.bedGraph.gz:md5,75be0d8be0134a48fd55e15a49568d3f" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + [ + "CHG_OB_SRR389222_sub1_bismark_hisat2.txt.gz:md5,11ae58ebe6513375468ebe63a521d6e3", + "CHG_OT_SRR389222_sub1_bismark_hisat2.txt.gz:md5,2963a2702be660068818073a2989c790", + "CHH_OB_SRR389222_sub1_bismark_hisat2.txt.gz:md5,12e9e4ebf59e6b23fb52f4872f534251", + "CHH_OT_SRR389222_sub1_bismark_hisat2.txt.gz:md5,e6815f62702d06dc01d12ea18dbd9a03", + "CpG_OB_SRR389222_sub1_bismark_hisat2.txt.gz:md5,21d614f45ecf703117550cb2eebfa3d5", + "CpG_OT_SRR389222_sub1_bismark_hisat2.txt.gz:md5,1a07c82f278b4ed637d33833df1e9749" + ] + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.bismark.cov.gz:md5,ab610e006112a52e5ecb230a1704f236" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2_splitting_report.txt:md5,e6b01d28d7248361750dbd23eef667df" + ] + ], + [ + [ + { + "id": "test", + "single_end": true + }, + "SRR389222_sub1_bismark_hisat2.M-bias.txt:md5,bbfe49763c571a55a50d1e46c0fb4e46" + ] + ], + [ + "SRR389222_sub1_bismark_hisat2_SE_report.html" + ], + "bismark_summary_report.txt:md5,7a466655c11944b6035901910378e559", + [ + "SRR389222_sub1_bismark_hisat2.M-bias.txt", + "SRR389222_sub1_bismark_hisat2_SE_report.html", + "SRR389222_sub1_bismark_hisat2_SE_report.txt", + "SRR389222_sub1_bismark_hisat2_splitting_report.txt", + "bismark_summary_report.html", + "bismark_summary_report.txt" + ], + [ + "versions.yml:md5,46e043d0037e7446544ae1e25b65536c", + "versions.yml:md5,6c516cc03aa69228e503c540ee3794aa", + "versions.yml:md5,6f2f867f534bb0ae0ffbdc52d036afce", + "versions.yml:md5,9b480db277eebeb6265cbd003553c22c", + "versions.yml:md5,aa64d431e6cfc5ffcbd5012a76b0c1a3", + "versions.yml:md5,df1a6bcb24960ee4a22a2cc914c65272" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-15T15:21:33.809590706" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config new file mode 100644 index 00000000000..903bb19fa88 --- /dev/null +++ b/subworkflows/nf-core/fastq_align_dedup_bismark/tests/nextflow.config @@ -0,0 +1,15 @@ +params { + aligner = "bismark" + cytosine_report = false + skip_deduplication = false +} + +process { + withName: 'BISMARK_ALIGN' { + ext.args = { params.aligner == 'bismark_hisat' ? ' --hisat2' : ' --bowtie2' } + } + + withName: 'SAMTOOLS_SORT_DEDUPLICATED' { + ext.prefix = { "${meta.id}.sorted" } + } +} From 7ed0aec1b50be96284fb7526ce79ae54d4f5f2ed Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 16 Nov 2024 13:27:19 +0000 Subject: [PATCH 11/12] chore(deps): pin dependencies (#7004) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/actions/nf-test-action/action.yml | 4 ++-- .github/workflows/test.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 4dd6df48919..e1e1f4bc57d 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -11,9 +11,9 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-java@v4 + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 with: distribution: "temurin" java-version: "17" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0ff73656067..736b2c9eae9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: with: fetch-depth: 2 # To retrieve the preceding commit. - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 id: filter with: filters: "tests/config/pytest_modules.yml" From 8ad72fdac2212c9e84b6187e85b9cefb507d1575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 18 Nov 2024 11:19:41 +0100 Subject: [PATCH 12/12] update test yaml --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 736b2c9eae9..a2ba0b53a62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -662,6 +662,7 @@ jobs: - path: modules/nf-core/parabricks/fq2bammeth - path: modules/nf-core/parabricks/fq2bam - path: modules/nf-core/parabricks/applybqsr + - path: modules/nf-core/parabricks/indexgvcf - profile: conda path: modules/nf-core/xeniumranger/relabel - profile: conda @@ -698,6 +699,8 @@ jobs: profile: [docker_self_hosted, singularity] - path: modules/nf-core/parabricks/fq2bammeth profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/indexgvcf + profile: [docker_self_hosted, singularity] env: NXF_ANSI_LOG: false