Merge branch 'dev' into nf_validation

nf-core · Aug 2, 2023 · 7ac16f8 · 7ac16f8
2 parents 043e260 + 256d002
commit 7ac16f8
Show file tree

Hide file tree

Showing 64 changed files with 690 additions and 519 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,16 +11,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - [#1130](https://github.com/nf-core/sarek/pull/1130) - Back to dev
 - [#1013](https://github.com/nf-core/sarek/pull/1013) - Mutect2 multi sample mode with `--joint_mutect2`
+- [#1153](https://github.com/nf-core/sarek/pull/1153) - Add input validation for Sentieon & FGBio UMI incompatibility
+- [#1158](https://github.com/nf-core/sarek/pull/1158) - Add preprint
+- [#1159](https://github.com/nf-core/sarek/pull/1159) - ISMB Poster
 
 ### Changed
 
+- [#1151](https://github.com/nf-core/sarek/pull/1151) - Refactor codebase
+- [#1157](https://github.com/nf-core/sarek/pull/1157) - Move all vep args from `ext.args` to `params.vep_custom_args` to allow easier modifications
 - [#1059](https://github.com/nf-core/sarek/pull/1059) - Add `nf-validation` for samplesheet validation
+- [#1160](https://github.com/nf-core/sarek/pull/1160) - Updating tiddit to v3.6.1
 
 ### Fixed
 
 - [#1143](https://github.com/nf-core/sarek/pull/1143) - `snpeff_db` is now a string
 - [#1145](https://github.com/nf-core/sarek/pull/1145) - Fixed Zenodo links in `README.md` and in `WorkflowMain.groovy`
 - [#1149](https://github.com/nf-core/sarek/pull/1149) - Update `Manta` modules and fix usage of `--exome` flag
+- [#1155](https://github.com/nf-core/sarek/pull/1155) - Restore proper rendering in `usage.md`
+- [#1163](https://github.com/nf-core/sarek/pull/1163) - Correcting location of output folder for joint variant calling with GATK's haplotypecaller
 
 ## [3.2.3](https://github.com/nf-core/sarek/releases/tag/3.2.3) - Gällivare
 

diff --git a/README.md b/README.md
@@ -34,7 +34,7 @@ It's listed on [Elixir - Tools and Data Services Registry](https://bio.tools/nf-
 Depending on the options and samples provided, the pipeline can currently perform the following:
 
 - Form consensus reads from UMI sequences (`fgbio`)
-- Sequencing quality control and trimming (`FastQC`, `fastp`)
+- Sequencing quality control and trimming (enabled by `--trim_fastq`) (`FastQC`, `fastp`)
 - Map Reads to Reference (`BWA-mem`, `BWA-mem2`, `dragmap` or `Sentieon BWA-mem`)
 - Process BAM file (`GATK MarkDuplicates`, `GATK BaseRecalibrator` and `GATK ApplyBQSR` or `Sentieon LocusCollector` and `Sentieon Dedup`)
 - Summarise alignment statistics (`samtools stats`, `mosdepth`)
@@ -180,6 +180,8 @@ For further information or help, don't hesitate to get in touch on the [Slack `#
 
 If you use `nf-core/sarek` for your analysis, please cite the `Sarek` article as follows:
 
+> Friederike Hanssen, Maxime U Garcia, Lasse Folkersen, Anders Sune Pedersen, Francesco Lescai, Susanne Jodoin, Edmund Miller, Oskar Wacker, Nicholas Smith, nf-core community, Gisela Gabernet, Sven Nahnsen **Scalable and efficient DNA sequencing analysis on different compute infrastructures aiding variant discovery** _bioRxiv_ [doi: 10.1101/2023.07.19.549462](https://doi.org/10.1101/2023.07.19.549462).
+
 > Garcia M, Juhos S, Larsson M et al. **Sarek: A portable workflow for whole-genome sequencing analysis of germline and somatic variants [version 2; peer review: 2 approved]** _F1000Research_ 2020, 9:63 [doi: 10.12688/f1000research.16665.2](http://dx.doi.org/10.12688/f1000research.16665.2).
 
 You can cite the sarek zenodo record for a specific version using the following [doi: 10.5281/zenodo.3476425](https://doi.org/10.5281/zenodo.3476425)

diff --git a/conf/modules/aligner.config b/conf/modules/aligner.config
@@ -33,13 +33,7 @@ process {
             ext.when         = { params.aligner == "sentieon-bwamem" }
         }
 
-
-
-        withName: "(BWAMEM.*_MEM|DRAGMAP_ALIGN)" {
-            // Markduplicates Spark NEEDS name-sorted reads or runtime goes through the roof
-            // However if it's skipped, reads need to be coordinate-sorted
-            // Only name sort if Spark for Markduplicates + duplicate marking is not skipped
-            ext.args2        = { params.use_gatk_spark && params.use_gatk_spark.contains('markduplicates') && (!params.skip_tools || (params.skip_tools && !params.skip_tools.split(',').contains('markduplicates'))) ? '-n' : '' }
+        withName: "(BWAMEM.*_MEM|DRAGMAP_ALIGN|SENTIEON_BWAMEM)" {
             ext.prefix       = { params.split_fastq > 1 ? "${meta.id}".concat('.').concat(reads.get(0).name.tokenize('.')[0]) : "${meta.id}.sorted" }
             publishDir       = [
                 mode: params.publish_dir_mode,
@@ -61,29 +55,12 @@ process {
             ]
         }
 
-
-        withName: "SENTIEON_BWAMEM" {
-            // Markduplicates Spark NEEDS name-sorted reads or runtime goes through the roof.
-            // However, currently SENTIEON_BWAMEM only supports coordinate sorting the reads.
-            ext.prefix       = { params.split_fastq > 1 ? "${meta.id}".concat('.').concat(reads.get(0).name.tokenize('.')[0]) : "${meta.id}.sorted" }
-            publishDir       = [
-                mode: params.publish_dir_mode,
-                path: { "${params.outdir}/preprocessing/" },
-                pattern: "*bam",
-                // Only save if save_output_as_bam AND
-                // (save_mapped OR no_markduplicates OR sentieon_dedup) AND
-                // only a single BAM file per sample
-                saveAs: {
-                    if (params.save_output_as_bam &&
-                        (
-                            params.save_mapped ||
-                            (params.skip_tools && params.skip_tools.split(',').contains('markduplicates')) &&
-                            !(params.tools && params.tools.split(',').contains('sentieon_dedup'))
-                        ) && (meta.size * meta.num_lanes == 1)
-                    ) { "mapped/${meta.id}/${it}" }
-                    else { null }
-                }
-            ]
+        withName: "(BWAMEM.*_MEM|DRAGMAP_ALIGN)" {
+            // Markduplicates Spark NEEDS name-sorted reads or runtime goes through the roof
+            // However if it's skipped, reads need to be coordinate-sorted
+            // Only name sort if Spark for Markduplicates + duplicate marking is not skipped
+            // Currently SENTIEON_BWAMEM only supports coordinate sorting the reads.
+            ext.args2        = { params.use_gatk_spark && params.use_gatk_spark.contains('markduplicates') && (!params.skip_tools || (params.skip_tools && !params.skip_tools.split(',').contains('markduplicates'))) ? '-n' : '' }
         }
 
         withName: "BWAMEM.*_MEM|SENTIEON_BWAMEM" {

diff --git a/conf/modules/annotate.config b/conf/modules/annotate.config
@@ -36,7 +36,6 @@ process {
     if (params.tools && (params.tools.split(',').contains('vep') || params.tools.split(',').contains('merge'))) {
         withName: 'ENSEMBLVEP_VEP' {
             ext.args         = { [
-                '--everything --filter_common --per_gene --total_length --offline --format vcf',
                 (params.vep_dbnsfp && params.dbnsfp && !params.dbnsfp_consequence)    ? "--plugin dbNSFP,${params.dbnsfp.split("/")[-1]},${params.dbnsfp_fields}"                                              : '',
                 (params.vep_dbnsfp && params.dbnsfp && params.dbnsfp_consequence)     ? "--plugin dbNSFP,'consequence=${params.dbnsfp_consequence}',${params.dbnsfp.split("/")[-1]},${params.dbnsfp_fields}"   : '',
                 (params.vep_loftee)                                                   ? "--plugin LoF,loftee_path:/opt/conda/envs/nf-core-vep-${params.vep_version}/share/ensembl-vep-${params.vep_version}-0" : '',

diff --git a/conf/modules/markduplicates.config b/conf/modules/markduplicates.config
@@ -33,6 +33,16 @@ process {
         ]
     }
 
+    withName: 'NFCORE_SAREK:SAREK:(BAM_MARKDUPLICATES|BAM_MARKDUPLICATES_SPARK):CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
+        ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
+        ext.prefix       = { "${meta.id}.md.cram" }
+        publishDir       = [
+            mode: params.publish_dir_mode,
+            path: { "${params.outdir}/reports/samtools/${meta.id}" },
+            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+        ]
+    }
+
     withName: 'BAM_TO_CRAM_MAPPING' {
         // Run only when mapping should be saved as CRAM or when no MD is done
         ext.when         = (params.save_mapped && !params.save_output_as_bam) ||

diff --git a/conf/modules/modules.config b/conf/modules/modules.config
@@ -40,26 +40,6 @@ process {
         ]
     }
 
-    withName: 'NFCORE_SAREK:SAREK:(BAM_MARKDUPLICATES|BAM_MARKDUPLICATES_SPARK):CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
-        ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
-        ext.prefix       = { "${meta.id}.md.cram" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reports/samtools/${meta.id}" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
-    withName: 'NFCORE_SAREK:SAREK:BAM_SENTIEON_DEDUP:CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
-        ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
-        ext.prefix       = { "${meta.id}.dedup.cram" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reports/samtools/${meta.id}" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
-        ]
-    }
-
     withName: 'NFCORE_SAREK:SAREK:CRAM_QC_NO_MD:SAMTOOLS_STATS' {
         ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
         ext.prefix       = { "${meta.id}.sorted.cram" }

diff --git a/conf/modules/mutect2.config b/conf/modules/mutect2.config
@@ -15,6 +15,7 @@
 
 process {
     if (params.tools && params.tools.split(',').contains('mutect2')) {
+
         withName: 'GATK4_MUTECT2' {
             ext.prefix       = { meta.num_intervals <= 1 ? "${meta.id}.mutect2" : "${meta.id}.mutect2.${intervals.simpleName}" }
             ext.when         = { params.tools && params.tools.split(',').contains('mutect2') }

diff --git a/conf/modules/post_variant_calling.config b/conf/modules/post_variant_calling.config
@@ -15,7 +15,9 @@
 // Like, for instance, concatenating the unannotated, germline vcf-files
 
 process {
+
     withName: 'GERMLINE_VCFS_CONCAT'{
+        ext.when         =  params.concatenate_vcfs
         publishDir       = [
             //specify to avoid publishing, overwritten otherwise
             enabled: false
@@ -24,6 +26,7 @@ process {
 
     withName: 'GERMLINE_VCFS_CONCAT_SORT'{
         ext.prefix       = { "${meta.id}.germline" }
+        ext.when         =  params.concatenate_vcfs
         publishDir       = [
             mode: params.publish_dir_mode,
                 path: { "${params.outdir}/variant_calling/concat/${meta.id}/" }
@@ -32,10 +35,12 @@ process {
 
     withName: 'TABIX_EXT_VCF' {
         ext.prefix       = { "${input.baseName}" }
+        ext.when         =  params.concatenate_vcfs
     }
 
     withName: 'TABIX_GERMLINE_VCFS_CONCAT_SORT'{
         ext.prefix       = { "${meta.id}.germline" }
+        ext.when         =  params.concatenate_vcfs
         publishDir       = [
             mode: params.publish_dir_mode,
                 path: { "${params.outdir}/variant_calling/concat/${meta.id}/" }

diff --git a/conf/modules/prepare_cache.config b/conf/modules/prepare_cache.config
@@ -14,6 +14,7 @@
 // PREPARE_CACHE
 
 process {
+
     // SNPEFF
     withName: 'SNPEFF_DOWNLOAD' {
         ext.when         = { params.tools && (params.tools.split(',').contains('snpeff') || params.tools.split(',').contains('merge')) }

diff --git a/conf/modules/sentieon_dedup.config b/conf/modules/sentieon_dedup.config
@@ -34,4 +34,16 @@ process {
         ]
     }
 
+    if (params.tools && params.tools.contains('sentieon_dedup')) {
+        withName: 'NFCORE_SAREK:SAREK:BAM_SENTIEON_DEDUP:CRAM_QC_MOSDEPTH_SAMTOOLS:SAMTOOLS_STATS' {
+            ext.when         = { !(params.skip_tools && params.skip_tools.split(',').contains('samtools')) }
+            ext.prefix       = { "${meta.id}.dedup.cram" }
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/reports/samtools/${meta.id}" },
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
+            ]
+        }
+    }
+
 }
diff --git a/conf/modules/sentieon_haplotyper.config b/conf/modules/sentieon_haplotyper.config
@@ -45,14 +45,16 @@ process {
         ]
     }
 
-    withName: '.*BAM_VARIANT_CALLING_SENTIEON_HAPLOTYPER:VCF_VARIANT_FILTERING_GATK:FILTERVARIANTTRANCHES' {
-        ext.prefix       = {"${meta.id}.haplotyper"}
-        ext.args         = { "--info-key CNN_1D" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/sentieon_haplotyper/${meta.id}/"},
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
-        ]
+    if (params.tools && params.tools.contains('sentieon_haplotyper')) {
+        withName: '.*BAM_VARIANT_CALLING_SENTIEON_HAPLOTYPER:VCF_VARIANT_FILTERING_GATK:FILTERVARIANTTRANCHES' {
+            ext.prefix       = {"${meta.id}.haplotyper"}
+            ext.args         = { "--info-key CNN_1D" }
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/variant_calling/sentieon_haplotyper/${meta.id}/"},
+                pattern: "*{vcf.gz,vcf.gz.tbi}"
+            ]
+        }
     }
 
 }
diff --git a/conf/modules/sentieon_joint_germline.config b/conf/modules/sentieon_joint_germline.config
@@ -15,7 +15,7 @@
 
 process {
 
-    withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:SENTIEON_GVCFTYPER' {
+    withName: 'SENTIEON_GVCFTYPER' {
         ext.args         = { "--allow-old-rms-mapping-quality-annotation-data" }
         ext.prefix       = { meta.intervals_name }
         publishDir       = [
@@ -24,32 +24,32 @@ process {
     }
 
     if (params.tools && params.tools.contains('sentieon_haplotyper') && params.joint_germline) {
-        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON::BCFTOOLS_SORT' {
+        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:BCFTOOLS_SORT' {
             ext.prefix       = { vcf.baseName - ".vcf" + ".sort" }
             publishDir       = [
                 enabled: false
             ]
         }
-    }
 
-    withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_GENOTYPEGVCFS' {
-        ext.prefix       = "joint_germline"
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/" },
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
-        ]
-    }
+        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_GENOTYPEGVCFS' {
+            ext.prefix       = "joint_germline"
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/" },
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
+                pattern: "*{vcf.gz,vcf.gz.tbi}"
+            ]
+        }
 
-    withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_VQSR' {
-        ext.prefix       = "joint_germline_recalibrated"
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/"},
-            saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
-            pattern: "*{vcf.gz,vcf.gz.tbi}"
-        ]
+        withName: 'NFCORE_SAREK:SAREK:BAM_VARIANT_CALLING_GERMLINE_ALL:BAM_JOINT_CALLING_GERMLINE_SENTIEON:MERGE_VQSR' {
+            ext.prefix       = "joint_germline_recalibrated"
+            publishDir       = [
+                mode: params.publish_dir_mode,
+                path: { "${params.outdir}/variant_calling/sentieon_haplotyper/joint_variant_calling/"},
+                saveAs: { filename -> filename.equals('versions.yml') ? null : filename },
+                pattern: "*{vcf.gz,vcf.gz.tbi}"
+            ]
+        }
     }
 
     withName: 'SENTIEON_VARCAL_INDEL' {
@@ -78,5 +78,4 @@ process {
         ext.args      = '--sensitivity 99.9 --var_type SNP'
     }
 
-
 }
diff --git a/conf/modules/umi.config b/conf/modules/umi.config
@@ -62,6 +62,7 @@ process {
             enabled: false
         ]
     }
+
     withName: 'GROUPREADSBYUMI' {
         publishDir       = [
             [   path: { "${params.outdir}/reports/umi/" },

diff --git a/conf/test.config b/conf/test.config
@@ -46,6 +46,7 @@ params {
 }
 
 process {
+
     withName:'.*:FREEC_SOMATIC'{
         ext.args = {
             [
@@ -69,27 +70,26 @@ process {
         }
     }
 
-    if (params.tools && params.tools.split(',').contains('mutect2')) {
-        if (params.joint_mutect2) {
-            withName: 'MUTECT2_PAIRED' {
-                ext.args         = { params.ignore_soft_clipped_bases ?
-                                        "--dont-use-soft-clipped-bases true --f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" :
-                                        "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" }
-            }
+    if (params.joint_mutect2) {
+        withName: 'MUTECT2_PAIRED' {
+            ext.args         = { params.ignore_soft_clipped_bases ?
+                                    "--dont-use-soft-clipped-bases true --f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" :
+                                    "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample ${meta.normal_id}" }
         }
-        else {
-            withName: '.*MUTECT2_PAIRED'{
-                //sample name from when the test data was generated
-                ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
-            }
+    }
+    else {
+        withName: 'MUTECT2_PAIRED'{
+            //sample name from when the test data was generated
+            ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
         }
     }
 
-    withName: '.*:FILTERVARIANTTRANCHES'{
+    withName: 'FILTERVARIANTTRANCHES'{
         ext.args = { "--info-key CNN_1D --indel-tranche 0" }
     }
 }
 
+
 // Enable container engines/virtualisation envs for CI testing
 // only works when specified with the profile ENV
 // otherwise tests can be done with the regular provided profiles

diff --git a/conf/test/cache.config b/conf/test/cache.config
@@ -63,6 +63,7 @@ process {
         ext.sentieon_auth_data_base64 = secrets.SENTIEON_AUTH_DATA_BASE64
     }
 
+    // This must contain .* in order to properly overwrite the standard config in test cases
     withName:'.*:FREEC_SOMATIC'{
         ext.args = {
             [
@@ -86,14 +87,12 @@ process {
         }
     }
 
-    if (params.tools && params.tools.split(',').contains('mutect2')) {
-        withName: '.*MUTECT2_PAIRED'{
-            //sample name from when the test data was generated
-            ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
-        }
+    withName: 'MUTECT2_PAIRED'{
+        //sample name from when the test data was generated
+        ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
     }
 
-    withName: '.*:FILTERVARIANTTRANCHES'{
+    withName: 'FILTERVARIANTTRANCHES'{
         ext.args = { "--info-key CNN_1D --indel-tranche 0" }
     }
 }

diff --git a/docs/images/sarek_subway.png b/docs/images/sarek_subway.png