POC: Usage for nf-core/references in nf-core/sarek

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ on:
 env:
   NFT_DIFF: "pdiff"
   NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
-  NFT_VER: "0.9.2"
+  NFT_VER: 0.9.2
   NFT_WORKDIR: "~"
   NXF_ANSI_LOG: false
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
@@ -34,8 +34,8 @@ jobs:
       fail-fast: false
       matrix:
         NXF_VER:
-          - "24.04.2"
-          - "latest-everything"
+          - 24.04.2
+          - latest-everything
         filter: ["workflow", "function", "pipeline"]
         # filter: ["process", "workflow", "function", "pipeline"]
         profile: ["conda", "docker", "singularity"]

.gitignore

@@ -13,3 +13,5 @@ null/
 test-datasets/
 test.tap
 test.xml
+.cursorrules
+TODO.md

.nf-core.yml

@@ -4,6 +4,8 @@ lint:
   files_exist:
     - .github/workflows/awsfulltest.yml
     - .github/workflows/awstest.yml
+    - conf/igenomes.config
+    - conf/igenomes_ignored.config
     - conf/modules.config
   files_unchanged:
     - .gitignore

conf/modules/prepare_genome.config

@@ -15,26 +15,6 @@
 
 process {
 
-    withName: 'BWAMEM1_INDEX' {
-        ext.when         = { !params.bwa && params.step == "mapping" && (params.aligner == "bwa-mem" || params.aligner == "sentieon-bwamem")}
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference" },
-            pattern: "bwa",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'BWAMEM2_INDEX' {
-        ext.when         = { !params.bwamem2 && params.step == "mapping" && params.aligner == "bwa-mem2" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference" },
-            pattern: "bwamem2",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
     withName: 'CNVKIT_ANTITARGET' {
         ext.when         = { params.tools && params.tools.split(',').contains('cnvkit') }
         publishDir       = [
@@ -56,106 +36,6 @@ process {
         ]
     }
 
-    withName: 'DRAGMAP_HASHTABLE' {
-        ext.when         = { !params.dragmap && params.step == "mapping" && params.aligner == "dragmap" }
-        publishDir = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference" },
-            pattern: "dragmap",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'GATK4_CREATESEQUENCEDICTIONARY' {
-        ext.when         = { !params.dict && params.step != "annotate" && params.step != "controlfreec" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/dict" },
-            pattern: "*dict",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'MSISENSORPRO_SCAN' {
-        ext.when         = { params.tools && params.tools.split(',').contains('msisensorpro') }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/msi" },
-            pattern: "*list",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'SAMTOOLS_FAIDX' {
-        ext.when         = { !params.fasta_fai && params.step != "annotate" }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/fai" },
-            pattern: "*fai",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'TABIX_BCFTOOLS_ANNOTATIONS' {
-        ext.when         = { !params.bcftools_annotations_tbi && params.bcftools_annotations && params.tools && params.tools.split(',').contains('bcfann') }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/bcfann" },
-            pattern: "*vcf.gz.tbi",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'TABIX_DBSNP' {
-        ext.when         = { !params.dbsnp_tbi && params.dbsnp && ((params.step == "mapping" || params.step == "markduplicates" || params.step == "prepare_recalibration") || params.tools && (params.tools.split(',').contains('controlfreec') || params.tools.split(',').contains('haplotypecaller') || params.tools.split(',').contains('sentieon_haplotyper') || params.tools.split(',').contains('sentieon_dnascope') || params.tools.split(',').contains('mutect2'))) }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/dbsnp" },
-            pattern: "*vcf.gz.tbi",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'TABIX_GERMLINE_RESOURCE' {
-        ext.when         = { !params.germline_resource_tbi && params.germline_resource && params.tools && params.tools.split(',').contains('mutect2') }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/germline_resource" },
-            pattern: "*vcf.gz.tbi",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'TABIX_KNOWN_INDELS' {
-        ext.when         = { !params.known_indels_tbi && params.known_indels && (params.step == 'mapping' || params.step == "markduplicates" || params.step == 'prepare_recalibration' || (params.tools && (params.tools.split(',').contains('haplotypecaller') || params.tools.split(',').contains('sentieon_haplotyper') || params.tools.split(',').contains('sentieon_dnascope'))) ) }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/known_indels" },
-            pattern: "*vcf.gz.tbi",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'TABIX_KNOWN_SNPS' {
-        ext.when         = { !params.known_snps_tbi && params.known_snps && (params.step == 'mapping' || params.step == "markduplicates" || params.step == 'prepare_recalibration' || (params.tools && (params.tools.split(',').contains('haplotypecaller') || params.tools.split(',').contains('sentieon_haplotyper') )) ) }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/known_snps" },
-            pattern: "*vcf.gz.tbi",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
-    withName: 'TABIX_PON' {
-        ext.when         = { !params.pon_tbi && params.pon && params.tools && params.tools.split(',').contains('mutect2') }
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference/pon" },
-            pattern: "*vcf.gz.tbi",
-            saveAs: { params.save_reference || params.build_only_index ? it : null }
-        ]
-    }
-
     withName: 'UNZIP_ALLELES|UNZIP_LOCI|UNZIP_GC|UNZIP_RT' {
         ext.when         = { params.tools && params.tools.split(',').contains('ascat')}
         publishDir       = [

conf/modules/prepare_intervals.config

@@ -14,37 +14,14 @@
 // PREPARE INTERVALS
 
 process {
-
-    withName: 'BUILD_INTERVALS' {
-        ext.args   = { "-v FS='\t' -v OFS='\t' '{ print \$1, \"0\", \$2 }'" }
-        ext.suffix = { "bed" }
-    }
-
-    withName: 'CREATE_INTERVALS_BED' {
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference" },
-            pattern: "*bed",
-            saveAs: { params.save_reference || params.build_only_index ? "intervals/${it}" : null }
-        ]
-    }
-
-    withName: 'GATK4_INTERVALLISTTOBED' {
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference" },
-            pattern: "*bed",
-            saveAs: { params.save_reference || params.build_only_index ? "intervals/${it}" : null }
+    withName: 'NFCORE_SAREK:PREPARE_INTERVALS:PREPARE_INTERVALS:.*' {
+        publishDir = [
+            //specify to avoid publishing, overwritten otherwise
+            enabled: false
         ]
     }
 
     withName: 'TABIX_BGZIPTABIX_INTERVAL_SPLIT|TABIX_BGZIPTABIX_INTERVAL_COMBINED' {
         ext.prefix       = {"${meta.id}"}
-        publishDir       = [
-            mode: params.publish_dir_mode,
-            path: { "${params.outdir}/reference" },
-            pattern: "*bed.gz",
-            saveAs: { params.save_reference || params.build_only_index ? "intervals/${it}" : null }
-        ]
     }
 }

conf/test.config

@@ -27,16 +27,16 @@ params {
     // Input data
     input = "${projectDir}/tests/csv/3.0/fastq_single.csv"
 
-    // small genome on igenomes
-    igenomes_base   = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/'
-    genome          = 'testdata.nf-core.sarek'
+    // small reference genome
+    igenomes_base = 's3://nf-core-references/test_data/'
+
+    // TODO: VERSION
+    references = 'https://raw.githubusercontent.com/nf-core/references-assets/main/genomes/Homo_sapiens/test/GRCh38_chr22.yml'
 
     // Small reference genome
     bcftools_annotations     = "${params.modules_testdata_base_path}/genomics/sarscov2/illumina/vcf/test2.vcf.gz"
     bcftools_annotations_tbi = "${params.modules_testdata_base_path}/genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi"
     bcftools_header_lines    = "${projectDir}/tests/config/bcfann_test_header.txt"
-    snpeff_cache             = null
-    vep_cache                = null
 
     // Sentieon
     sentieon_dnascope_model    = "s3://ngi-igenomes/igenomes/Homo_sapiens/GATK/GRCh38/Annotation/Sentieon/SentieonDNAscopeModel1.1.model"