From d779da3d6521a0f7c262cc79b34fd8a97b43f46b Mon Sep 17 00:00:00 2001 From: a-price Date: Thu, 20 Apr 2023 14:23:11 +0000 Subject: [PATCH 01/18] update container images afanc version 0.10.0-alpha --- .github/workflows/build-push-quay.yml | 1 + ....clockwork-0.9.6 => Dockerfile.clockwork-0.9.7} | 0 ...essing-0.9.6 => Dockerfile.preprocessing-0.9.7} | 12 +++++++++--- ...cfpredict-0.9.6 => Dockerfile.vcfpredict-0.9.7} | 0 ...clockwork-0.9.6 => Singularity.clockwork-0.9.7} | 0 ...ssing-0.9.6 => Singularity.preprocessing-0.9.7} | 14 +++++++++++--- ...fpredict-0.9.6 => Singularity.vcfpredict-0.9.7} | 0 7 files changed, 21 insertions(+), 6 deletions(-) rename docker/{Dockerfile.clockwork-0.9.6 => Dockerfile.clockwork-0.9.7} (100%) rename docker/{Dockerfile.preprocessing-0.9.6 => Dockerfile.preprocessing-0.9.7} (93%) rename docker/{Dockerfile.vcfpredict-0.9.6 => Dockerfile.vcfpredict-0.9.7} (100%) rename singularity/{Singularity.clockwork-0.9.6 => Singularity.clockwork-0.9.7} (100%) rename singularity/{Singularity.preprocessing-0.9.6 => Singularity.preprocessing-0.9.7} (94%) rename singularity/{Singularity.vcfpredict-0.9.6 => Singularity.vcfpredict-0.9.7} (100%) diff --git a/.github/workflows/build-push-quay.yml b/.github/workflows/build-push-quay.yml index 7ec270d..38dab37 100644 --- a/.github/workflows/build-push-quay.yml +++ b/.github/workflows/build-push-quay.yml @@ -4,6 +4,7 @@ on: branches: - main - v0.9.6 + - 0.9.7-dev paths: - '**/Dockerfile*' diff --git a/docker/Dockerfile.clockwork-0.9.6 b/docker/Dockerfile.clockwork-0.9.7 similarity index 100% rename from docker/Dockerfile.clockwork-0.9.6 rename to docker/Dockerfile.clockwork-0.9.7 diff --git a/docker/Dockerfile.preprocessing-0.9.6 b/docker/Dockerfile.preprocessing-0.9.7 similarity index 93% rename from docker/Dockerfile.preprocessing-0.9.6 rename to docker/Dockerfile.preprocessing-0.9.7 index 0f31247..eb55023 100644 --- a/docker/Dockerfile.preprocessing-0.9.6 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -12,10 +12,11 @@ fastp_version=0.20.1 \ fastqc_version=0.11.9 \ fqtools_version=2.3 \ kraken2_version=2.1.1 \ -afanc_version=0.9.2 \ +afanc_version=0.10.0 \ mykrobe_version=0.12.1 \ bwa_version=0.7.17 \ -mash_version=2.3 +mash_version=2.3 \ +fastani_version=1.33 ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \ PYTHON="python3 python3-pip python3-dev" \ @@ -90,7 +91,12 @@ RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_vers && cd .. \ && curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ && mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ -&& rm -r mash-Linux* +&& rm -r mash-Linux* \ +&& curl -fsSL "https://github.com/ParBLiSS/FastANI/archive/refs/tags/v${fastani_version}.tar.gz" | tar -xz \ +&& cd FastANI-${fastani_version} \ +&& ./bootstrap.sh \ +&& ./configure --prefix=/usr/local \ +&& cd .. RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ && mkdir -p /opt/edirect \ diff --git a/docker/Dockerfile.vcfpredict-0.9.6 b/docker/Dockerfile.vcfpredict-0.9.7 similarity index 100% rename from docker/Dockerfile.vcfpredict-0.9.6 rename to docker/Dockerfile.vcfpredict-0.9.7 diff --git a/singularity/Singularity.clockwork-0.9.6 b/singularity/Singularity.clockwork-0.9.7 similarity index 100% rename from singularity/Singularity.clockwork-0.9.6 rename to singularity/Singularity.clockwork-0.9.7 diff --git a/singularity/Singularity.preprocessing-0.9.6 b/singularity/Singularity.preprocessing-0.9.7 similarity index 94% rename from singularity/Singularity.preprocessing-0.9.6 rename to singularity/Singularity.preprocessing-0.9.7 index 3e92a4e..ba1a73a 100644 --- a/singularity/Singularity.preprocessing-0.9.6 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -16,10 +16,11 @@ fastp_version=0.20.1 fastqc_version=0.11.9 fqtools_version=2.3 kraken2_version=2.1.1 -afanc_version=0.9.2 +afanc_version=0.10.0 mykrobe_version=0.12.1 bwa_version=0.7.17 mash_version=2.3 +fastani_version=1.33 PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" PYTHON="python3 python3-pip python3-dev" @@ -94,7 +95,13 @@ curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version} && cd .. \ && curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ && mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ -&& rm -r mash-Linux* +&& rm -r mash-Linux* \ +&& curl -fsSL "https://github.com/ParBLiSS/FastANI/archive/refs/tags/v${fastani_version}.tar.gz" | tar -xz \ +&& cd FastANI-${fastani_version} \ +&& ./bootstrap.sh \ +&& ./configure --prefix=/usr/local \ +&& cd .. + sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ && mkdir -p /opt/edirect \ @@ -131,10 +138,11 @@ export fastp_version=0.20.1 export fastqc_version=0.11.9 export fqtools_version=2.3 export kraken2_version=2.1.1 -export afanc_version=0.9.2 +export afanc_version=0.10.0 export mykrobe_version=0.12.1 export bwa_version=0.7.17 export mash_version=2.3 +export fastani_version=1.33 export PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" export PYTHON="python3 python3-pip python3-dev" export PYTHON_PACKAGES="biopython" diff --git a/singularity/Singularity.vcfpredict-0.9.6 b/singularity/Singularity.vcfpredict-0.9.7 similarity index 100% rename from singularity/Singularity.vcfpredict-0.9.6 rename to singularity/Singularity.vcfpredict-0.9.7 From 023782f0a0d6466031fa36b5f61be46160fc126d Mon Sep 17 00:00:00 2001 From: a-price Date: Thu, 20 Apr 2023 15:02:21 +0000 Subject: [PATCH 02/18] add GNU Scientific Library --- docker/Dockerfile.preprocessing-0.9.7 | 2 +- singularity/Singularity.preprocessing-0.9.7 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index eb55023..2331cd7 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -18,7 +18,7 @@ bwa_version=0.7.17 \ mash_version=2.3 \ fastani_version=1.33 -ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \ +ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \ PYTHON="python3 python3-pip python3-dev" \ PYTHON_PACKAGES="biopython" diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index ba1a73a..81e18f8 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -22,7 +22,7 @@ bwa_version=0.7.17 mash_version=2.3 fastani_version=1.33 -PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" +PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" PYTHON="python3 python3-pip python3-dev" PYTHON_PACKAGES="biopython" @@ -143,7 +143,7 @@ export mykrobe_version=0.12.1 export bwa_version=0.7.17 export mash_version=2.3 export fastani_version=1.33 -export PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" +export PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" export PYTHON="python3 python3-pip python3-dev" export PYTHON_PACKAGES="biopython" export PATH=${PATH}:/usr/local/bin/mccortex/bin:/usr/local/bin/bwa-${bwa_version}:/opt/edirect From 3982da7ab84d494e987b46edb6743306e88d3283 Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 22 May 2023 14:33:18 +0000 Subject: [PATCH 03/18] update workflow so afanc runs for poor quality samples --- modules/preprocessingModules.nf | 39 +++++++++++++++++++++++---------- workflows/preprocessing.nf | 2 +- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/modules/preprocessingModules.nf b/modules/preprocessingModules.nf index 860d827..4a4a66d 100644 --- a/modules/preprocessingModules.nf +++ b/modules/preprocessingModules.nf @@ -253,7 +253,7 @@ process fastQC { process kraken2 { /** - * @QCcheckpoint only pass to Afanc if Kraken's top family classification is Mycobacteriaceae + * @QCcheckpoint if Kraken's top family classification is NOT Mycobacteriaceae, sample will not proceed further than afanc */ tag { sample_name } @@ -301,7 +301,7 @@ process kraken2 { run_afanc=\$(jq '.afanc' ${kraken2_json}) - if [ \$run_afanc == '\"true\"' ]; then printf "${sample_name}"; else echo '{"error":"Kraken's top family hit either wasn't Mycobacteriaceae, or there were < 100k Mycobacteriaceae reads"}' | jq '.' > ${error_log} && printf "no" && jq -s ".[0] * .[1]" ${software_json} ${error_log} > ${report_json}; fi + if [ \$run_afanc == '\"true\"' ]; then printf "${sample_name}"; else echo '{"error":"Kraken's top family hit either wasn't Mycobacteriaceae, or there were < 100k Mycobacteriaceae reads. Sample will not proceed further than afanc."}' | jq '.' > ${error_log} && printf "no" && jq -s ".[0] * .[1]" ${software_json} ${error_log} > ${report_json}; fi """ stub: @@ -334,26 +334,41 @@ process afanc { label 'medium_memory' label 'retryAfanc' - publishDir "${params.output_dir}/$sample_name/speciation_reports_for_reads_postFastP", mode: 'copy', pattern: '*.json' + errorStrategy 'ignore' + + publishDir "${params.output_dir}/$sample_name/speciation_reports_for_reads_postFastP", mode: 'copy', pattern: '*_afanc_report.json' + publishDir "${params.output_dir}/$sample_name", mode: 'copy', overwrite: 'true', pattern: '*{_err.json,_report.json}' input: - tuple val(sample_name), path(fq1), path(fq2), val(run_afanc), path(software_json) + tuple val(sample_name), path(fq1), path(fq2), val(run_afanc), path(software_json), path(kraken_report), path(kraken_json) path(afanc_myco_db) - when: - run_afanc =~ /${sample_name}/ - output: tuple val(sample_name), path("${sample_name}_afanc_report.json"), stdout, emit: afanc_report + path "${sample_name}_err.json", emit: afanc_log optional true + path "${sample_name}_report.json", emit: afanc_report optional true // tuple val(sample_name), path(fq1), path(fq2), stdout, emit: afanc_fqs script: afanc_report = "${sample_name}_afanc_report.json" + error_log = "${sample_name}_err.json" + report_json = "${sample_name}_report.json" """ - afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 5.0 -n 1000 -o ${sample_name} -t ${task.cpus} - python3 ${baseDir}/bin/reformat_afanc_json.py ${sample_name}/${sample_name}.json - printf ${sample_name} + if [[ ${run_afanc} =~ /${sample_name}/ ]] + then + afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 5.0 -n 1000 -o ${sample_name} -t ${task.cpus} + python3 ${baseDir}/bin/reformat_afanc_json.py ${sample_name}/${sample_name}.json + printf ${sample_name} + else + afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 2.0 -n 500 -o ${sample_name} -t ${task.cpus} + python3 ${baseDir}/bin/reformat_afanc_json.py ${sample_name}/${sample_name}.json + + python3 ${baseDir}/bin/identify_tophit_and_contaminants2.py ${afanc_report} ${kraken_json} ${baseDir}/resources/assembly_summary_refseq.txt ${params.species} ${params.unmix_myco} ${baseDir}/resources null + + echo '{"error":"Kraken's top family hit either wasn't Mycobacteriaceae, or there were < 100k Mycobacteriaceae reads. Sample will not proceed further than afanc."}' | jq '.' > ${error_log} && printf "no" && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json} + fi + """ stub: @@ -490,7 +505,7 @@ process identifyBacterialContaminants { if [ \$contam_to_remove == 'yes' ]; then cp ${sample_name}_species_in_sample.json ${sample_name}_species_in_sample_previous.json; fi - if [ \$contam_to_remove == 'yes' ]; then printf "NOW_DECONTAMINATE_${sample_name}"; elif [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'yes' ]; then printf "NOW_ALIGN_TO_REF_${sample_name}" && mv $fq1 ${sample_name}_nocontam_1.fq.gz && mv $fq2 ${sample_name}_nocontam_2.fq.gz; elif [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'no' ]; then jq -n --arg key "\$top_hit" '{"error": ("top hit " + \$key + " is not one of the 10 accepted mycobacteria")}' > ${error_log} && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json}; fi + if [ \$contam_to_remove == 'yes' ]; then printf "NOW_DECONTAMINATE_${sample_name}"; elif [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'yes' ]; then printf "NOW_ALIGN_TO_REF_${sample_name}" && mv $fq1 ${sample_name}_nocontam_1.fq.gz && mv $fq2 ${sample_name}_nocontam_2.fq.gz; elif [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'no' ]; then jq -n --arg key "\$top_hit" '{"error": ("top hit " + \$key + " does not have a reference genome. Sample will not proceed beyond preprocessing workflow.")}' > ${error_log} && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json}; fi """ stub: @@ -759,7 +774,7 @@ process summarise { if [ \$contam_to_remove == 'yes' ]; then echo '{"error":"sample remains contaminated, even after attempting to resolve this"}' | jq '.' > ${error_log} && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json}; fi - if [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'yes' ]; then printf "NOW_ALIGN_TO_REF_${sample_name}"; elif [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'no' ]; then jq -n --arg key "\$top_hit" '{"error": ("top hit " + \$key + " is not one of the 10 accepted mycobacteria")}' > ${error_log} && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json}; fi + if [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'yes' ]; then printf "NOW_ALIGN_TO_REF_${sample_name}"; elif [ \$contam_to_remove == 'no' ] && [ \$acceptable_species == 'no' ]; then jq -n --arg key "\$top_hit" '{"error": ("top hit " + \$key + " does not have a reference genome. Sample will not proceed beyond preprocessing workflow.")}' > ${error_log} && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json}; fi """ stub: diff --git a/workflows/preprocessing.nf b/workflows/preprocessing.nf index 5c7adff..b5805e7 100644 --- a/workflows/preprocessing.nf +++ b/workflows/preprocessing.nf @@ -55,7 +55,7 @@ workflow preprocessing { mykrobe(kraken2.out.kraken2_fqs) - afanc(kraken2.out.kraken2_fqs, afanc_myco_db) + afanc(kraken2.out.kraken2_fqs.join(kraken2.out.kraken2_json, by: 0), afanc_myco_db) // set speciation report speciation_report = afanc.out.afanc_report From a508343db1d1ec7f017db12cea0e31f54a271dfd Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 22 May 2023 14:35:55 +0000 Subject: [PATCH 04/18] update afanc to 0.10.1 --- docker/Dockerfile.preprocessing-0.9.7 | 2 +- singularity/Singularity.preprocessing-0.9.7 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index 2331cd7..a89e672 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -12,7 +12,7 @@ fastp_version=0.20.1 \ fastqc_version=0.11.9 \ fqtools_version=2.3 \ kraken2_version=2.1.1 \ -afanc_version=0.10.0 \ +afanc_version=0.10.1 \ mykrobe_version=0.12.1 \ bwa_version=0.7.17 \ mash_version=2.3 \ diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index 81e18f8..6a2fb0c 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -16,7 +16,7 @@ fastp_version=0.20.1 fastqc_version=0.11.9 fqtools_version=2.3 kraken2_version=2.1.1 -afanc_version=0.10.0 +afanc_version=0.10.1 mykrobe_version=0.12.1 bwa_version=0.7.17 mash_version=2.3 @@ -138,7 +138,7 @@ export fastp_version=0.20.1 export fastqc_version=0.11.9 export fqtools_version=2.3 export kraken2_version=2.1.1 -export afanc_version=0.10.0 +export afanc_version=0.10.1 export mykrobe_version=0.12.1 export bwa_version=0.7.17 export mash_version=2.3 From cbd355e6e9722815afa266a8e6f511c4e78e06eb Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 22 May 2023 14:37:36 +0000 Subject: [PATCH 05/18] update config --- nextflow.config | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nextflow.config b/nextflow.config index 93e1b12..b9abba7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -50,7 +50,7 @@ params { config_dir = "${baseDir}/nextflow.config" // path to the Afanc Myco database - afanc_myco_db = "${baseDir}/Mycobacteriaciae_DB_6.0/" + afanc_myco_db = "${baseDir}/Mycobacteriaciae_DB_7.0/" } @@ -70,11 +70,11 @@ profiles { withLabel:high_memory { memory = '18GB' } withLabel:getversion { - container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.6" + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.7" } withLabel:preprocessing { - container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.6" + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.7" } withName:downloadContamGenomes { @@ -90,11 +90,11 @@ profiles { } withLabel:clockwork { - container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.6" + container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.7" } withLabel:vcfpredict { - container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.6" + container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.7" } } @@ -118,11 +118,11 @@ profiles { withLabel:high_memory { memory = '18GB' } withLabel:getversion { - container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.6" + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.7" } withLabel:preprocessing { - container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.6" + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.7" } withName:downloadContamGenomes { @@ -137,11 +137,11 @@ profiles { } withLabel:clockwork { - container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.6" + container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.7" } withLabel:vcfpredict { - container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.6" + container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.7" } } } @@ -161,11 +161,11 @@ profiles { withLabel:high_memory { memory = '18GB' } withLabel:getversion { - container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.6" + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.7" } withLabel:preprocessing { - container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.6" + container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.7" } withName:downloadContamGenomes { @@ -180,11 +180,11 @@ profiles { } withLabel:clockwork { - container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.6" + container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.7" } withLabel:vcfpredict { - container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.6" + container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.7" } } } From 9179e87a9f5f951db39d335ac75c87424350a88a Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 22 May 2023 14:56:08 +0000 Subject: [PATCH 06/18] remove duplicate channel name --- modules/preprocessingModules.nf | 2 +- workflows/preprocessing.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/preprocessingModules.nf b/modules/preprocessingModules.nf index 4a4a66d..5ab45a0 100644 --- a/modules/preprocessingModules.nf +++ b/modules/preprocessingModules.nf @@ -344,7 +344,7 @@ process afanc { path(afanc_myco_db) output: - tuple val(sample_name), path("${sample_name}_afanc_report.json"), stdout, emit: afanc_report + tuple val(sample_name), path("${sample_name}_afanc_report.json"), stdout, emit: afanc_json path "${sample_name}_err.json", emit: afanc_log optional true path "${sample_name}_report.json", emit: afanc_report optional true // tuple val(sample_name), path(fq1), path(fq2), stdout, emit: afanc_fqs diff --git a/workflows/preprocessing.nf b/workflows/preprocessing.nf index b5805e7..2046f91 100644 --- a/workflows/preprocessing.nf +++ b/workflows/preprocessing.nf @@ -58,7 +58,7 @@ workflow preprocessing { afanc(kraken2.out.kraken2_fqs.join(kraken2.out.kraken2_json, by: 0), afanc_myco_db) // set speciation report - speciation_report = afanc.out.afanc_report + speciation_report = afanc.out.afanc_json bowtie2(kraken2.out.kraken2_fqs, bowtie_dir.toList()) From 0bb547c3c58eab459956932c7a0b4e76fca2c691 Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 22 May 2023 22:23:51 +0000 Subject: [PATCH 07/18] add missing make command for fastani --- docker/Dockerfile.preprocessing-0.9.7 | 1 + singularity/Singularity.preprocessing-0.9.7 | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index a89e672..cd6a8db 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -96,6 +96,7 @@ RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_vers && cd FastANI-${fastani_version} \ && ./bootstrap.sh \ && ./configure --prefix=/usr/local \ +&& make \ && cd .. RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index 6a2fb0c..1df5224 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -100,6 +100,7 @@ curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version} && cd FastANI-${fastani_version} \ && ./bootstrap.sh \ && ./configure --prefix=/usr/local \ +&& make \ && cd .. From fd7d7b0c9244c5a5a8872c432b44c52c075b6b24 Mon Sep 17 00:00:00 2001 From: annacprice Date: Tue, 23 May 2023 13:39:37 +0000 Subject: [PATCH 08/18] fix fastani install --- docker/Dockerfile.preprocessing-0.9.7 | 9 +++------ singularity/Singularity.preprocessing-0.9.7 | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index cd6a8db..7248f04 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -92,12 +92,9 @@ RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_vers && curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ && mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ && rm -r mash-Linux* \ -&& curl -fsSL "https://github.com/ParBLiSS/FastANI/archive/refs/tags/v${fastani_version}.tar.gz" | tar -xz \ -&& cd FastANI-${fastani_version} \ -&& ./bootstrap.sh \ -&& ./configure --prefix=/usr/local \ -&& make \ -&& cd .. +&& wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ +&& unzip fastANI-Linux64-v${fastani_version}.zip \ +&& mv FastANI /usr/local/bin RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ && mkdir -p /opt/edirect \ diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index 1df5224..8eb6968 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -96,12 +96,9 @@ curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version} && curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \ && mv mash-Linux64-v${mash_version}/mash /usr/local/bin \ && rm -r mash-Linux* \ -&& curl -fsSL "https://github.com/ParBLiSS/FastANI/archive/refs/tags/v${fastani_version}.tar.gz" | tar -xz \ -&& cd FastANI-${fastani_version} \ -&& ./bootstrap.sh \ -&& ./configure --prefix=/usr/local \ -&& make \ -&& cd .. +&& wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ +&& unzip fastANI-Linux64-v${fastani_version}.zip \ +&& mv FastANI /usr/local/bin sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ From 244a299238d52292ee7c68ef0fd9712bc9f3b478 Mon Sep 17 00:00:00 2001 From: annacprice Date: Tue, 23 May 2023 15:25:49 +0000 Subject: [PATCH 09/18] update fastani install --- docker/Dockerfile.preprocessing-0.9.7 | 2 +- singularity/Singularity.preprocessing-0.9.7 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index 7248f04..ba6d63f 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -94,7 +94,7 @@ RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_vers && rm -r mash-Linux* \ && wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ && unzip fastANI-Linux64-v${fastani_version}.zip \ -&& mv FastANI /usr/local/bin +&& mv fastANI /usr/local/bin RUN sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ && mkdir -p /opt/edirect \ diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index 8eb6968..8c18b41 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -98,7 +98,7 @@ curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version} && rm -r mash-Linux* \ && wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \ && unzip fastANI-Linux64-v${fastani_version}.zip \ -&& mv FastANI /usr/local/bin +&& mv fastANI /usr/local/bin sh -c "$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \ From 8be2a001e2d6ed7c7979d1bd5af5faab8cfe23ef Mon Sep 17 00:00:00 2001 From: ArthurVM Date: Mon, 26 Jun 2023 16:29:38 +0100 Subject: [PATCH 10/18] new afac json support --- bin/reformat_afanc_json.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/bin/reformat_afanc_json.py b/bin/reformat_afanc_json.py index 28442b7..cdb9df9 100644 --- a/bin/reformat_afanc_json.py +++ b/bin/reformat_afanc_json.py @@ -17,15 +17,28 @@ def reformat_json(afanc_report): new_json[sample_id]["phylogenetics"]["species"] = {} species_dict = new_json[sample_id]["phylogenetics"]["species"] - for event in data["results"]["Detection_events"]: - if "closest_variant" in event: - event = event["closest_variant"] + cr = data["results"]["Detection_events"]["Clustering_results"] + vp = data["results"]["Detection_events"]["Variant_profile"] + + for c_event in cr: + event = c_event + species = "_".join(c_event["name"].split(" ")[:2]) + id = species.replace(".", "") + + ## capture closest variant info + if "closest_variant" in c_event: + event = c_event["closest_variant"] + id = event["name"].replace(" ", "_") + + ## check to see if the top species hit exists within the variant profile + if species in vp and len(vp[species]) > 0: + ## if so, construct a new hit id + id = species + "_" + sorted(list(vp[species].keys()))[-1] - id = event["name"].replace(" ", "_") - id = id.replace(".", "") species_dict[id] = { "percent_coverage" : event["reference_cov"]*100, "median_depth" : event["median_DOC"] } new_json["arguments"] = data["arguments"] + new_json["versions"] = data["versions"] with open(f"{sample_id}_afanc_report.json", "w") as fout: json.dump(new_json, fout, indent = 4) From 12c09c559fb9a45768ad5c2192ac04b0690d024f Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 26 Jun 2023 20:19:08 +0000 Subject: [PATCH 11/18] update afanc --- docker/Dockerfile.preprocessing-0.9.7 | 2 +- singularity/Singularity.preprocessing-0.9.7 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index ba6d63f..a6702cd 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -12,7 +12,7 @@ fastp_version=0.20.1 \ fastqc_version=0.11.9 \ fqtools_version=2.3 \ kraken2_version=2.1.1 \ -afanc_version=0.10.1 \ +afanc_version=0.10.2 \ mykrobe_version=0.12.1 \ bwa_version=0.7.17 \ mash_version=2.3 \ diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index 8c18b41..0b839f9 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -16,7 +16,7 @@ fastp_version=0.20.1 fastqc_version=0.11.9 fqtools_version=2.3 kraken2_version=2.1.1 -afanc_version=0.10.1 +afanc_version=0.10.2 mykrobe_version=0.12.1 bwa_version=0.7.17 mash_version=2.3 @@ -136,7 +136,7 @@ export fastp_version=0.20.1 export fastqc_version=0.11.9 export fqtools_version=2.3 export kraken2_version=2.1.1 -export afanc_version=0.10.1 +export afanc_version=0.10.2 export mykrobe_version=0.12.1 export bwa_version=0.7.17 export mash_version=2.3 From 8254d578687642dbb129fb39a50c1fdb7e43d67f Mon Sep 17 00:00:00 2001 From: annacprice Date: Mon, 26 Jun 2023 20:19:42 +0000 Subject: [PATCH 12/18] update config --- nextflow.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nextflow.config b/nextflow.config index b9abba7..d4ef2e1 100644 --- a/nextflow.config +++ b/nextflow.config @@ -85,7 +85,7 @@ profiles { withLabel:retryAfanc { // Afanc sometimes fails curl in slurm, retry if so (error is masked as error status 1) - errorStrategy = {task.exitStatus == 1 ? 'retry' : 'terminate' } + errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } maxRetries = 5 } @@ -132,7 +132,7 @@ profiles { } withLabel:retryAfanc { - errorStrategy = {task.exitStatus == 1 ? 'retry' : 'terminate' } + errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } maxRetries = 5 } @@ -175,7 +175,7 @@ profiles { } withLabel:retryAfanc { - errorStrategy = {task.exitStatus == 1 ? 'retry' : 'terminate' } + errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } maxRetries = 5 } From e03592f1e952266654c53752429b4afee676388e Mon Sep 17 00:00:00 2001 From: annacprice Date: Wed, 28 Jun 2023 17:15:10 +0000 Subject: [PATCH 13/18] add missing bcftools dependency for afanc --- docker/Dockerfile.preprocessing-0.9.7 | 7 ++++++- singularity/Singularity.preprocessing-0.9.7 | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.preprocessing-0.9.7 b/docker/Dockerfile.preprocessing-0.9.7 index a6702cd..4fab8b6 100644 --- a/docker/Dockerfile.preprocessing-0.9.7 +++ b/docker/Dockerfile.preprocessing-0.9.7 @@ -5,6 +5,7 @@ LABEL maintainer="pricea35@cardiff.ac.uk" \ about.summary="container for the preprocessing workflow" ENV samtools_version=1.12 \ +bcftools_version=1.12 \ htslib_version=1.12 \ bedtools_version=2.29.2 \ bowtie2_version=2.4.2 \ @@ -37,7 +38,11 @@ RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}. && curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ && make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ && make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ -&& rm -r samtools-${samtools_version} +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} RUN curl -fsSL https://github.com/alastair-droop/fqtools/archive/v${fqtools_version}.tar.gz | tar -xz \ && mv htslib-${htslib_version} fqtools-${fqtools_version} \ diff --git a/singularity/Singularity.preprocessing-0.9.7 b/singularity/Singularity.preprocessing-0.9.7 index 0b839f9..7ca3b35 100644 --- a/singularity/Singularity.preprocessing-0.9.7 +++ b/singularity/Singularity.preprocessing-0.9.7 @@ -9,6 +9,7 @@ about.summary="container for the preprocessing workflow" samtools_version=1.12 +bcftools_version=1.12 htslib_version=1.12 bedtools_version=2.29.2 bowtie2_version=2.4.2 @@ -41,7 +42,11 @@ curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar. && curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \ && make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \ && make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ -&& rm -r samtools-${samtools_version} +&& rm -r samtools-${samtools_version} \ +&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \ +&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \ +&& rm -r bcftools-${bcftools_version} curl -fsSL https://github.com/alastair-droop/fqtools/archive/v${fqtools_version}.tar.gz | tar -xz \ && mv htslib-${htslib_version} fqtools-${fqtools_version} \ @@ -129,6 +134,7 @@ LANGUAGE=en_US.UTF-8 %environment export samtools_version=1.12 +export bcftools_version=1.12 export htslib_version=1.12 export bedtools_version=2.29.2 export bowtie2_version=2.4.2 From 7bbdb2f31a8e8d5a15da05a6b1cea0455b5579ca Mon Sep 17 00:00:00 2001 From: annacprice Date: Wed, 28 Jun 2023 19:28:13 +0000 Subject: [PATCH 14/18] update version.json --- version.json | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/version.json b/version.json index a9832b5..2e6d615 100644 --- a/version.json +++ b/version.json @@ -1,7 +1,8 @@ { - "lodestone-0.9.6" : { - "preprocessing-0.9.6" : { + "lodestone-0.9.7" : { + "preprocessing-0.9.7" : { "samtools_version" : "1.12", + "bcftools_version" : "1.12", "htslib_version" : "1.12", "bedtools_version" : "2.29.2", "bowtie2_version" : "2.4.2", @@ -9,19 +10,20 @@ "fastqc_version" : "0.11.9", "fqtools_version" : "2.3", "kraken2_version" : "2.1.1", - "afanc_version" : "0.9.2", + "afanc_version" : "0.10.2", "mykrobe_version" : "0.12.1", "bwa_version" : "0.7.17", - "mash_version" : "2.3" + "mash_version" : "2.3", + "fastani_version" : "1.33" }, - "vcfpredict-0.9.6" : { + "vcfpredict-0.9.7" : { "vcfmix_version" : "d4693344bf612780723e39ce27c8ae3868f95417", "gumpy_version" : "1.0.15", "piezo_version" : "0.3", "gnomonicus_version" : "1.1.2", "tuberculosis_amr_catalogues" : "12d38733ad2e238729a3de9f725081e1d4872968" }, - "clockwork-0.9.6" : { + "clockwork-0.9.7" : { "samtools_version" : "1.12", "htslib_version" : "1.12", "bcftools_version" : "1.12", @@ -42,7 +44,7 @@ "bowtie2_index" : "bowtie2", "bowtie_index_name" : "hg19_1kgmaj", "amr_cat" : "/tuberculosis_amr_catalogues/catalogues/NC_000962.3/NC_000962.3_WHO-UCN-GTB-PCI-2021.7_v1.0_GARC1_RUS.csv", - "afanc_myco_db" : "${baseDir}/afanc/Mycobacteriaciae_DB_6.0/" + "afanc_myco_db" : "${baseDir}/afanc/Mycobacteriaciae_DB_7.0/" } } } \ No newline at end of file From a496ebb9e1b6f5550318759be53ae77148bfbe73 Mon Sep 17 00:00:00 2001 From: annacprice Date: Wed, 28 Jun 2023 19:29:05 +0000 Subject: [PATCH 15/18] update actions workflow --- .github/workflows/build-push-quay.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-push-quay.yml b/.github/workflows/build-push-quay.yml index 38dab37..ab8e2a4 100644 --- a/.github/workflows/build-push-quay.yml +++ b/.github/workflows/build-push-quay.yml @@ -2,7 +2,6 @@ name: build-push-quay on: push: branches: - - main - v0.9.6 - 0.9.7-dev paths: From ed81f076992f0245a5db84ca1eda592e87412e23 Mon Sep 17 00:00:00 2001 From: annacprice Date: Thu, 29 Jun 2023 20:02:22 +0000 Subject: [PATCH 16/18] add variant profile command to afanc --- modules/preprocessingModules.nf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/preprocessingModules.nf b/modules/preprocessingModules.nf index 5ab45a0..e4005f5 100644 --- a/modules/preprocessingModules.nf +++ b/modules/preprocessingModules.nf @@ -334,8 +334,6 @@ process afanc { label 'medium_memory' label 'retryAfanc' - errorStrategy 'ignore' - publishDir "${params.output_dir}/$sample_name/speciation_reports_for_reads_postFastP", mode: 'copy', pattern: '*_afanc_report.json' publishDir "${params.output_dir}/$sample_name", mode: 'copy', overwrite: 'true', pattern: '*{_err.json,_report.json}' @@ -357,16 +355,17 @@ process afanc { """ if [[ ${run_afanc} =~ /${sample_name}/ ]] then - afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 5.0 -n 1000 -o ${sample_name} -t ${task.cpus} + afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 5.0 -n 1000 -o ${sample_name} -t ${task.cpus} -v ${afanc_myco_db}/lineage_profiles/TB_variants.tsv python3 ${baseDir}/bin/reformat_afanc_json.py ${sample_name}/${sample_name}.json printf ${sample_name} else - afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 2.0 -n 500 -o ${sample_name} -t ${task.cpus} + afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 2.0 -n 500 -o ${sample_name} -t ${task.cpus} -v ${afanc_myco_db}/lineage_profiles/TB_variants.tsv python3 ${baseDir}/bin/reformat_afanc_json.py ${sample_name}/${sample_name}.json python3 ${baseDir}/bin/identify_tophit_and_contaminants2.py ${afanc_report} ${kraken_json} ${baseDir}/resources/assembly_summary_refseq.txt ${params.species} ${params.unmix_myco} ${baseDir}/resources null echo '{"error":"Kraken's top family hit either wasn't Mycobacteriaceae, or there were < 100k Mycobacteriaceae reads. Sample will not proceed further than afanc."}' | jq '.' > ${error_log} && printf "no" && jq -s ".[0] * .[1] * .[2]" ${software_json} ${error_log} ${sample_name}_species_in_sample.json > ${report_json} + fi """ @@ -695,7 +694,7 @@ process reAfanc { afanc_report = "${sample_name}_afanc_report.json" """ - afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 5.0 -n 1000 -o ${sample_name} -t ${task.cpus} + afanc screen ${afanc_myco_db} ${fq1} ${fq2} -p 5.0 -n 1000 -o ${sample_name} -t ${task.cpus} -v ${afanc_myco_db}/lineage_profiles/TB_variants.tsv python3 ${baseDir}/bin/reformat_afanc_json.py ${sample_name}/${sample_name}.json printf ${sample_name} """ From e4b3691550434b6388861380ea896cd750da115d Mon Sep 17 00:00:00 2001 From: annacprice Date: Thu, 29 Jun 2023 20:02:51 +0000 Subject: [PATCH 17/18] update config --- nextflow.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nextflow.config b/nextflow.config index d4ef2e1..4f53070 100644 --- a/nextflow.config +++ b/nextflow.config @@ -84,6 +84,7 @@ profiles { } withLabel:retryAfanc { + shell = ['/bin/bash','-u'] // Afanc sometimes fails curl in slurm, retry if so (error is masked as error status 1) errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } maxRetries = 5 @@ -132,6 +133,7 @@ profiles { } withLabel:retryAfanc { + shell = ['/bin/bash','-u'] errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } maxRetries = 5 } @@ -175,6 +177,7 @@ profiles { } withLabel:retryAfanc { + shell = ['/bin/bash','-u'] errorStrategy = {task.exitStatus == 1 ? 'retry' : 'ignore' } maxRetries = 5 } From 80f869c47d645ea38f0fb9e06b38aa2b5312289d Mon Sep 17 00:00:00 2001 From: annacprice Date: Fri, 30 Jun 2023 16:09:36 +0000 Subject: [PATCH 18/18] update README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 3706865..f14831d 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,7 @@ Run [vcfmix](https://github.com/AlexOrlek/VCFMIX), yes or no. Set to no for synt Run [gnomonicus](https://github.com/oxfordmmm/gnomonicus), yes or no
* **amr_cat**
Path to AMR catalogue for gnomonicus
-* **afanc_myco_db**
-Path to the [afanc](https://github.com/ArthurVM/Afanc) database used for speciation. Obtain from https://s3.climb.ac.uk/microbial-bioin-sp3/Mycobacteriaciae_DB_6.0.tar.gz +* **afanc_myco_db**
Path to the [afanc](https://github.com/ArthurVM/Afanc) database used for speciation. Obtain from https://s3.climb.ac.uk/microbial-bioin-sp3/Mycobacteriaciae_DB_7.0.tar.gz
For more information on the parameters run `nextflow run main.nf --help`