From 53aa4b1a8b5a0592d4bcbcd567b9e0e1b85cac1c Mon Sep 17 00:00:00 2001 From: Andre Watson Date: Mon, 15 Jul 2024 12:44:45 -0600 Subject: [PATCH 1/5] Reduced Docker image size: 2.87GB->1.3GB --- runReadsToContig/Dockerfile | 56 +++++++++++++++++++--------- runReadsToContig/nextflow.config | 5 ++- runReadsToContig/runReadsToContig.nf | 1 + 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/runReadsToContig/Dockerfile b/runReadsToContig/Dockerfile index 24e2128..8ffea85 100644 --- a/runReadsToContig/Dockerfile +++ b/runReadsToContig/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM continuumio/miniconda3:23.5.2-0 +FROM continuumio/miniconda3:23.5.2-0 AS build ENV container=docker @@ -7,20 +7,40 @@ ENV container=docker RUN conda config --add channels conda-forge \ && conda config --add channels bioconda -# install dependencies -RUN conda install -c conda-forge r-base -RUN conda install -c conda-forge python=3.11 -RUN conda install -c bioconda perl-json -RUN conda install -c bioconda samclip=0.4.0 -RUN conda install -c bioconda bwa -RUN conda install -c bioconda bowtie2 -RUN conda install -c bioconda samtools=1.6 - -ADD bin/bam_to_fastq.pl /opt/conda/bin -ADD bin/contig_stats.pl /opt/conda/bin -ADD bin/ContigCoverageFold_plots_from_samPileup.pl /opt/conda/bin -ADD bin/fastq_utility.pm /opt/conda/bin -ADD bin/runReadsToContig.pl /opt/conda/bin -ADD bin/tab2Json_for_dataTable.pl /opt/conda/bin - -CMD ["/bin/bash"] \ No newline at end of file +RUN conda init bash \ + && . ~/.bashrc \ + && conda create --name readsToContig \ + && conda activate readsToContig + +RUN conda install -n readsToContig -c conda-forge r-base +RUN conda install -n readsToContig -c conda-forge python=3.11 +RUN conda install -n readsToContig -c bioconda perl-json +RUN conda install -n readsToContig -c bioconda samclip=0.4.0 +RUN conda install -n readsToContig -c bioconda bwa +RUN conda install -n readsToContig -c bioconda bowtie2 +RUN conda install -n readsToContig -c bioconda samtools=1.6 +RUN conda install -c conda-forge conda-pack + + +ADD bin/bam_to_fastq.pl /opt/conda/envs/readsToContig/bin +ADD bin/contig_stats.pl /opt/conda/envs/readsToContig/bin +ADD bin/ContigCoverageFold_plots_from_samPileup.pl /opt/conda/envs/readsToContig/bin +ADD bin/fastq_utility.pm /opt/conda/envs/readsToContig/bin +ADD bin/runReadsToContig.pl /opt/conda/envs/readsToContig/bin +ADD bin/tab2Json_for_dataTable.pl /opt/conda/envs/readsToContig/bin + +RUN conda-pack -n readsToContig -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar + +RUN /venv/bin/conda-unpack + +FROM debian:buster AS runtime +#RUN apk add --no-cache bash + +COPY --from=build /venv /venv +ENV PERL5LIB=/venv/lib/perl5/core_perl +ENV PATH=/venv/bin:$PATH + +SHELL ["/bin/bash", "-c"] +CMD /bin/bash \ No newline at end of file diff --git a/runReadsToContig/nextflow.config b/runReadsToContig/nextflow.config index b87796e..c7e508c 100644 --- a/runReadsToContig/nextflow.config +++ b/runReadsToContig/nextflow.config @@ -1,3 +1,6 @@ +process.container = 'apwat/run_r2c:1.3' +docker.enabled=true + params { pairFile = "nf_assets/NO_FILE" unpairFile = "nf_assets/NO_FILE2" @@ -18,5 +21,3 @@ workflow.onComplete = { "rm -rf nf_assets".execute().text } -docker.enabled=true -process.container = 'apwat/run_r2c:1.1' \ No newline at end of file diff --git a/runReadsToContig/runReadsToContig.nf b/runReadsToContig/runReadsToContig.nf index eb1db53..9bb5504 100644 --- a/runReadsToContig/runReadsToContig.nf +++ b/runReadsToContig/runReadsToContig.nf @@ -1,6 +1,7 @@ #!/usr/bin/env nextflow process r2c { + debug true publishDir( path: "$params.outDir/AssemblyBasedAnalysis/readsMappingToContig", mode: 'copy' From 5bf8798987db08b7832c0585ee25b35336d13eb5 Mon Sep 17 00:00:00 2001 From: Andre Watson Date: Tue, 16 Jul 2024 14:07:23 -0600 Subject: [PATCH 2/5] runFaQCs now uses multistage build --- runFaQCs/Dockerfile | 30 ++++++++++++++++++++++-------- runFaQCs/nextflow.config | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/runFaQCs/Dockerfile b/runFaQCs/Dockerfile index 5a65a30..86e2582 100644 --- a/runFaQCs/Dockerfile +++ b/runFaQCs/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 -FROM continuumio/miniconda3:23.5.2-0 +FROM continuumio/miniconda3:23.5.2-0 AS build -ENV container docker +ENV container=docker # dependencies for faqcs RUN apt-get update -y \ @@ -12,14 +12,28 @@ RUN apt-get update -y \ RUN conda config --add channels conda-forge \ && conda config --add channels bioconda +RUN conda init bash \ + && . ~/.bashrc \ + && conda create --name runFaQCs \ + && conda activate runFaQCs + # install faqcs -RUN conda install -c bioconda faqcs=2.10 +RUN conda install -n runFaQCs -c bioconda faqcs=2.10 +#RUN conda install -n runFaQCs -c conda-forge jellyfish +#RUN conda install -n runFaQCs -c conda-forge zlib +RUN conda install -c conda-forge conda-pack + +RUN conda-pack -n runFaQCs -o /tmp/env.tar && \ + mkdir /venv && cd /venv && tar xf /tmp/env.tar && \ + rm /tmp/env.tar -# add files -# ADD runQC_subroutines/* /opt/conda/bin/runQC_subroutines/ -ADD *.wdl /opt/conda/bin/ +RUN /venv/bin/conda-unpack -# ENV PATH="${PATH}:/opt/conda/bin/runQC_subroutines" +FROM debian:buster AS runtime -CMD ["/bin/bash"] +COPY --from=build /venv /venv +ENV PATH=/venv/bin:$PATH + +SHELL ["/bin/bash", "-c"] +CMD /bin/bash diff --git a/runFaQCs/nextflow.config b/runFaQCs/nextflow.config index 54200e2..9214d42 100644 --- a/runFaQCs/nextflow.config +++ b/runFaQCs/nextflow.config @@ -1,4 +1,4 @@ -process.container = 'kaijli/runqc:1.0' +process.container = 'apwat/run_faqcs:1.1' docker.enabled = true workflow.onComplete = { "rm -rf nf_assets".execute().text From c675ddfbc5f03f9e552a94c6ceb523d723d7c0e2 Mon Sep 17 00:00:00 2001 From: Andre Watson Date: Mon, 22 Jul 2024 13:47:24 -0600 Subject: [PATCH 3/5] charliecloud now used to run sra2fastq and runFaQCs --- runFaQCs/Dockerfile | 2 +- runFaQCs/nextflow.config | 4 ++-- sra2fastq/nextflow.config | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runFaQCs/Dockerfile b/runFaQCs/Dockerfile index 86e2582..06e3c5c 100644 --- a/runFaQCs/Dockerfile +++ b/runFaQCs/Dockerfile @@ -29,7 +29,7 @@ RUN conda-pack -n runFaQCs -o /tmp/env.tar && \ RUN /venv/bin/conda-unpack -FROM debian:buster AS runtime +FROM debian:latest AS runtime COPY --from=build /venv /venv ENV PATH=/venv/bin:$PATH diff --git a/runFaQCs/nextflow.config b/runFaQCs/nextflow.config index 9214d42..5c349ea 100644 --- a/runFaQCs/nextflow.config +++ b/runFaQCs/nextflow.config @@ -1,5 +1,5 @@ -process.container = 'apwat/run_faqcs:1.1' -docker.enabled = true +process.container = 'apwat/run_faqcs:1.2' +charliecloud.enabled = true //must set $NXF_CHARLIECLOUD_CACHEDIR workflow.onComplete = { "rm -rf nf_assets".execute().text } diff --git a/sra2fastq/nextflow.config b/sra2fastq/nextflow.config index 134a2fb..4ebf807 100644 --- a/sra2fastq/nextflow.config +++ b/sra2fastq/nextflow.config @@ -7,7 +7,7 @@ params { } process.container = 'kaijli/sra2fastq:1.6.3' -docker.enabled = true +charliecloud.enabled = true //must set $NXF_CHARLIECLOUD_CACHEDIR executor { submitRateLimit = '1/5sec' } \ No newline at end of file From fd09340c3bd7f0c88a5fe9bf86b4097c6ccae72b Mon Sep 17 00:00:00 2001 From: Andre Watson Date: Mon, 22 Jul 2024 14:47:12 -0600 Subject: [PATCH 4/5] readsToContig now uses Charliecloud --- runReadsToContig/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runReadsToContig/nextflow.config b/runReadsToContig/nextflow.config index c7e508c..23e815f 100644 --- a/runReadsToContig/nextflow.config +++ b/runReadsToContig/nextflow.config @@ -1,5 +1,5 @@ process.container = 'apwat/run_r2c:1.3' -docker.enabled=true +charliecloud.enabled=true params { pairFile = "nf_assets/NO_FILE" From d990ebb4e31b8ecf05dd77783c19c43d2b2b8bb1 Mon Sep 17 00:00:00 2001 From: Andre Watson Date: Mon, 26 Aug 2024 08:46:03 -0600 Subject: [PATCH 5/5] Per discussions with Mark, prefer Singularity execution over Charliecloud --- runFaQCs/nextflow.config | 2 +- runReadsToContig/nextflow.config | 2 +- sra2fastq/nextflow.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runFaQCs/nextflow.config b/runFaQCs/nextflow.config index 5c349ea..203245b 100644 --- a/runFaQCs/nextflow.config +++ b/runFaQCs/nextflow.config @@ -1,5 +1,5 @@ process.container = 'apwat/run_faqcs:1.2' -charliecloud.enabled = true //must set $NXF_CHARLIECLOUD_CACHEDIR +singularity.enabled = true workflow.onComplete = { "rm -rf nf_assets".execute().text } diff --git a/runReadsToContig/nextflow.config b/runReadsToContig/nextflow.config index 23e815f..b4a673d 100644 --- a/runReadsToContig/nextflow.config +++ b/runReadsToContig/nextflow.config @@ -1,5 +1,5 @@ process.container = 'apwat/run_r2c:1.3' -charliecloud.enabled=true +singularity.enabled=true params { pairFile = "nf_assets/NO_FILE" diff --git a/sra2fastq/nextflow.config b/sra2fastq/nextflow.config index 4ebf807..258a1d7 100644 --- a/sra2fastq/nextflow.config +++ b/sra2fastq/nextflow.config @@ -7,7 +7,7 @@ params { } process.container = 'kaijli/sra2fastq:1.6.3' -charliecloud.enabled = true //must set $NXF_CHARLIECLOUD_CACHEDIR +singularity.enabled = true executor { submitRateLimit = '1/5sec' } \ No newline at end of file