From 546df0ef169dc18708918b916dd402ec4a65b78c Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Wed, 30 Sep 2020 10:54:33 -0400 Subject: [PATCH 1/8] manual add of increased mem --- pipes/WDL/tasks/tasks_nextstrain.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index da16d827a..9b2337019 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -132,7 +132,7 @@ task filter_subsample_sequences { } runtime { docker: docker - memory: "3 GB" + memory: "6 GB" cpu : 4 disks: "local-disk 100 HDD" dx_instance_type: "mem1_ssd1_v2_x4" From 5175bc63a450babf6990e5e1d74953da7422d9f5 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Wed, 30 Sep 2020 11:08:31 -0400 Subject: [PATCH 2/8] dynamically size memory based on fasta size --- pipes/WDL/tasks/tasks_nextstrain.wdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 9b2337019..ced62b59b 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -104,6 +104,7 @@ task filter_subsample_sequences { } } String out_fname = sub(sub(basename(sequences_fasta), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") + Int mem_size = ceil(size(sequences_fasta, "GB") * 3) command { set -e augur version > VERSION @@ -132,7 +133,7 @@ task filter_subsample_sequences { } runtime { docker: docker - memory: "6 GB" + memory: mem_size cpu : 4 disks: "local-disk 100 HDD" dx_instance_type: "mem1_ssd1_v2_x4" From c1860d7988a3d0283973d4eb91131109fd1cbd05 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Wed, 30 Sep 2020 11:17:50 -0400 Subject: [PATCH 3/8] dynamic size mem filter_to_list task --- pipes/WDL/tasks/tasks_nextstrain.wdl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index ced62b59b..efdc51e9b 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -104,7 +104,6 @@ task filter_subsample_sequences { } } String out_fname = sub(sub(basename(sequences_fasta), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") - Int mem_size = ceil(size(sequences_fasta, "GB") * 3) command { set -e augur version > VERSION @@ -133,7 +132,7 @@ task filter_subsample_sequences { } runtime { docker: docker - memory: mem_size + memory: "3 GB" cpu : 4 disks: "local-disk 100 HDD" dx_instance_type: "mem1_ssd1_v2_x4" @@ -171,6 +170,7 @@ task filter_sequences_to_list { } } String out_fname = sub(sub(basename(sequences), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") + Int mem_size = ceil(size(sequences, "GB") * 3) command { set -e augur version > VERSION @@ -196,7 +196,7 @@ task filter_sequences_to_list { } runtime { docker: docker - memory: "3 GB" + memory: mem_size cpu : 2 disks: "local-disk 100 HDD" dx_instance_type: "mem1_ssd1_v2_x2" From 499a4d419b1e031b4e71500c4a57a33fea2d1cf4 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Wed, 30 Sep 2020 11:26:42 -0400 Subject: [PATCH 4/8] 10GB --- pipes/WDL/tasks/tasks_nextstrain.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index efdc51e9b..f95b4605f 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -170,7 +170,7 @@ task filter_sequences_to_list { } } String out_fname = sub(sub(basename(sequences), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") - Int mem_size = ceil(size(sequences, "GB") * 3) + # Int mem_size = ceil(size(sequences, "GB") * 3) command { set -e augur version > VERSION @@ -196,7 +196,7 @@ task filter_sequences_to_list { } runtime { docker: docker - memory: mem_size + memory: "10 GB" cpu : 2 disks: "local-disk 100 HDD" dx_instance_type: "mem1_ssd1_v2_x2" From a105345ab69e5baf909208d0a606de004f231f5b Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Wed, 30 Sep 2020 11:38:30 -0400 Subject: [PATCH 5/8] fasta_sequences * 5x memory --- pipes/WDL/tasks/tasks_nextstrain.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index f95b4605f..d82581d5c 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -170,7 +170,7 @@ task filter_sequences_to_list { } } String out_fname = sub(sub(basename(sequences), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") - # Int mem_size = ceil(size(sequences, "GB") * 3) + Int mem_size = ceil(size(sequences, "GB") * 5) command { set -e augur version > VERSION @@ -196,7 +196,7 @@ task filter_sequences_to_list { } runtime { docker: docker - memory: "10 GB" + memory: mem_size + " GB" cpu : 2 disks: "local-disk 100 HDD" dx_instance_type: "mem1_ssd1_v2_x2" From 13f9f30ece6d32054c70ded9ca4d12e7025982ee Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Fri, 2 Oct 2020 11:55:35 -0400 Subject: [PATCH 6/8] final 1.1x dynamic mem --- pipes/WDL/tasks/tasks_nextstrain.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index d82581d5c..12bb7c67a 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -170,7 +170,7 @@ task filter_sequences_to_list { } } String out_fname = sub(sub(basename(sequences), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") - Int mem_size = ceil(size(sequences, "GB") * 5) + Int mem_size = ceil(size(sequences, "GB") * 1.1) command { set -e augur version > VERSION From a8b903ffcaa82558a5929b1b6aeafc4d0436c86b Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 5 Oct 2020 16:08:27 -0400 Subject: [PATCH 7/8] up 1.1 to 2.0x memory --- pipes/WDL/tasks/tasks_nextstrain.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 12bb7c67a..b736520b0 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -170,7 +170,7 @@ task filter_sequences_to_list { } } String out_fname = sub(sub(basename(sequences), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") - Int mem_size = ceil(size(sequences, "GB") * 1.1) + Int mem_size = ceil(size(sequences, "GB") * 2) command { set -e augur version > VERSION From 647cd11bc8959f7a91f3848886e774ac187feff8 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi <43828038+schaluva@users.noreply.github.com> Date: Mon, 5 Oct 2020 21:06:04 -0400 Subject: [PATCH 8/8] Update pipes/WDL/tasks/tasks_nextstrain.wdl Co-authored-by: Daniel Park --- pipes/WDL/tasks/tasks_nextstrain.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index b736520b0..4b828a9fe 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -170,7 +170,7 @@ task filter_sequences_to_list { } } String out_fname = sub(sub(basename(sequences), ".vcf", ".filtered.vcf"), ".fasta$", ".filtered.fasta") - Int mem_size = ceil(size(sequences, "GB") * 2) + Int mem_size = ceil(size(sequences, "GB") * 2 + 0.001) command { set -e augur version > VERSION