From 257735cd2d0fb4a995ace94fbba04998f7847b7b Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Fri, 29 Jan 2021 10:10:15 -0500 Subject: [PATCH 01/19] PoC: call api from workflow scripts --- pipes/WDL/tasks/tasks_call_api.wdl | 28 ++++++++++++++++++++++ pipes/WDL/workflows/update_data_tables.wdl | 28 ++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 pipes/WDL/tasks/tasks_call_api.wdl create mode 100644 pipes/WDL/workflows/update_data_tables.wdl diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl new file mode 100644 index 000000000..d546706b5 --- /dev/null +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -0,0 +1,28 @@ +version 1.0 + +task upload_entities_tsv{ + input { + String workspace_name + String terra_project + File tsv_file + + String docker="schaluvadi/pathogen-genomic-surveillance:api-wdl" + } + + command { + + python3 create_data_tables.py -t ~{tsv_file} \ + -p ~{terra_project} \ + -w ~{workspace_name} + + } + + runtime { + docker: docker + preemptible: 0 + } + + output { + String status = read_string(stdout()) + } +} \ No newline at end of file diff --git a/pipes/WDL/workflows/update_data_tables.wdl b/pipes/WDL/workflows/update_data_tables.wdl new file mode 100644 index 000000000..3b40b48d2 --- /dev/null +++ b/pipes/WDL/workflows/update_data_tables.wdl @@ -0,0 +1,28 @@ +version 1.0 + +import "../tasks/tasks_call_api.wdl" as update_entities_api + +workflow update_data_tables { + meta { + description: "Create data tables in Terra workspace from provided tsv load file." + author: "Broad Viral Genomics" + email: "viral-ngs@broadinstitute.org" + } + + input { + String workspace_name + String terra_project + File tsv_file + } + + call update_entities_api.upload_entities_tsv as create_table { + input: + workspace_name = workspace_name, + terra_project = terra_project, + tsv_file = tsv_file + } + + output { + String status_message = create_table.status + } +} \ No newline at end of file From 2fb260740134bbbf88392564ed5b328331e4110c Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Fri, 29 Jan 2021 16:39:29 -0500 Subject: [PATCH 02/19] add wdl to .dockstore.yml --- .dockstore.yml | 5 +++++ pipes/WDL/tasks/tasks_call_api.wdl | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.dockstore.yml b/.dockstore.yml index 9d6c58b4d..daea2475a 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -270,3 +270,8 @@ workflows: primaryDescriptorPath: /pipes/WDL/workflows/trimal.wdl testParameterFiles: - empty.json + - name: update_data_tables + subclass: WDL + primaryDescriptorPath: /pipes/WDL/workflows/update_data_tables.wdl + testParameterFiles: + - empty.json diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index d546706b5..be9e7e8de 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -11,7 +11,7 @@ task upload_entities_tsv{ command { - python3 create_data_tables.py -t ~{tsv_file} \ + python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ -p ~{terra_project} \ -w ~{workspace_name} From d452d1b5e06b1110bfccd45a89686a7bbcf19e16 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Tue, 2 Feb 2021 16:09:05 -0500 Subject: [PATCH 03/19] insert task to make data tables into illumina full pipe --- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index bac163709..38a39cbab 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -4,6 +4,7 @@ import "../tasks/tasks_read_utils.wdl" as read_utils import "../tasks/tasks_ncbi.wdl" as ncbi import "../tasks/tasks_nextstrain.wdl" as nextstrain import "../tasks/tasks_reports.wdl" as reports +import "../tasks/tasks_call_api.wdl" as fapi_tables import "demux_deplete.wdl" import "assemble_refbased.wdl" @@ -48,6 +49,9 @@ workflow sarscov2_illumina_full { String sra_title Int min_genome_bases = 15000 + + String workspace_name + String terra_project } Int taxid = 2697049 String gisaid_prefix = 'hCoV-19/' @@ -232,6 +236,14 @@ workflow sarscov2_illumina_full { out_name = "gisaid-meta-~{flowcell_id}.tsv" } + # create data tables with assembly_meta_tsv + call fapi_tables.upload_entities_tsv as data_tables { + input: + workspace_name = workspace_name, + terra_project = terra_project, + tsv_file = assembly_meta_tsv.combined + } + output { Array[File] raw_reads_unaligned_bams = demux_deplete.raw_reads_unaligned_bams Array[File] cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams @@ -280,5 +292,6 @@ workflow sarscov2_illumina_full { Int num_submittable = length(select_all(submittable_id)) Int num_failed_annotation = length(select_all(failed_annotation_id)) Int num_samples = length(group_bams_by_sample.sample_names) + String data_table_status = data_tables.status } } From e7701ce635d12e98c15945dbdf1c47c9e4d68d74 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 10:25:46 -0500 Subject: [PATCH 04/19] updated docker, test str and file inputs for bams --- pipes/WDL/tasks/tasks_call_api.wdl | 13 ++++++++++++- pipes/WDL/workflows/demux_deplete.wdl | 1 + pipes/WDL/workflows/sarscov2_illumina_full.wdl | 6 +++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index be9e7e8de..51901d53d 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -5,6 +5,9 @@ task upload_entities_tsv{ String workspace_name String terra_project File tsv_file + Array[File] cleaned_reads_unaligned_bams + Array[String] cleaned_reads_unaligned_bams_string + File meta_by_filename_json String docker="schaluvadi/pathogen-genomic-surveillance:api-wdl" } @@ -13,7 +16,15 @@ task upload_entities_tsv{ python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ -p ~{terra_project} \ - -w ~{workspace_name} + -w ~{workspace_name} \ + -b ~{sep="," cleaned_reads_unaligned_bams} \ + -j ~{meta_by_filename_json} + + python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ + -p ~{terra_project} \ + -w ~{workspace_name} \ + -b ~{sep="," cleaned_reads_unaligned_bams_string} \ + -j ~{meta_by_filename_json} } diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index 8afa436dd..461b5c0b6 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -148,6 +148,7 @@ workflow demux_deplete { File meta_by_sample_json = meta_sample.merged_json Array[File] cleaned_reads_unaligned_bams = select_all(cleaned_bam_passing) + Array[String] cleaned_reads_unaligned_bams_string = select_all(cleaned_bam_passing) Array[File] cleaned_bams_tiny = select_all(empty_bam) Array[Int] read_counts_depleted = deplete.depletion_read_count_post diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 38a39cbab..dcd4c1935 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -241,12 +241,16 @@ workflow sarscov2_illumina_full { input: workspace_name = workspace_name, terra_project = terra_project, - tsv_file = assembly_meta_tsv.combined + tsv_file = assembly_meta_tsv.combined, + cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams, + cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams_string, + meta_by_filename_json = demux_deplete.meta_by_filename_json } output { Array[File] raw_reads_unaligned_bams = demux_deplete.raw_reads_unaligned_bams Array[File] cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams + Array[String] cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams_string Array[File] cleaned_bams_tiny = demux_deplete.cleaned_bams_tiny File meta_by_filename_json = demux_deplete.meta_by_filename_json From 6e63acf4b37e943f0c01459a7644c4faf0743228 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 11:17:59 -0500 Subject: [PATCH 05/19] add variable memory to data tables task --- pipes/WDL/tasks/tasks_call_api.wdl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 51901d53d..6bf556c04 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -10,6 +10,7 @@ task upload_entities_tsv{ File meta_by_filename_json String docker="schaluvadi/pathogen-genomic-surveillance:api-wdl" + Int? machine_mem_gb } command { @@ -31,6 +32,7 @@ task upload_entities_tsv{ runtime { docker: docker preemptible: 0 + memory: select_first([machine_mem_gb, 10]) + " GB" } output { From f5fc2d22ea600ef703e2dcade67bcf42e06bf933 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 11:18:36 -0500 Subject: [PATCH 06/19] remove array[file] --- pipes/WDL/tasks/tasks_call_api.wdl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 6bf556c04..1d837eece 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -15,11 +15,11 @@ task upload_entities_tsv{ command { - python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ - -p ~{terra_project} \ - -w ~{workspace_name} \ - -b ~{sep="," cleaned_reads_unaligned_bams} \ - -j ~{meta_by_filename_json} + # python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ + # -p ~{terra_project} \ + # -w ~{workspace_name} \ + # -b ~{sep="," cleaned_reads_unaligned_bams} \ + # -j ~{meta_by_filename_json} python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ -p ~{terra_project} \ From 35198c475e44e629f27d24d83f2771066b0ccb99 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 11:20:06 -0500 Subject: [PATCH 07/19] remove instances of array[file] from all wdls - testing --- pipes/WDL/tasks/tasks_call_api.wdl | 2 +- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 1d837eece..54ed2783c 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -5,7 +5,7 @@ task upload_entities_tsv{ String workspace_name String terra_project File tsv_file - Array[File] cleaned_reads_unaligned_bams + # Array[File] cleaned_reads_unaligned_bams Array[String] cleaned_reads_unaligned_bams_string File meta_by_filename_json diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index dcd4c1935..ce04bf072 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -242,7 +242,7 @@ workflow sarscov2_illumina_full { workspace_name = workspace_name, terra_project = terra_project, tsv_file = assembly_meta_tsv.combined, - cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams, + # cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams, cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams_string, meta_by_filename_json = demux_deplete.meta_by_filename_json } From 06edede0286b5ffff4974b15816d1cd511f85991 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 11:33:41 -0500 Subject: [PATCH 08/19] change command variable --- pipes/WDL/tasks/tasks_call_api.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 54ed2783c..8121ed591 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -18,7 +18,7 @@ task upload_entities_tsv{ # python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ # -p ~{terra_project} \ # -w ~{workspace_name} \ - # -b ~{sep="," cleaned_reads_unaligned_bams} \ + # -b ~{sep="," cleaned_reads_unaligned_bams_string} \ # -j ~{meta_by_filename_json} python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ From bf1c2f4cdcdc688e4dfdd3c236fb406b808d1b7f Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 11:36:53 -0500 Subject: [PATCH 09/19] edit var names --- pipes/WDL/tasks/tasks_call_api.wdl | 7 ------- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 1 - 2 files changed, 8 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 8121ed591..4db8a1f1f 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -5,7 +5,6 @@ task upload_entities_tsv{ String workspace_name String terra_project File tsv_file - # Array[File] cleaned_reads_unaligned_bams Array[String] cleaned_reads_unaligned_bams_string File meta_by_filename_json @@ -15,12 +14,6 @@ task upload_entities_tsv{ command { - # python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ - # -p ~{terra_project} \ - # -w ~{workspace_name} \ - # -b ~{sep="," cleaned_reads_unaligned_bams_string} \ - # -j ~{meta_by_filename_json} - python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ -p ~{terra_project} \ -w ~{workspace_name} \ diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index ce04bf072..0866ba882 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -242,7 +242,6 @@ workflow sarscov2_illumina_full { workspace_name = workspace_name, terra_project = terra_project, tsv_file = assembly_meta_tsv.combined, - # cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams, cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams_string, meta_by_filename_json = demux_deplete.meta_by_filename_json } From 6e54eb32f74e736848799af4ad0fdf18884be2b3 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Thu, 4 Feb 2021 16:15:21 -0500 Subject: [PATCH 10/19] read cleaned bams as string --- pipes/WDL/tasks/tasks_call_api.wdl | 4 +++- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 4db8a1f1f..0e43572f2 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -13,11 +13,13 @@ task upload_entities_tsv{ } command { + + echo ~{sep="," cleaned_reads_unaligned_bams_string} > cleaned_bam_strings.txt python3 /projects/cdc-sabeti-covid-19/create_data_tables.py -t ~{tsv_file} \ -p ~{terra_project} \ -w ~{workspace_name} \ - -b ~{sep="," cleaned_reads_unaligned_bams_string} \ + -b cleaned_bam_strings.txt \ -j ~{meta_by_filename_json} } diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 0866ba882..bcf0c4d8a 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -249,7 +249,6 @@ workflow sarscov2_illumina_full { output { Array[File] raw_reads_unaligned_bams = demux_deplete.raw_reads_unaligned_bams Array[File] cleaned_reads_unaligned_bams = demux_deplete.cleaned_reads_unaligned_bams - Array[String] cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams_string Array[File] cleaned_bams_tiny = demux_deplete.cleaned_bams_tiny File meta_by_filename_json = demux_deplete.meta_by_filename_json From 59e38d4fac44696e5c1c646576e312f138521b62 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Fri, 5 Feb 2021 13:15:27 -0500 Subject: [PATCH 11/19] test coerce array[file] into array[string] input --- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index bcf0c4d8a..6963db4d1 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -242,7 +242,7 @@ workflow sarscov2_illumina_full { workspace_name = workspace_name, terra_project = terra_project, tsv_file = assembly_meta_tsv.combined, - cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams_string, + cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, meta_by_filename_json = demux_deplete.meta_by_filename_json } From 9f28a3819be47cb22c6dfc65379622f2d8cc2233 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Fri, 5 Feb 2021 13:56:24 -0500 Subject: [PATCH 12/19] remove array[string] dup of cleaned reads --- pipes/WDL/workflows/demux_deplete.wdl | 1 - 1 file changed, 1 deletion(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index 461b5c0b6..8afa436dd 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -148,7 +148,6 @@ workflow demux_deplete { File meta_by_sample_json = meta_sample.merged_json Array[File] cleaned_reads_unaligned_bams = select_all(cleaned_bam_passing) - Array[String] cleaned_reads_unaligned_bams_string = select_all(cleaned_bam_passing) Array[File] cleaned_bams_tiny = select_all(empty_bam) Array[Int] read_counts_depleted = deplete.depletion_read_count_post From 0bcd1b877be6df3d0d87bcf2330b7b42c5a79357 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:00:16 -0500 Subject: [PATCH 13/19] change task mem, make terra vars optional for api task --- pipes/WDL/tasks/tasks_call_api.wdl | 2 +- .../WDL/workflows/sarscov2_illumina_full.wdl | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 0e43572f2..0ec27c7ee 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -27,7 +27,7 @@ task upload_entities_tsv{ runtime { docker: docker preemptible: 0 - memory: select_first([machine_mem_gb, 10]) + " GB" + memory: select_first([machine_mem_gb, 2]) + " GB" } output { diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 6963db4d1..ba758c182 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -50,8 +50,8 @@ workflow sarscov2_illumina_full { Int min_genome_bases = 15000 - String workspace_name - String terra_project + String? workspace_name + String? terra_project } Int taxid = 2697049 String gisaid_prefix = 'hCoV-19/' @@ -236,14 +236,16 @@ workflow sarscov2_illumina_full { out_name = "gisaid-meta-~{flowcell_id}.tsv" } - # create data tables with assembly_meta_tsv - call fapi_tables.upload_entities_tsv as data_tables { - input: - workspace_name = workspace_name, - terra_project = terra_project, - tsv_file = assembly_meta_tsv.combined, - cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, - meta_by_filename_json = demux_deplete.meta_by_filename_json + # create data tables with assembly_meta_tsv if workspace name and project provided + if(defined(workspace_name) && defined(terra_project)) { + call fapi_tables.upload_entities_tsv as data_tables { + input: + workspace_name = workspace_name, + terra_project = terra_project, + tsv_file = assembly_meta_tsv.combined, + cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, + meta_by_filename_json = demux_deplete.meta_by_filename_json + } } output { From 8e381f460bac21ef5d38b9fe25539e8c32498b51 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:09:37 -0500 Subject: [PATCH 14/19] optional workflow inputs --- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index ba758c182..08b0733de 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -50,8 +50,8 @@ workflow sarscov2_illumina_full { Int min_genome_bases = 15000 - String? workspace_name - String? terra_project + String? workspace_name + String? terra_project } Int taxid = 2697049 String gisaid_prefix = 'hCoV-19/' From c3e9a19da3270b74e880db907e53f65b32e0e227 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:11:02 -0500 Subject: [PATCH 15/19] test optional --- .../WDL/workflows/sarscov2_illumina_full.wdl | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 08b0733de..ab4ff643f 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -237,16 +237,16 @@ workflow sarscov2_illumina_full { } # create data tables with assembly_meta_tsv if workspace name and project provided - if(defined(workspace_name) && defined(terra_project)) { - call fapi_tables.upload_entities_tsv as data_tables { - input: - workspace_name = workspace_name, - terra_project = terra_project, - tsv_file = assembly_meta_tsv.combined, - cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, - meta_by_filename_json = demux_deplete.meta_by_filename_json - } - } + # if(defined(workspace_name) && defined(terra_project)) { + # call fapi_tables.upload_entities_tsv as data_tables { + # input: + # workspace_name = workspace_name, + # terra_project = terra_project, + # tsv_file = assembly_meta_tsv.combined, + # cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, + # meta_by_filename_json = demux_deplete.meta_by_filename_json + # } + # } output { Array[File] raw_reads_unaligned_bams = demux_deplete.raw_reads_unaligned_bams From 8a9230d439e825a6bba5dbaf9ae37234994c1371 Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:14:04 -0500 Subject: [PATCH 16/19] troubleshoot --- .../WDL/workflows/sarscov2_illumina_full.wdl | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index ab4ff643f..36e524e79 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -50,8 +50,8 @@ workflow sarscov2_illumina_full { Int min_genome_bases = 15000 - String? workspace_name - String? terra_project + String workspace_name + String terra_project } Int taxid = 2697049 String gisaid_prefix = 'hCoV-19/' @@ -236,17 +236,17 @@ workflow sarscov2_illumina_full { out_name = "gisaid-meta-~{flowcell_id}.tsv" } - # create data tables with assembly_meta_tsv if workspace name and project provided - # if(defined(workspace_name) && defined(terra_project)) { - # call fapi_tables.upload_entities_tsv as data_tables { - # input: - # workspace_name = workspace_name, - # terra_project = terra_project, - # tsv_file = assembly_meta_tsv.combined, - # cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, - # meta_by_filename_json = demux_deplete.meta_by_filename_json - # } - # } + create data tables with assembly_meta_tsv if workspace name and project provided + if(defined(workspace_name) && defined(terra_project)) { + call fapi_tables.upload_entities_tsv as data_tables { + input: + workspace_name = workspace_name, + terra_project = terra_project, + tsv_file = assembly_meta_tsv.combined, + cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, + meta_by_filename_json = demux_deplete.meta_by_filename_json + } + } output { Array[File] raw_reads_unaligned_bams = demux_deplete.raw_reads_unaligned_bams From 5befab86243751e7c58b9505f174a083b42db61e Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:16:48 -0500 Subject: [PATCH 17/19] updated api_wdl output as optional --- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 36e524e79..224d3c65f 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -50,8 +50,8 @@ workflow sarscov2_illumina_full { Int min_genome_bases = 15000 - String workspace_name - String terra_project + String? workspace_name + String? terra_project } Int taxid = 2697049 String gisaid_prefix = 'hCoV-19/' @@ -236,8 +236,8 @@ workflow sarscov2_illumina_full { out_name = "gisaid-meta-~{flowcell_id}.tsv" } - create data tables with assembly_meta_tsv if workspace name and project provided - if(defined(workspace_name) && defined(terra_project)) { + # create data tables with assembly_meta_tsv if workspace name and project provided + if (defined(workspace_name) && defined(terra_project)) { call fapi_tables.upload_entities_tsv as data_tables { input: workspace_name = workspace_name, @@ -296,6 +296,6 @@ workflow sarscov2_illumina_full { Int num_submittable = length(select_all(submittable_id)) Int num_failed_annotation = length(select_all(failed_annotation_id)) Int num_samples = length(group_bams_by_sample.sample_names) - String data_table_status = data_tables.status + String? data_table_status = data_tables.status } } From 611eee6a00923b8c466f85da4adf8581511881fb Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:25:05 -0500 Subject: [PATCH 18/19] optional inputs to just task --- pipes/WDL/tasks/tasks_call_api.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index 0ec27c7ee..e820a72ba 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -2,8 +2,8 @@ version 1.0 task upload_entities_tsv{ input { - String workspace_name - String terra_project + String? workspace_name + String? terra_project File tsv_file Array[String] cleaned_reads_unaligned_bams_string File meta_by_filename_json From b464d45a7b19adc8daf7d674510718d104c31f4b Mon Sep 17 00:00:00 2001 From: Sushma Chaluvadi Date: Mon, 8 Feb 2021 09:53:29 -0500 Subject: [PATCH 19/19] fix select_first for opt inputs to workflow --- pipes/WDL/tasks/tasks_call_api.wdl | 4 ++-- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipes/WDL/tasks/tasks_call_api.wdl b/pipes/WDL/tasks/tasks_call_api.wdl index e820a72ba..0ec27c7ee 100644 --- a/pipes/WDL/tasks/tasks_call_api.wdl +++ b/pipes/WDL/tasks/tasks_call_api.wdl @@ -2,8 +2,8 @@ version 1.0 task upload_entities_tsv{ input { - String? workspace_name - String? terra_project + String workspace_name + String terra_project File tsv_file Array[String] cleaned_reads_unaligned_bams_string File meta_by_filename_json diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 224d3c65f..73f068f8e 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -240,8 +240,8 @@ workflow sarscov2_illumina_full { if (defined(workspace_name) && defined(terra_project)) { call fapi_tables.upload_entities_tsv as data_tables { input: - workspace_name = workspace_name, - terra_project = terra_project, + workspace_name = select_first([workspace_name]), + terra_project = select_first([terra_project]), tsv_file = assembly_meta_tsv.combined, cleaned_reads_unaligned_bams_string = demux_deplete.cleaned_reads_unaligned_bams, meta_by_filename_json = demux_deplete.meta_by_filename_json