From 4ce834cc5ef44a296b366435fcda74ed0f708de8 Mon Sep 17 00:00:00 2001 From: Rebecca Asch Date: Tue, 12 Jul 2022 14:14:44 -0400 Subject: [PATCH 1/6] first stab --- .../wdl/GvsQuickstartIntegration.wdl | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl index 206ae4c71de..365dc223155 100644 --- a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl +++ b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl @@ -89,6 +89,14 @@ workflow GvsQuickstartIntegration { expected_output_csv = expected_output_prefix + "cost_observability_expected.csv" } + call AssertTableSizesAreExpected { + input: + go = GvsUnified.done, + dataset_name = BuildGATKJarAndCreateDataset.dataset_name, + project_id = project_id, + expected_output_csv = "gs://gvs-internal-quickstart/integration/2022-07-05/table_sizes_expected.csv" + } + output { Array[File] output_vcfs = GvsUnified.output_vcfs Array[File] output_vcf_indexes = GvsUnified.output_vcf_indexes @@ -283,6 +291,57 @@ task AssertCostIsTrackedAndExpected { } output { - File cost_observability_output_csv = "cost_observability_output.csv" + File cost_observability_output_csv = "cost_observability_output.csv" + } +} + +task AssertTableSizesAreExpected { + meta { + # we want to check the database each time this runs + volatile: true + } + + input { + Boolean go = true + String dataset_name + String project_id + File expected_output_csv + } + + command <<< + set -o errexit + set -o nounset + set -o pipefail + set -o xtrace + + echo "project_id = ~{project_id}" > ~/.bigqueryrc + bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false \ + "SELECT 'vet_total' AS total_name, sum(total_billable_bytes) AS total_bytes + FROM \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` + WHERE table_name LIKE 'vet_%' + UNION ALL + SELECT 'ref_ranges_total' AS total_name, sum(total_billable_bytes) AS total_bytes + FROM \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` + WHERE table_name LIKE 'ref_ranges_%'" > table_size_output.csv + + set +o errexit + diff -w table_size_output.csv ~{expected_output_csv} > differences.txt + set -o errexit + + if [[ -s differences.txt ]]; then + echo "Differences found:" + cat differences.txt + exit 1 + fi + >>> + + runtime { + docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:latest" + disks: "local-disk 10 HDD" + } + + output { + File table_size_output_csv = "cost_observability_output.csv" + File differences = "differences.txt" } } From 25708dce053fc66f0fd014612b1fbe766184d478 Mon Sep 17 00:00:00 2001 From: Rebecca Asch Date: Tue, 12 Jul 2022 14:17:07 -0400 Subject: [PATCH 2/6] dockstore --- .dockstore.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.dockstore.yml b/.dockstore.yml index ac75c9358f6..fb7a171312d 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -215,8 +215,7 @@ workflows: branches: - master - ah_var_store - - rsa_metadata_from_python - - gg_VS-492_BetaUserJarRelease + - rsa_vs_501_add_size_check - name: GvsIngestTieout subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsIngestTieout.wdl From bcb7654e2e6f28e1d6432e530daf57016f4ade5d Mon Sep 17 00:00:00 2001 From: Rebecca Asch Date: Tue, 12 Jul 2022 19:23:07 -0400 Subject: [PATCH 3/6] fixes --- .../variantstore/wdl/GvsQuickstartIntegration.wdl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl index 365dc223155..1a452cd154d 100644 --- a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl +++ b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl @@ -316,12 +316,11 @@ task AssertTableSizesAreExpected { echo "project_id = ~{project_id}" > ~/.bigqueryrc bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false \ - "SELECT 'vet_total' AS total_name, sum(total_billable_bytes) AS total_bytes - FROM \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` - WHERE table_name LIKE 'vet_%' - UNION ALL - SELECT 'ref_ranges_total' AS total_name, sum(total_billable_bytes) AS total_bytes - FROM \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` + "SELECT 'vet_total' AS total_name, sum(total_billable_bytes) AS total_bytes FROM \ + `~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` WHERE table_name LIKE 'vet_%' \ + UNION ALL \ + SELECT 'ref_ranges_total' AS total_name, sum(total_billable_bytes) AS total_bytes \ + FROM \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` \ WHERE table_name LIKE 'ref_ranges_%'" > table_size_output.csv set +o errexit @@ -341,7 +340,7 @@ task AssertTableSizesAreExpected { } output { - File table_size_output_csv = "cost_observability_output.csv" + File table_size_output_csv = "table_size_output.csv" File differences = "differences.txt" } } From 3dec6c6a89a07738867b3ca1ed1e3222d09f52ad Mon Sep 17 00:00:00 2001 From: Rebecca Asch Date: Tue, 2 Aug 2022 08:42:37 -0400 Subject: [PATCH 4/6] missed an escaping --- scripts/variantstore/wdl/GvsQuickstartIntegration.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl index 1a452cd154d..d996010a968 100644 --- a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl +++ b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl @@ -317,7 +317,7 @@ task AssertTableSizesAreExpected { echo "project_id = ~{project_id}" > ~/.bigqueryrc bq query --location=US --project_id=~{project_id} --format=csv --use_legacy_sql=false \ "SELECT 'vet_total' AS total_name, sum(total_billable_bytes) AS total_bytes FROM \ - `~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` WHERE table_name LIKE 'vet_%' \ + \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` WHERE table_name LIKE 'vet_%' \ UNION ALL \ SELECT 'ref_ranges_total' AS total_name, sum(total_billable_bytes) AS total_bytes \ FROM \`~{dataset_name}.INFORMATION_SCHEMA.PARTITIONS\` \ From 964bd67da486a68a0ab1856ecc19251489f2e7b8 Mon Sep 17 00:00:00 2001 From: Rebecca Asch Date: Wed, 3 Aug 2022 11:30:03 -0400 Subject: [PATCH 5/6] PR feedback --- scripts/variantstore/wdl/GvsQuickstartIntegration.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl index d996010a968..2d53a1260de 100644 --- a/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl +++ b/scripts/variantstore/wdl/GvsQuickstartIntegration.wdl @@ -94,7 +94,7 @@ workflow GvsQuickstartIntegration { go = GvsUnified.done, dataset_name = BuildGATKJarAndCreateDataset.dataset_name, project_id = project_id, - expected_output_csv = "gs://gvs-internal-quickstart/integration/2022-07-05/table_sizes_expected.csv" + expected_output_csv = expected_output_prefix + "table_sizes_expected.csv" } output { From 8712dcda33bdf4985ce29f70baf129651790e276 Mon Sep 17 00:00:00 2001 From: Rebecca Asch Date: Wed, 3 Aug 2022 12:33:20 -0400 Subject: [PATCH 6/6] clean up dockstore yaml --- .dockstore.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.dockstore.yml b/.dockstore.yml index b00df978eb5..7579acf33fd 100644 --- a/.dockstore.yml +++ b/.dockstore.yml @@ -65,7 +65,6 @@ workflows: branches: - master - ah_var_store - - rsa_metadata_from_python - name: GvsAoUReblockGvcf subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsAoUReblockGvcf.wdl @@ -98,7 +97,6 @@ workflows: branches: - master - ah_var_store - - rsa_metadata_from_python - name: GvsCreateTables subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateTables.wdl @@ -132,7 +130,6 @@ workflows: branches: - master - ah_var_store - - rsa_metadata_from_python - name: GvsCreateVAT subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateVAT.wdl @@ -191,7 +188,6 @@ workflows: branches: - master - ah_var_store - - rsa_metadata_from_python - gg_VS-492_BetaUserJarRelease - name: GvsJointVariantCalling subclass: WDL @@ -222,7 +218,6 @@ workflows: branches: - master - ah_var_store - - rsa_vs_501_add_size_check - name: GvsIngestTieout subclass: WDL primaryDescriptorPath: /scripts/variantstore/wdl/GvsIngestTieout.wdl