added job ID to alt_allele population call output [VS-194] (#7473)

broadinstitute · Sep 17, 2021 · d1d0a36 · d1d0a36
1 parent 185b5f4
commit d1d0a36
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 5 deletions.
diff --git a/.dockstore.yml b/.dockstore.yml
@@ -83,6 +83,7 @@ workflows:
        branches:
          - master
          - ah_var_store
+         - rsa_alt_allele_job_id
    - name: GvsExtractCallset
      subclass: WDL
      primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallset.wdl

diff --git a/scripts/variantstore/wdl/GvsCreateAltAllele.wdl b/scripts/variantstore/wdl/GvsCreateAltAllele.wdl
@@ -193,7 +193,7 @@ task PopulateAltAlleleTable {
   }
 
   runtime {
-    docker: "us.gcr.io/broad-dsde-methods/variantstore:ah_var_store_20210903"
+    docker: "us.gcr.io/broad-dsde-methods/variantstore:ah_var_store_20210917"
     memory: "3 GB"
     disks: "local-disk 10 HDD"
     cpu: 1

diff --git a/scripts/variantstore/wdl/extract/populate_alt_allele_table.py b/scripts/variantstore/wdl/extract/populate_alt_allele_table.py
@@ -16,16 +16,17 @@ def execute_with_retry(label, sql):
     while len(retry_delay) > 0:
         try:
             query_label = label.replace(" ","-").strip().lower()
-
             existing_labels = client._default_query_job_config.labels
             job_labels = existing_labels
             job_labels["gvs_query_name"] = query_label
             job_config = bigquery.QueryJobConfig(labels=job_labels)
             query = client.query(sql, job_config=job_config)
-
-            print(f"STARTING - {label}")
+            print(f"STARTING - {label} (jobid: {query.job_id})")
             results = query.result()
-            print(f"COMPLETED ({time.time() - start} s, {3-len(retry_delay)} retries) - {label}")
+            job = client.get_job(query.job_id)
+            mb_billed = int(0 if job.total_bytes_billed is None else job.total_bytes_billed)/(1024 * 1024)
+            print(f"COMPLETED ({time.time() - start} seconds, {3-len(retry_delay)} retries, {mb_billed} MBs) - {label}")
+
             return results
         except Exception as err:
             # if there are no retries left... raise