Skip to content

Commit

Permalink
Cleaning up code
Browse files Browse the repository at this point in the history
Editing cdode according to the comments
  • Loading branch information
Marianie-Simeon committed Mar 9, 2021
1 parent 60972a6 commit ee01670
Show file tree
Hide file tree
Showing 6 changed files with 18 additions and 199 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import org.apache.logging.log4j.Logger;
import org.broadinstitute.hellbender.utils.bigquery.BigQueryUtils;
import org.broadinstitute.hellbender.utils.bigquery.TableReference;
import org.broadinstitute.hellbender.utils.bigquery.UID;


public class SampleList {
static final Logger logger = LogManager.getLogger(SampleList.class);
Expand Down Expand Up @@ -120,7 +120,7 @@ private TableResult querySampleTable(String fqSampleTableName, String whereClaus

// Create labels
private TableResult querySampleTable(String fqSampleTableName, String whereClause, boolean printDebugInformation) {
UID run_uid = new UID();
UUID run_uid = UUID.randomUUID();
Map<String, String> labels = new HashMap<String, String>();
labels.put("query", "Run_SampleTable_" + run_uid.toString() );
TableResult result = querySampleTable(fqSampleTableName, whereClause, printDebugInformation, labels);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public void traverse() {
logger.debug("using query api with order by");
}
// create the query string
UID run_uid = new UID();
UUID run_uid = UUID.randomUUID();
Map<String, String> labels = new HashMap<String, String>();
labels.put("query", "extract_cohort_"+ run_uid.toString());
String q = "SELECT " + StringUtils.join(SchemaUtils.COHORT_FIELDS,",") + " FROM " + cohortTableRef.getFQTableName() + " ORDER BY " + SchemaUtils.LOCATION_FIELD_NAME;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public void traverse() {
final String featureQueryString =

ExtractFeaturesBQ.getVQSRFeatureExtractQueryString(altAlleleTable, sampleListTable, minLocation, maxLocation, trainingSitesOnly, SNP_QUAL_THRESHOLD, INDEL_QUAL_THRESHOLD);
UID run_uid = new UID();
UUID run_uid = UUID.randomUUID();
Map<String, String> labels = new HashMap<String, String>();
labels.put("query", "extract_features_"+ run_uid.toString());
logger.info(featureQueryString);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ public static BigQuery getBigQueryEndPoint() {
* Will block until results are returned.
* For more information on querying BigQuery tables, see: https://cloud.google.com/bigquery/sql-reference/
* @param queryString The {@link BigQuery} query string to execute. Must use standard SQL syntax. Must contain the project ID, data set, and table name in the `FROM` clause for the table from which to retrieve data.
* @param labels The {@link BigQuery} label to add the job run. Must use Map<String, String>. Must contain Job label and gatk_execution_id.
* @return A {@link TableResult} object containing the results of the query executed.
*/
public static TableResult executeQuery(final String queryString, final Map<String, String> labels) {
Expand All @@ -56,15 +57,11 @@ public static TableResult executeQuery(final String queryString, final Map<Strin
* Will block until results are returned.
* For more information on querying BigQuery tables, see: https://cloud.google.com/bigquery/sql-reference/
* @param queryString The {@link BigQuery} query string to execute. Must use standard SQL syntax. Must contain the project ID, data set, and table name in the `FROM` clause for the table from which to retrieve data.
* @param labels The {@link BigQuery} label to add the job run. Must use Map<String, String>. Must contain Job label and gatk_execution_id.
* @param runQueryInBatchMode If true, run the query in batch mode, which is lower priority but has no limit on the number of concurrent queries
* @return A {@link TableResult} object containing the results of the query executed.
*/

// TODO: add Collections.EMPTY_MAP
// public static TableResult executeQuery(final String queryString, final boolean runQueryInBatchMode) {
// return executeQuery(getBigQueryEndPoint(), queryString, runQueryInBatchMode, Collections.EMPTY_MAP);
// }

public static TableResult executeQuery(final String queryString, final boolean runQueryInBatchMode, final Map<String, String> labels) {
return executeQuery(getBigQueryEndPoint(), queryString, runQueryInBatchMode, labels);
}
Expand Down Expand Up @@ -360,14 +357,13 @@ private static TableResult submitQueryAndWaitForResults( final BigQuery bigQuery
return result;
}

private static long getQueryCostBytesProcessedEstimate(String queryString, Map<String, String> labels) {
private static long getQueryCostBytesProcessedEstimate(String queryString) {
final QueryJobConfiguration dryRunQueryConfig =
QueryJobConfiguration.newBuilder( queryString )
.setUseLegacySql(false)
.setDryRun(true)
.setUseQueryCache(false)
.setPriority(QueryJobConfiguration.Priority.INTERACTIVE)
.setLabels(labels)
.build();

Job dryRunJob = getBigQueryEndPoint().create(JobInfo.newBuilder(dryRunQueryConfig).build());
Expand All @@ -385,7 +381,7 @@ public static StorageAPIAvroReader executeQueryWithStorageAPI(final String query
final String tempTableName = UUID.randomUUID().toString().replace('-', '_');
final String tempTableFullyQualified = String.format("%s.%s.%s", projectID, tempTableDataset, tempTableName);

long bytesProcessed = getQueryCostBytesProcessedEstimate(queryString, labels);
long bytesProcessed = getQueryCostBytesProcessedEstimate(queryString);
logger.info(String.format("Estimated %s MB scanned", bytesProcessed/1000000));

final String queryStringIntoTempTable = "CREATE TABLE `" + tempTableFullyQualified + "`\n" +
Expand All @@ -394,8 +390,6 @@ public static StorageAPIAvroReader executeQueryWithStorageAPI(final String query
") AS\n" +
queryString;

// TODO: add label

executeQuery(queryStringIntoTempTable, runQueryInBatchMode, labels);

final Table tableInfo = getBigQueryEndPoint().getTable( TableId.of(projectID, tempTableDataset, tempTableName) );
Expand Down
183 changes: 0 additions & 183 deletions src/main/java/org/broadinstitute/hellbender/utils/bigquery/UID.java

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,12 @@ public class BigQueryUtilsUnitTest extends GATKBaseTest {
private static final String BIGQUERY_FULLY_QUALIFIED_TABLE = String.format("%s.%s.%s",
BIGQUERY_TEST_PROJECT, BIGQUERY_TEST_DATASET, BIGQUERY_TEST_TABLE);

private static final UID run_uid = new UID();
private static final UUID run_uid = UUID.randomUUID();

@Test(groups = {"cloud"})
public void testExecuteQueryAllRecords() {
final String query = String.format("SELECT * FROM `%s`", BIGQUERY_FULLY_QUALIFIED_TABLE);
Map<String, String> labels = new HashMap<String, String>();
String test = run_uid.toString();
labels.put("test_query", "get_all_records" + run_uid.toString() );
final TableResult result = BigQueryUtils.executeQuery(query, labels);

Expand Down Expand Up @@ -123,6 +122,15 @@ public void testQueryWithEmptyDatasetStorageAPI() {
Assert.assertTrue(retrievedNames.isEmpty(), "No Result expected");
}

@Test(groups = {"cloud"})
public void testQueryWithNullLabel() {
final String query = String.format("SELECT * FROM `%s`", BIGQUERY_FULLY_QUALIFIED_TABLE);
Map<String, String> labels = null;
final TableResult result = BigQueryUtils.executeQuery(query, labels);

checkQueryResults(result, getAllExpectedNamesAndAges(), query);
}

private Map<String, String> getAllExpectedNamesAndAges() {
final Map<String, String> expectedNamesAndAges = new HashMap<>();
expectedNamesAndAges.put("Fred", "35");
Expand Down

0 comments on commit ee01670

Please sign in to comment.