From 9035a2259667207a2de1e58dccd4326bac8d8929 Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Tue, 19 Nov 2024 19:27:55 +0200 Subject: [PATCH 1/2] tests: Improve logging in test_purge_db() --- test_main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test_main.py b/test_main.py index 0ac3ec2f..066378c1 100644 --- a/test_main.py +++ b/test_main.py @@ -333,7 +333,8 @@ def filter_test_data(data): for obj_list_name in min_io_version.graph: if obj_list_name: assert len(dump.get(obj_list_name, [])) == 2, \ - f"Invalid number of {obj_list_name}" + f"Invalid number of {obj_list_name} in " \ + f"{database} database" # Trigger the purge at the boundary publisher.publish( @@ -355,7 +356,7 @@ def filter_test_data(data): break assert dump == client.get_schema()[1].upgrade( data_after if purging else data - ) + ), "Unexpected data in {database} database" def test_archive(empty_deployment): From b6c237ca9785a0aaea9d98c349cc5a4d17a3c7cd Mon Sep 17 00:00:00 2001 From: Nikolai Kondrashov Date: Tue, 19 Nov 2024 19:28:39 +0200 Subject: [PATCH 2/2] bigquery: Disable cache when dumping Disable query caching when dumping data from BigQuery in an attempt to fix the database purging test (test_purge_db()). Revert if there's no effect on test stability. --- kcidb/db/bigquery/v04_00.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kcidb/db/bigquery/v04_00.py b/kcidb/db/bigquery/v04_00.py index cf25d694..f417df05 100644 --- a/kcidb/db/bigquery/v04_00.py +++ b/kcidb/db/bigquery/v04_00.py @@ -79,7 +79,8 @@ def __init__(self, params): except GoogleNotFound as exc: raise NotFound(params) from exc - def query_create(self, query_string, query_parameters=None): + def query_create(self, query_string, query_parameters=None, + use_query_cache=True): """ Creates a Query job configured for a given query string and optional parameters. BigQuery can run the job to query the database. @@ -89,6 +90,8 @@ def query_create(self, query_string, query_parameters=None): query_parameters: A list containing the optional query parameters (google.cloud.bigquery.ArrayQueryParameter). The default is an empty list. + use_query_cache: True if BigQuery query cache should be used, + False otherwise. Returns: The Query job (google.cloud.bigquery.job.QueryJob) @@ -97,7 +100,10 @@ def query_create(self, query_string, query_parameters=None): query_parameters = [] LOGGER.debug("Query string: %s", query_string) LOGGER.debug("Query params: %s", query_parameters) + if not use_query_cache: + LOGGER.debug("Query cache: DISABLED") job_config = bigquery.job.QueryJobConfig( + use_query_cache=use_query_cache, query_parameters=query_parameters, default_dataset=self.dataset_ref) return self.client.query(query_string, job_config=job_config) @@ -806,7 +812,8 @@ def dump_iter(self, objects_per_report, with_metadata, after, until): bigquery.ScalarQueryParameter(None, ts_field.field_type, v) for v in (table_after, table_until) if v ] - query_job = self.conn.query_create(query_string, query_parameters) + query_job = self.conn.query_create(query_string, query_parameters, + use_query_cache=False) obj_list = None for row in query_job: if obj_list is None: