fix(bigquery/ingestor): handle quota exceeded for project.list requests

datahub-project · Jul 15, 2024 · 1520e35 · 1520e35
1 parent aa92a99
commit 1520e35
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py
@@ -173,7 +173,11 @@ def _should_retry(exc: BaseException) -> bool:
                 # to restrict the request calls inside list_project but issue still occured.
                 projects_iterator = self.bq_client.list_projects(
                     retry=retry.Retry(
-                        predicate=_should_retry, initial=10, maximum=180, timeout=900
+                        predicate=_should_retry,
+                        initial=10,
+                        maximum=180,
+                        multiplier=2,
+                        timeout=900,
                     )
                 )
                 projects: List[BigqueryProject] = [