From 1feb4a010851441e36ab5411220bfe3dbd231991 Mon Sep 17 00:00:00 2001 From: Tomas Votava Date: Tue, 25 Jul 2023 10:11:35 +0200 Subject: [PATCH] feat: check for bucket's existence before creation This get-or-create approach, contrary to the original create-or-fail, does not require the caller to have storage.buckets.create permission in cases when the bucket already exists. --- target_bigquery/gcs_stage.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/target_bigquery/gcs_stage.py b/target_bigquery/gcs_stage.py index cfadbc0..727fd83 100644 --- a/target_bigquery/gcs_stage.py +++ b/target_bigquery/gcs_stage.py @@ -216,18 +216,16 @@ def create_bucket_if_not_exists(self) -> storage.Bucket: location: str = self.config.get("location", self.default_bucket_options()["location"]) if not hasattr(self, "_gcs_bucket"): - try: - self._gcs_bucket = self.client.create_bucket(self.as_bucket(), location=location) - except Conflict: - gcs_bucket = self.client.get_bucket(self.as_bucket()) - if gcs_bucket.location.lower() != location.lower(): + self._gcs_bucket = self.client.get_bucket(self.as_bucket()) + if self._gcs_bucket is not None: + if self._gcs_bucket.location.lower() != location.lower(): raise Exception( "Location of existing GCS bucket " - f"{self.bucket_name} ({gcs_bucket.location.lower()}) does not match " + f"{self.bucket_name} ({self._gcs_bucket.location.lower()}) does not match " f"specified location: {location}" ) - else: - self._gcs_bucket = gcs_bucket + else: + self._gcs_bucket = self.client.create_bucket(self.as_bucket(), location=location) else: # Wait for eventual consistency time.sleep(5)