Pass credentials explicitly to Google clients

Avoids issue documented in move-coop#1039 where credentials for all GCP clients
are stored in the same environment variable, leading to overwrites if
multiple clients are initialized in the same environment.

parsons/google/google_admin.py

@@ -1,9 +1,13 @@
-from oauth2client.service_account import ServiceAccountCredentials
-from parsons.etl.table import Table
-from parsons.google.utilities import setup_google_application_credentials
-import httplib2
 import json
-import os
+import uuid
+
+from google.auth.transport.requests import AuthorizedSession
+
+from parsons.etl.table import Table
+from parsons.google.utilities import (
+    load_google_application_credentials,
+    setup_google_application_credentials,
+)
 
 
 class GoogleAdmin(object):
@@ -23,17 +27,16 @@ class GoogleAdmin(object):
     """
 
     def __init__(self, app_creds=None, sub=None):
-        setup_google_application_credentials(app_creds)
-
-        self.client = (
-            ServiceAccountCredentials.from_json_keyfile_name(
-                os.environ["GOOGLE_APPLICATION_CREDENTIALS"],
-                ["https://www.googleapis.com/auth/admin.directory.group"],
-            )
-            .create_delegated(sub)
-            .authorize(httplib2.Http())
+        env_credentials_path = str(uuid.uuid4())
+        setup_google_application_credentials(app_creds, target_env_var_name=env_credentials_path)
+        credentials = load_google_application_credentials(
+            env_credentials_path,
+            scopes=["https://www.googleapis.com/auth/admin.directory.group"],
+            subject=sub,
         )
 
+        self.client = AuthorizedSession(credentials)
+
     def _paginate_request(self, endpoint, collection, params=None):
         # Build query params
         param_arr = []

parsons/google/google_bigquery.py

@@ -17,7 +17,10 @@
 from parsons.databases.table import BaseTable
 from parsons.etl import Table
 from parsons.google.google_cloud_storage import GoogleCloudStorage
-from parsons.google.utitities import setup_google_application_credentials
+from parsons.google.utilities import (
+    load_google_application_credentials,
+    setup_google_application_credentials,
+)
 from parsons.utilities import check_env
 from parsons.utilities.files import create_temp_file
 
@@ -160,8 +163,9 @@ def __init__(
         if isinstance(app_creds, Credentials):
             self.credentials = app_creds
         else:
-            self.credentials = None
-            setup_google_application_credentials(app_creds)
+            env_credential_path = str(uuid.uuid4())
+            setup_google_application_credentials(app_creds, target_env_var_name=env_credential_path)
+            self.credentials = load_google_application_credentials(env_credential_path)
 
         self.project = project
         self.location = location
@@ -695,7 +699,7 @@ def copy_s3(
 
         # copy from S3 to GCS
         tmp_gcs_bucket = check_env.check("GCS_TEMP_BUCKET", tmp_gcs_bucket)
-        gcs_client = gcs_client or GoogleCloudStorage()
+        gcs_client = gcs_client or GoogleCloudStorage(app_creds=self.app_creds)
         temp_blob_uri = gcs_client.copy_s3_to_gcs(
             aws_source_bucket=bucket,
             aws_access_key_id=aws_access_key_id,
@@ -808,7 +812,7 @@ def copy(
             schema.append(schema_row)
         job_config.schema = schema
 
-        gcs_client = gcs_client or GoogleCloudStorage()
+        gcs_client = gcs_client or GoogleCloudStorage(app_creds=self.app_creds)
         temp_blob_name = f"{uuid.uuid4()}.{data_type}"
         temp_blob_uri = gcs_client.upload_table(tbl, tmp_gcs_bucket, temp_blob_name)
 

parsons/google/google_cloud_storage.py

@@ -1,16 +1,20 @@
-import google
-from google.cloud import storage
-from google.cloud import storage_transfer
-from parsons.google.utilities import setup_google_application_credentials
-from parsons.utilities import files
 import datetime
+import gzip
 import logging
 import time
 import uuid
-import gzip
 import zipfile
 from typing import Optional
 
+import google
+from google.cloud import storage, storage_transfer
+
+from parsons.google.utilities import (
+    load_google_application_credentials,
+    setup_google_application_credentials,
+)
+from parsons.utilities import files
+
 logger = logging.getLogger(__name__)
 
 
@@ -41,17 +45,19 @@ class GoogleCloudStorage(object):
     """
 
     def __init__(self, app_creds=None, project=None):
-        setup_google_application_credentials(app_creds)
+        env_credentials_path = str(uuid.uuid4())
+        setup_google_application_credentials(app_creds, target_env_var_name=env_credentials_path)
+        credentials = load_google_application_credentials(env_credentials_path)
         self.project = project
 
         # Throws an error if you pass project=None, so adding if/else statement.
         if not self.project:
-            self.client = storage.Client()
+            self.client = storage.Client(credentials=credentials)
             """
             Access all methods of `google.cloud` package
             """
         else:
-            self.client = storage.Client(project=self.project)
+            self.client = storage.Client(credentials=credentials, project=self.project)
 
     def list_buckets(self):
         """

parsons/google/google_sheets.py

@@ -1,12 +1,14 @@
-import os
-import json
 import logging
-
-from parsons.etl.table import Table
-from parsons.google.utitities import setup_google_application_credentials, hexavigesimal
+import uuid
 
 import gspread
-from google.oauth2.service_account import Credentials
+
+from parsons.etl.table import Table
+from parsons.google.utilities import (
+    load_google_application_credentials,
+    setup_google_application_credentials,
+    hexavigesimal,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -32,12 +34,15 @@ def __init__(self, google_keyfile_dict=None, subject=None):
             "https://www.googleapis.com/auth/drive",
         ]
 
-        setup_google_application_credentials(google_keyfile_dict, "GOOGLE_DRIVE_CREDENTIALS")
-        google_credential_file = open(os.environ["GOOGLE_DRIVE_CREDENTIALS"])
-        credentials_dict = json.load(google_credential_file)
+        env_credentials_path = str(uuid.uuid4())
+        setup_google_application_credentials(
+            google_keyfile_dict,
+            "GOOGLE_DRIVE_CREDENTIAL",
+            target_env_var_name=env_credentials_path,
+        )
 
-        credentials = Credentials.from_service_account_info(
-            credentials_dict, scopes=scope, subject=subject
+        credentials = load_google_application_credentials(
+            env_credentials_path, scopes=scope, subject=subject
         )
 
         self.gspread_client = gspread.authorize(credentials)
@@ -47,12 +52,16 @@ def _get_worksheet(self, spreadsheet_id, worksheet=0):
 
         # Check if the worksheet is an integer, if so find the sheet by index
         if isinstance(worksheet, int):
-            return self.gspread_client.open_by_key(spreadsheet_id).get_worksheet(worksheet)
+            return self.gspread_client.open_by_key(spreadsheet_id).get_worksheet(
+                worksheet
+            )
 
         elif isinstance(worksheet, str):
             idx = self.list_worksheets(spreadsheet_id).index(worksheet)
             try:
-                return self.gspread_client.open_by_key(spreadsheet_id).get_worksheet(idx)
+                return self.gspread_client.open_by_key(spreadsheet_id).get_worksheet(
+                    idx
+                )
             except:  # noqa: E722
                 raise ValueError(f"Couldn't find worksheet {worksheet}")
 
@@ -276,7 +285,9 @@ def append_to_sheet(
 
         # If the existing sheet is blank, then just overwrite the table.
         if existing_table.num_rows == 0:
-            return self.overwrite_sheet(spreadsheet_id, table, worksheet, user_entered_value)
+            return self.overwrite_sheet(
+                spreadsheet_id, table, worksheet, user_entered_value
+            )
 
         cells = []
         for row_num, row in enumerate(table.data):