🎉 Source Google Directory: support oauth

* Source Google Directory #6265 - add oauth support * Source Google Directory #6265 - update credentials * Source Google Directory #6265 - fixing according to PR * Source Google directory #6265 - update docs * Source Google directory #5190 - update doc * Source Google Directory #6265 - resolve merge conflict * Source Google Directory #6265 - SAT for oauth * Source Google Directory #6265 - bump version and update changelog * Source Google Directory #6265 - bump version and update changelog (update publish)
airbytehq · Nov 5, 2021 · 1f295f2 · 1f295f2
1 parent 497858d
commit 1f295f2
Show file tree

Hide file tree

Showing 14 changed files with 161 additions and 29 deletions.
diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -107,6 +107,7 @@ jobs:
           GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
           GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
           GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
+          GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -102,6 +102,7 @@ jobs:
           GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
           GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
           GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
+          GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}

diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/d19ae824-e289-4b14-995a-0632eb46d246.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246",
   "name": "Google Directory",
   "dockerRepository": "airbyte/source-google-directory",
-  "dockerImageTag": "0.1.6",
+  "dockerImageTag": "0.1.8",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-directory"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -198,7 +198,7 @@
 - name: Google Directory
   sourceDefinitionId: d19ae824-e289-4b14-995a-0632eb46d246
   dockerRepository: airbyte/source-google-directory
-  dockerImageTag: 0.1.6
+  dockerImageTag: 0.1.8
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-directory
   sourceType: api
 - name: Google Search Console

diff --git a/airbyte-integrations/connectors/source-google-directory/Dockerfile b/airbyte-integrations/connectors/source-google-directory/Dockerfile
@@ -34,5 +34,5 @@ COPY source_google_directory ./source_google_directory
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.6
+LABEL io.airbyte.version=0.1.8
 LABEL io.airbyte.name=airbyte/source-google-directory
diff --git a/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml
@@ -7,16 +7,22 @@ tests:
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
+    - config_path: "integration_tests/invalid_config_oauth.json"
+      status: "failed"
   discovery:
     - config_path: "secrets/config.json"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      # API returns different lastLoginTime for some users, eteg is generated on all data and also same time different
+      # API returns different lastLoginTime for some users, eteg is generated based on all data, so also sometime are different
       ignored_fields:
         "users": ["etag", "lastLoginTime"]
diff --git a/...tegrations/connectors/source-google-directory/integration_tests/invalid_config_oauth.json b/...tegrations/connectors/source-google-directory/integration_tests/invalid_config_oauth.json
@@ -0,0 +1,7 @@
+{
+  "credentials": {
+    "client_id": "<invalid_client_id>",
+    "client_secret": "<invalid_client_secret>",
+    "refresh_token": "<invalid_refresh_token>"
+  }
+}
diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-google-directory/integration_tests/sample_config.json
@@ -1,4 +1,6 @@
 {
-  "credentials_json": "<The contents of the JSON service account key>",
-  "email": "[email protected]"
+  "credentials": {
+    "credentials_json": "<The contents of the JSON service account key>",
+    "email": "[email protected]"
+  }
 }
diff --git a/...ntegrations/connectors/source-google-directory/integration_tests/sample_config_oauth.json b/...ntegrations/connectors/source-google-directory/integration_tests/sample_config_oauth.json
@@ -0,0 +1,7 @@
+{
+  "credentials": {
+    "client_id": "<Client ID>",
+    "client_secret": "<Client secret>",
+    "refresh_token": "<Refresh token>"
+  }
+}
diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/api.py
@@ -6,10 +6,12 @@
 import json
 from abc import ABC, abstractmethod
 from functools import partial
-from typing import Callable, Dict, Iterator, Sequence
+from typing import Any, Callable, Dict, Iterator, Mapping, Sequence
 
 import backoff
+from google.auth.transport.requests import Request
 from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError as GoogleApiHttpError
 
@@ -19,20 +21,41 @@
 
 
 class API:
-    def __init__(self, credentials_json: str, email: str):
+    def __init__(self, credentials: Mapping[str, Any]):
         self._creds = None
-        self._credentials_json = credentials_json
-        self._admin_email = email
+        self._raw_credentials = credentials
         self._service = None
 
-    def _load_account_info(self) -> Dict:
-        account_info = json.loads(self._credentials_json)
+    @staticmethod
+    def _load_account_info(credentials_json: str) -> Dict:
+        account_info = json.loads(credentials_json)
         return account_info
 
-    def _obtain_creds(self):
-        account_info = self._load_account_info()
+    def _obtain_service_account_creds(self) -> service_account.Credentials:
+        """Obtaining creds based on Service account scenario"""
+        credentials_json = self._raw_credentials.get("credentials_json")
+        admin_email = self._raw_credentials.get("email")
+        account_info = self._load_account_info(credentials_json)
         creds = service_account.Credentials.from_service_account_info(account_info, scopes=SCOPES)
-        self._creds = creds.with_subject(self._admin_email)
+        self._creds = creds.with_subject(admin_email)
+
+    def _obtain_web_app_creds(self) -> Credentials:
+        """Obtaining creds based on Web server application scenario"""
+        info = {
+            "client_id": self._raw_credentials.get("client_id"),
+            "client_secret": self._raw_credentials.get("client_secret"),
+            "refresh_token": self._raw_credentials.get("refresh_token"),
+        }
+        creds = Credentials.from_authorized_user_info(info)
+        if creds.expired:
+            creds.refresh(Request())
+        self._creds = creds
+
+    def _obtain_creds(self):
+        if "credentials_json" in self._raw_credentials:
+            self._obtain_service_account_creds()
+        elif "client_id" and "client_secret" in self._raw_credentials:
+            self._obtain_web_app_creds()
 
     def _construct_resource(self):
         if not self._creds:

diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/client.py
@@ -11,8 +11,11 @@
 
 
 class Client(BaseClient):
-    def __init__(self, credentials_json: str, email: str):
-        self._api = API(credentials_json, email)
+    def __init__(self, credentials: Mapping[str, Any] = None, credentials_json: str = None, email: str = None):
+        # supporting old config format
+        if not credentials:
+            credentials = {"credentials_json": credentials_json, "email": email}
+        self._api = API(credentials)
         self._apis = {"users": UsersAPI(self._api), "groups": GroupsAPI(self._api), "group_members": GroupMembersAPI(self._api)}
         super().__init__()
 

diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json b/airbyte-integrations/connectors/source-google-directory/source_google_directory/spec.json
@@ -4,18 +4,87 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Google Directory Spec",
     "type": "object",
-    "required": ["credentials_json", "email"],
-    "additionalProperties": false,
+    "required": [],
+    "additionalProperties": true,
     "properties": {
-      "credentials_json": {
-        "type": "string",
-        "description": "The contents of the JSON service account key. See the <a href=\"https://developers.google.com/admin-sdk/directory/v1/guides/delegation\">docs</a> for more information on how to generate this key.",
-        "airbyte_secret": true
-      },
-      "email": {
-        "type": "string",
-        "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs."
+      "credentials": {
+        "title": "Google Credentials",
+        "description": "Google APIs use the OAuth 2.0 protocol for authentication and authorization. The Source supports <a href=\"https://developers.google.com/identity/protocols/oauth2#webserver\" target=\"_blank\">Web server application</a> and <a href=\"https://developers.google.com/identity/protocols/oauth2#serviceaccount\" target=\"_blank\">Service accounts</a> scenarios",
+        "type": "object",
+        "oneOf": [
+          {
+            "title": "Sign in via Google (Oauth)",
+            "description": "For these scenario user only needs to give permission to read Google Directory data",
+            "type": "object",
+            "required": ["client_id", "client_secret", "refresh_token"],
+            "properties": {
+              "credentials_title": {
+                "type": "string",
+                "title": "Credentials title",
+                "description": "Authentication scenario",
+                "const": "Web server app",
+                "enum": ["Web server app"],
+                "default": "Web server app",
+                "order": 0
+              },
+              "client_id": {
+                "title": "Client ID",
+                "type": "string",
+                "description": "The client ID of developer application",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "title": "Client secret",
+                "type": "string",
+                "description": "The client secret of developer application",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "title": "Refresh Token",
+                "type": "string",
+                "description": "The token for obtaining new access token",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "Service account Key",
+            "description": "For these scenario user should obtain service account's credentials from the Google API Console and provide delegated email",
+            "type": "object",
+            "required": ["credentials_json", "email"],
+            "properties": {
+              "credentials_title": {
+                "type": "string",
+                "title": "Credentials title",
+                "description": "Authentication scenario",
+                "const": "Service accounts",
+                "enum": ["Service accounts"],
+                "default": "Service accounts",
+                "order": 0
+              },
+              "credentials_json": {
+                "type": "string",
+                "title": "Credentials JSON",
+                "description": "The contents of the JSON service account key. See the <a href=\"https://developers.google.com/admin-sdk/directory/v1/guides/delegation\">docs</a> for more information on how to generate this key.",
+                "airbyte_secret": true
+              },
+              "email": {
+                "type": "string",
+                "title": "Email",
+                "description": "The email of the user, which has permissions to access the Google Workspace Admin APIs."
+              }
+            }
+          }
+        ]
       }
     }
+  },
+  "authSpecification": {
+    "auth_type": "oauth2.0",
+    "oauth2Specification": {
+      "rootObject": ["credentials", 0],
+      "oauthFlowInitParameters": [["client_id"], ["client_secret"]],
+      "oauthFlowOutputParameters": [["refresh_token"]]
+    }
   }
 }
diff --git a/docs/integrations/sources/google-directory.md b/docs/integrations/sources/google-directory.md
@@ -35,9 +35,19 @@ This Source is capable of syncing the following Streams:
 
 This connector attempts to back off gracefully when it hits Directory API's rate limits. To find more information about limits, see [Google Directory's Limits and Quotas](https://developers.google.com/admin-sdk/directory/v1/limits) documentation.
 
-## Getting started
+## Getting Started \(Airbyte Cloud\)
 
-### Requirements
+1. Click `OAuth2.0 authorization` then `Authenticate your Google Directory account`.
+2. You're done.
+
+## Getting Started \(Airbyte Open-Source\)
+
+Google APIs use the OAuth 2.0 protocol for authentication and authorization. This connector supports [Web server application](https://developers.google.com/identity/protocols/oauth2#webserver) and [Service accounts](https://developers.google.com/identity/protocols/oauth2#serviceaccount) scenarios. Therefore, there are 2 options of setting up authorization for this source:
+
+* Use your Google account and authorize over Google's OAuth on connection setup. Select "Default OAuth2.0 authorization" from dropdown list.
+* Create service account specifically for Airbyte.
+
+### Service account requirements
 
 * Credentials to a Google Service Account with delegated Domain Wide Authority
 * Email address of the workspace admin which created the Service Account
@@ -58,6 +68,8 @@ You should now be ready to use the Google Directory connector in Airbyte.
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.1.8  | 2021-11-02 | [7409](https://github.com/airbytehq/airbyte/pull/7409) | Support oauth (update publish) |
+| 0.1.7  | 2021-11-02 | [7409](https://github.com/airbytehq/airbyte/pull/7409) | Support oauth |
 | 0.1.6  | 2021-11-02 | [7464](https://github.com/airbytehq/airbyte/pull/7464) | Migrate to the CDK |
 | 0.1.5  | 2021-10-20 | [6930](https://github.com/airbytehq/airbyte/pull/6930) | Fix crash when a group don't have members |
 | 0.1.4  | 2021-10-19 | [7167](https://github.com/airbytehq/airbyte/pull/7167) | Add organizations and phones to `users` schema |
diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh
@@ -86,6 +86,7 @@ write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS
 write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_SRV_ACC" "service_config.json"
 write_standard_creds source-google-analytics-v4 "$GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD" "old_config.json"
 write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS"
+write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS_OAUTH" "config_oauth.json"
 write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS"
 write_standard_creds source-google-search-console "$GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC" "service_account_config.json"
 write_standard_creds source-google-sheets "$GOOGLE_SHEETS_TESTS_CREDS"