🎉 Source Github: use GraphQL for reviews stream (#13989)

Signed-off-by: Sergey Chvalyuk <[email protected]>
airbytehq · Jun 28, 2022 · 4072d46 · 4072d46
1 parent 26a33df
commit 4072d46
Show file tree

Hide file tree

Showing 17 changed files with 639 additions and 162 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -303,7 +303,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.2.37
+  dockerImageTag: 0.2.38
   documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2584,7 +2584,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.2.37"
+- dockerImage: "airbyte/source-github:0.2.38"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.37
+LABEL io.airbyte.version=0.2.38
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml
@@ -42,7 +42,7 @@ tests:
         releases: ["airbytehq/integration-test", "created_at"]
         repositories: ["airbytehq", "updated_at"]
         review_comments: ["airbytehq/integration-test", "updated_at"]
-        reviews: ["airbytehq/integration-test", "pull_request_updated_at"]
+        reviews: ["airbytehq/integration-test", "updated_at"]
         stargazers: ["airbytehq/integration-test", "starred_at"]
         workflow_runs: ["airbytehq/integration-test", "updated_at"]
         workflows: ["airbytehq/integration-test", "updated_at"]

diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json
@@ -121,7 +121,7 @@
   },
   "reviews": {
     "airbytehq/integration-test": {
-      "pull_request_updated_at": "2121-06-29T02:04:57Z"
+      "updated_at": "2121-06-29T02:04:57Z"
     }
   },
   "stargazers": {

diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json
@@ -311,12 +311,12 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["pull_request_updated_at"],
+        "default_cursor_field": ["updated_at"],
         "source_defined_primary_key": [["id"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append",
-      "cursor_field": ["pull_request_updated_at"]
+      "cursor_field": ["updated_at"]
     },
     {
       "stream": {

diff --git a/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-github/integration_tests/sample_state.json
@@ -61,7 +61,7 @@
   },
   "reviews": {
     "airbytehq/integration-test": {
-      "pull_request_updated_at": "2021-08-30T12:01:15Z"
+      "updated_at": "2021-08-30T12:01:15Z"
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-github/source_github/graphql.py b/airbyte-integrations/connectors/source-github/source_github/graphql.py
@@ -11,13 +11,16 @@
 _schema_root = _schema.github_schema
 
 
-def get_query(owner, name, page_size, next_page_token):
-    kwargs = {"first": page_size, "order_by": {"field": "UPDATED_AT", "direction": "ASC"}}
-    if next_page_token:
-        kwargs["after"] = next_page_token
+def get_query_pull_requests(owner, name, first, after, direction):
+    kwargs = {"first": first, "order_by": {"field": "UPDATED_AT", "direction": direction}}
+    if after:
+        kwargs["after"] = after
 
     op = sgqlc.operation.Operation(_schema_root.query_type)
-    pull_requests = op.repository(owner=owner, name=name).pull_requests(**kwargs)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    pull_requests = repository.pull_requests(**kwargs)
     pull_requests.nodes.__fields__(
         id="node_id",
         database_id="id",
@@ -32,7 +35,6 @@ def get_query(owner, name, page_size, next_page_token):
         maintainer_can_modify="maintainer_can_modify",
         merge_state_status="merge_state_status",
     )
-    pull_requests.nodes.repository.__fields__(name=True)
     pull_requests.nodes.comments.__fields__(total_count=True)
     pull_requests.nodes.commits.__fields__(total_count=True)
     reviews = pull_requests.nodes.reviews(first=100, __alias__="review_comments")
@@ -49,3 +51,48 @@ def get_query(owner, name, page_size, next_page_token):
     )
     pull_requests.page_info.__fields__(has_next_page=True, end_cursor=True)
     return str(op)
+
+
+def get_query_reviews(owner, name, first, after, number=None):
+    op = sgqlc.operation.Operation(_schema_root.query_type)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    if number:
+        pull_request = repository.pull_request(number=number)
+    else:
+        kwargs = {"first": first, "order_by": {"field": "UPDATED_AT", "direction": "ASC"}}
+        if after:
+            kwargs["after"] = after
+        pull_requests = repository.pull_requests(**kwargs)
+        pull_requests.page_info.__fields__(has_next_page=True, end_cursor=True)
+        pull_request = pull_requests.nodes
+
+    pull_request.__fields__(number=True, url=True)
+    kwargs = {"first": first}
+    if number and after:
+        kwargs["after"] = after
+    reviews = pull_request.reviews(**kwargs)
+    reviews.page_info.__fields__(has_next_page=True, end_cursor=True)
+    reviews.nodes.__fields__(
+        id="node_id",
+        database_id="id",
+        body=True,
+        state=True,
+        url="html_url",
+        author_association="author_association",
+        submitted_at="submitted_at",
+        created_at="created_at",
+        updated_at="updated_at",
+    )
+    reviews.nodes.commit.oid()
+    user = reviews.nodes.author(__alias__="user").__as__(_schema_root.User)
+    user.__fields__(
+        id="node_id",
+        database_id="id",
+        login=True,
+        avatar_url="avatar_url",
+        url="html_url",
+        is_site_admin="site_admin",
+    )
+    return str(op)
diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_stats.json b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_stats.json
@@ -26,6 +26,9 @@
     "merge_state_status": {
       "type": ["null", "string"]
     },
+    "merged_by": {
+      "$ref": "user_graphql.json"
+    },
     "merged_by": {
       "type": ["null", "object"],
       "properties": {

diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/reviews.json b/airbyte-integrations/connectors/source-github/source_github/schemas/reviews.json
@@ -12,7 +12,7 @@
       "type": ["null", "string"]
     },
     "user": {
-      "$ref": "user.json"
+      "$ref": "user_graphql.json"
     },
     "body": {
       "type": ["null", "string"]
@@ -51,7 +51,11 @@
       "type": ["null", "string"],
       "format": "date-time"
     },
-    "pull_request_updated_at": {
+    "created_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "updated_at": {
       "type": "string",
       "format": "date-time"
     },

diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/user_graphql.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/user_graphql.json
@@ -0,0 +1,26 @@
+{
+  "type": ["null", "object"],
+  "properties": {
+    "login": {
+      "type": ["null", "string"]
+    },
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "node_id": {
+      "type": ["null", "string"]
+    },
+    "avatar_url": {
+      "type": ["null", "string"]
+    },
+    "html_url": {
+      "type": ["null", "string"]
+    },
+    "type": {
+      "type": ["null", "string"]
+    },
+    "site_admin": {
+      "type": ["null", "boolean"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py
@@ -217,7 +217,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Releases(**repository_args_with_start_date),
             Repositories(**organization_args_with_start_date),
             ReviewComments(**repository_args_with_start_date),
-            Reviews(parent=pull_requests_stream, **repository_args_with_start_date),
+            Reviews(**repository_args_with_start_date),
             Stargazers(**repository_args_with_start_date),
             Tags(**repository_args),
             teams_stream,