Source Amazon SP: Update reports streams logics.

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.29.17-alpha
+current_version = 0.29.19-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -1,4 +1,4 @@
-VERSION=0.29.17-alpha
+VERSION=0.29.19-alpha
 
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker

.github/ISSUE_TEMPLATE/new-integration-request.md

@@ -12,9 +12,10 @@ assignees: ''
 * Do you need a specific version of the underlying data source e.g: you specifically need support for an older version of the API or DB? 
 
 ## Describe the context around this new connector
-* Which team in your company wants this integration, what for? This helps us understand the use case.
+* Why do you need this integration? How does your team intend to use the data? This helps us understand the use case.
 * How often do you want to run syncs?
 * If this is an API source connector, which entities/endpoints do you need supported?
+* If the connector is for a paid service, can we name you as a mutual user when we subscribe for an account? Which company should we name? 
 
 ## Describe the alternative you are considering or using
 What are you considering doing if you don’t have this integration through Airbyte?

.github/workflows/helm.yaml

@@ -0,0 +1,74 @@
+name: Helm
+on:
+  push:
+    paths:
+    - '.github/workflows/helm.yaml'
+    - 'charts/**'
+  pull_request:
+    paths:
+    - '.github/workflows/helm.yaml'
+    - 'charts/**'
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Kubectl
+      uses: azure/setup-kubectl@v1
+    - name: Setup Helm
+      uses: azure/setup-helm@v1
+      with:
+        version: '3.6.3'
+    - name: Lint Chart
+      working-directory: ./charts/airbyte
+      run: ./ci.sh lint
+
+  generate-docs:
+    name: Generate Docs Parameters
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - uses: actions/checkout@v2
+    - name: Checkout bitnami-labs/readme-generator-for-helm
+      uses: actions/checkout@v2
+      with:
+        repository: 'bitnami-labs/readme-generator-for-helm'
+        ref: '55cab5dd2191c4ffa7245cfefa428d4d9bb12730'
+        path: readme-generator-for-helm
+    - name: Install readme-generator-for-helm dependencies
+      working-directory: readme-generator-for-helm  
+      run: npm install -g
+    - name: Test can update README with generated parameters
+      working-directory: charts/airbyte
+      run: ./ci.sh check-docs-updated
+
+  install:
+    name: Install
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Kubectl
+      uses: azure/setup-kubectl@v1
+    - name: Setup Helm
+      uses: azure/setup-helm@v1
+      with:
+        version: '3.6.3'
+    - name: Setup Kind Cluster
+      uses: helm/[email protected]
+      with:
+        version: "v0.11.1"
+        image: "kindest/node:v1.21.1"
+    - name: Install airbyte chart
+      working-directory: ./charts/airbyte
+      run: ./ci.sh install
+    - if: always()
+      name: Print diagnostics
+      working-directory: ./charts/airbyte
+      run: ./ci.sh diagnostics
+    - if: success()
+      name: Test airbyte chart
+      working-directory: ./charts/airbyte
+      run: ./ci.sh test

.github/workflows/publish-command.yml

@@ -178,6 +178,7 @@ jobs:
           SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }}
           SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }}
           SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }}
+          DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }}
       - run: |
           echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
           ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache

.github/workflows/test-command.yml

@@ -173,6 +173,7 @@ jobs:
           SOURCE_CLOSE_COM_CREDS: ${{ secrets.SOURCE_CLOSE_COM_CREDS }}
           SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }}
           SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }}
+          DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }}
       - run: |
           ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
         name: test ${{ github.event.inputs.connector }}
@@ -194,6 +195,16 @@ jobs:
             **/normalization_test_output/**/build/compiled/airbyte_utils/**
             **/normalization_test_output/**/build/run/airbyte_utils/**
             **/normalization_test_output/**/models/generated/**
+
+      - name: Test coverage reports artifacts
+        if: github.event.inputs.comment-id && success()
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-reports
+          path: |
+            **/${{ github.event.inputs.connector }}/htmlcov/**
+          retention-days: 3
+
       - name: Report Status
         if: github.ref == 'refs/heads/master' && always()
         run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}}
@@ -208,6 +219,7 @@ jobs:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
             > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+            ${{env.PYTHON_UNITTEST_COVERAGE_REPORT}}
       - name: Add Failure Comment
         if: github.event.inputs.comment-id && failure()
         uses: peter-evans/create-or-update-comment@v1

.gitignore

@@ -10,6 +10,9 @@ data
 .project
 .settings
 
+# Logs
+acceptance_tests_logs/
+
 # Secrets
 secrets
 !airbyte-integrations/connector-templates/**/secrets
@@ -26,6 +29,21 @@ __pycache__
 .ipynb_checkpoints
 .pytest_
 
+# Python unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
 # dbt
 profiles.yml
 

airbyte-api/src/main/openapi/config.yaml

@@ -1234,7 +1234,7 @@ paths:
     post:
       tags:
         - oauth
-      summary: Given a source def ID and optional workspaceID generate an access/refresh token etc.
+      summary: Given a source def ID generate an access/refresh token etc.
       operationId: completeSourceOAuth
       requestBody:
         content:
@@ -1280,7 +1280,7 @@ paths:
     post:
       tags:
         - oauth
-      summary:
+      summary: Given a destination def ID generate an access/refresh token etc.
       operationId: completeDestinationOAuth
       requestBody:
         content:
@@ -1906,6 +1906,34 @@ components:
       description: The specification for what values are required to configure the sourceDefinition.
       type: object
       example: { user: { type: string } }
+    SourceAuthSpecification:
+      $ref: "#/components/schemas/AuthSpecification"
+    AuthSpecification:
+      type: object
+      properties:
+        auth_type:
+          type: string
+          enum: ["oauth2.0"] # Future auth types should be added here
+        oauth2Specification:
+          "$ref": "#/components/schemas/OAuth2Specification"
+    OAuth2Specification:
+      description: An object containing any metadata needed to describe this connector's Oauth flow
+      type: object
+      properties:
+        oauthFlowInitParameters:
+          description:
+            "Pointers to the fields in the ConnectorSpecification which are needed to obtain the initial refresh/access tokens for the OAuth flow.
+            Each inner array represents the path in the ConnectorSpecification of the referenced field.
+            For example.
+            Assume the ConnectorSpecification contains params 'app_secret', 'app_id' which are needed to get the initial refresh token.
+            If they are not nested in the config, then the array would look like this [['app_secret'], ['app_id']]
+            If they are nested inside, say, an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']]"
+          type: array
+          items:
+            description: A list of strings which describes each parameter's path inside the ConnectionSpecification
+            type: array
+            items:
+              type: string
     SourceDefinitionSpecificationRead:
       type: object
       required:
@@ -1918,6 +1946,8 @@ components:
           type: string
         connectionSpecification:
           $ref: "#/components/schemas/SourceDefinitionSpecification"
+        authSpecification:
+          $ref: "#/components/schemas/SourceAuthSpecification"
         jobInfo:
           $ref: "#/components/schemas/SynchronousJobRead"
     # SOURCE
@@ -2018,6 +2048,8 @@ components:
     DestinationDefinitionId:
       type: string
       format: uuid
+    DestinationAuthSpecification:
+      $ref: "#/components/schemas/AuthSpecification"
     DestinationDefinitionIdRequestBody:
       type: object
       required:
@@ -2101,6 +2133,8 @@ components:
           type: string
         connectionSpecification:
           $ref: "#/components/schemas/DestinationDefinitionSpecification"
+        authSpecification:
+          $ref: "#/components/schemas/DestinationAuthSpecification"
         jobInfo:
           $ref: "#/components/schemas/SynchronousJobRead"
         supportedDestinationSyncModes:

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.1.21
+Resolve nested schema references and move external references to single schema definitions.
+
+## 0.1.20
+- Allow using `requests.auth.AuthBase` as authenticators instead of custom CDK authenticators.
+- Implement Oauth2Authenticator, MultipleTokenAuthenticator and TokenAuthenticator authenticators.
+- Add support for both legacy and requests native authenticator to HttpStream class.
+
+## 0.1.19
+No longer prints full config files on validation error to prevent exposing secrets to log file: https://github.com/airbytehq/airbyte/pull/5879
+
 ## 0.1.18
 Fix incremental stream not saved state when internal limit config set.
 

airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/core.py

@@ -26,7 +26,10 @@
 from abc import ABC, abstractmethod
 from typing import Any, Mapping
 
+from deprecated import deprecated
 
+
+@deprecated(version="0.1.20", reason="Use requests.auth.AuthBase instead")
 class HttpAuthenticator(ABC):
     """
     Base abstract class for various HTTP Authentication strategies. Authentication strategies are generally
@@ -40,6 +43,7 @@ def get_auth_header(self) -> Mapping[str, Any]:
         """
 
 
+@deprecated(version="0.1.20", reason="Set `authenticator=None` instead")
 class NoAuth(HttpAuthenticator):
     def get_auth_header(self) -> Mapping[str, Any]:
         return {}

airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/oauth.py

@@ -27,10 +27,12 @@
 
 import pendulum
 import requests
+from deprecated import deprecated
 
 from .core import HttpAuthenticator
 
 
+@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.Oauth2Authenticator instead")
 class Oauth2Authenticator(HttpAuthenticator):
     """
     Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials.

airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/token.py

@@ -26,9 +26,12 @@
 from itertools import cycle
 from typing import Any, List, Mapping
 
+from deprecated import deprecated
+
 from .core import HttpAuthenticator
 
 
+@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.TokenAuthenticator instead")
 class TokenAuthenticator(HttpAuthenticator):
     def __init__(self, token: str, auth_method: str = "Bearer", auth_header: str = "Authorization"):
         self.auth_method = auth_method
@@ -39,6 +42,7 @@ def get_auth_header(self) -> Mapping[str, Any]:
         return {self.auth_header: f"{self.auth_method} {self._token}"}
 
 
+@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.MultipleTokenAuthenticator instead")
 class MultipleTokenAuthenticator(HttpAuthenticator):
     def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"):
         self.auth_method = auth_method

airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py

@@ -29,6 +29,7 @@
 import requests
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams.core import Stream
+from requests.auth import AuthBase
 
 from .auth.core import HttpAuthenticator, NoAuth
 from .exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException
@@ -46,10 +47,16 @@ class HttpStream(Stream, ABC):
     source_defined_cursor = True  # Most HTTP streams use a source defined cursor (i.e: the user can't configure it like on a SQL table)
     page_size = None  # Use this variable to define page size for API http requests with pagination support
 
-    def __init__(self, authenticator: HttpAuthenticator = NoAuth()):
-        self._authenticator = authenticator
+    # TODO: remove legacy HttpAuthenticator authenticator references
+    def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None):
         self._session = requests.Session()
 
+        self._authenticator = NoAuth()
+        if isinstance(authenticator, AuthBase):
+            self._session.auth = authenticator
+        elif authenticator:
+            self._authenticator = authenticator
+
     @property
     @abstractmethod
     def url_base(self) -> str:

airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py

@@ -0,0 +1,32 @@
+#
+# MIT License
+#
+# Copyright (c) 2020 Airbyte
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+from .oauth import Oauth2Authenticator
+from .token import MultipleTokenAuthenticator, TokenAuthenticator
+
+__all__ = [
+    "Oauth2Authenticator",
+    "TokenAuthenticator",
+    "MultipleTokenAuthenticator",
+]