feat: Build empty cookiecutters and run lint task during CI (#1410)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Edgar R. M <[email protected]>

.github/workflows/cookiecutter-e2e.yml

@@ -0,0 +1,72 @@
+name: E2E Cookiecutters
+
+on:
+  push:
+    branches: [main]
+    paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: "1"
+
+jobs:
+  lint:
+    name: Lint ${{ matrix.cookiecutter }}, Replay ${{ matrix.replay }} on ${{ matrix.python-version }} ${{ matrix.python-version }} / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9",  os: "ubuntu-latest" }
+          - { cookiecutter: "tap-template", replay: "tap-rest-jwt-github.json", python-version: "3.9",  os: "ubuntu-latest" }
+          - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9",  os: "ubuntu-latest" }
+
+    steps:
+    - name: Check out the repository
+      uses: actions/[email protected]
+
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: x64
+
+    - name: Upgrade pip
+      env:
+        PIP_CONSTRAINT: .github/workflows/constraints.txt
+      run: |
+        pip install pip
+        pip --version
+
+    - name: Install Poetry, Tox & Cookiecutter
+      env:
+        PIP_CONSTRAINT: .github/workflows/constraints.txt
+      run: |
+        pipx install poetry
+        pipx install cookiecutter
+        pipx install tox
+
+    - name: Build cookiecutter project
+      env:
+        CC_TEMPLATE: cookiecutter/${{ matrix.cookiecutter }}
+        REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }}
+      run: |
+        bash e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.replay }}
+        path: |
+          ${{ env.CC_TEST_OUTPUT }}/
+          !${{ env.CC_TEST_OUTPUT }}/.mypy_cache/
+
+    - name: Run lint
+      env:
+        REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }}
+      run: |
+        cd $CC_TEST_OUTPUT
+        poetry run tox -e lint

cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml

@@ -26,12 +26,16 @@ fs-s3fs = { version = "^1.1.1", optional = true}
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
 requests = "^2.28.2"
 {%- endif %}
+{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
+cached-property = "^1" # Remove after Python 3.7 support is dropped
+{%- endif %}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.1"
 flake8 = "^5.0.4"
+darglint = "^1.8.1"
 black = "^23.1.0"
-pydocstyle = "^6.3.0"
+pyupgrade = "^3.3.1"
 mypy = "^1.0.0"
 isort = "^5.11.5"
 {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %}
@@ -46,6 +50,10 @@ profile = "black"
 multi_line_output = 3 # Vertical Hanging Indent
 src_paths = "{{cookiecutter.library_name}}"
 
+[tool.mypy]
+python_version = "3.9"
+warn_unused_configs = true
+
 [build-system]
 requires = ["poetry-core>=1.0.8"]
 build-backend = "poetry.core.masonry.api"

cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini

@@ -40,11 +40,11 @@ commands =
     poetry run black --check --diff {{cookiecutter.library_name}}/
     poetry run isort --check {{cookiecutter.library_name}}
     poetry run flake8 {{cookiecutter.library_name}}
-    poetry run pydocstyle {{cookiecutter.library_name}}
     # refer to mypy.ini for specific settings
     poetry run mypy {{cookiecutter.library_name}} --exclude='{{cookiecutter.library_name}}/tests'
 
 [flake8]
+docstring-convention = google
 ignore = W503
 max-line-length = 88
 max-complexity = 10

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py

@@ -0,0 +1 @@
+"""Tap for {{ cookiecutter.source_name }}."""

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py

@@ -1,30 +1,17 @@
 """{{ cookiecutter.source_name }} tap class."""
 
-from typing import List
+from __future__ import annotations
 
-from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap, {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Stream
+from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap
 from singer_sdk import typing as th  # JSON schema typing helpers
 
 {%- if cookiecutter.stream_type == "SQL" %}
+
 from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream
 {%- else %}
-# TODO: Import your custom stream types here:
-from {{ cookiecutter.library_name }}.streams import (
-    {{ cookiecutter.source_name }}Stream,
-{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
-    UsersStream,
-    GroupsStream,
-{%- endif %}
-)
-{%- endif %}
 
-{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
-# TODO: Compile a list of custom stream types here
-#       OR rewrite discover_streams() below with your custom logic.
-STREAM_TYPES = [
-    UsersStream,
-    GroupsStream,
-]
+# TODO: Import your custom stream types here:
+from {{ cookiecutter.library_name }} import streams
 {%- endif %}
 
 
@@ -40,31 +27,38 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type ==
             th.StringType,
             required=True,
             secret=True,  # Flag config as protected.
-            description="The token to authenticate against the API service"
+            description="The token to authenticate against the API service",
         ),
         th.Property(
             "project_ids",
             th.ArrayType(th.StringType),
             required=True,
-            description="Project IDs to replicate"
+            description="Project IDs to replicate",
         ),
         th.Property(
             "start_date",
             th.DateTimeType,
-            description="The earliest record date to sync"
+            description="The earliest record date to sync",
         ),
         th.Property(
             "api_url",
             th.StringType,
             default="https://api.mysample.com",
-            description="The url for the API service"
+            description="The url for the API service",
         ),
     ).to_dict()
 {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %}
 
-    def discover_streams(self) -> List[Stream]:
-        """Return a list of discovered streams."""
-        return [stream_class(tap=self) for stream_class in STREAM_TYPES]
+    def discover_streams(self) -> list[streams.{{ cookiecutter.source_name }}Stream]:
+        """Return a list of discovered streams.
+
+        Returns:
+            A list of discovered streams.
+        """
+        return [
+            streams.GroupsStream(self),
+            streams.UsersStream(self),
+        ]
 {%- endif %}
 
 

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%}

@@ -1,11 +1,14 @@
 """GraphQL client handling, including {{ cookiecutter.source_name }}Stream base class."""
 
-import requests
-from pathlib import Path
-from typing import Any, Dict, Optional, Union, List, Iterable
+from __future__ import annotations
+
+from typing import Iterable
 
+import requests
 from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream
+
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
+
 from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator
 {%- endif %}
 
@@ -16,7 +19,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
     # TODO: Set the API's base URL here:
     @property
     def url_base(self) -> str:
-        """Return the API URL root, configurable via tap settings."""
+        """Return the API URL root, configurable via tap settings.
+
+        Returns:
+            The base URL for all requests.
+        """
         return self.config["api_url"]
 
     # Alternatively, use a static string for url_base:
@@ -25,14 +32,22 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
 {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %}
     @property
     def authenticator(self) -> {{ cookiecutter.source_name }}Authenticator:
-        """Return a new authenticator object."""
+        """Return a new authenticator object.
+
+        Returns:
+            An authenticator instance.
+        """
         return {{ cookiecutter.source_name }}Authenticator.create_for_stream(self)
 
 {%- endif %}
 
     @property
     def http_headers(self) -> dict:
-        """Return the http headers needed."""
+        """Return the http headers needed.
+
+        Returns:
+            A dictionary of HTTP headers.
+        """
         headers = {}
         if "user_agent" in self.config:
             headers["User-Agent"] = self.config.get("user_agent")
@@ -43,13 +58,28 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream)
         return headers
 
     def parse_response(self, response: requests.Response) -> Iterable[dict]:
-        """Parse the response and return an iterator of result records."""
+        """Parse the response and return an iterator of result records.
+
+        Args:
+            response: The HTTP ``requests.Response`` object.
+
+        Yields:
+            Each record from the source.
+        """
         # TODO: Parse response body and return a set of records.
         resp_json = response.json()
         for record in resp_json.get("<TODO>"):
             yield record
 
-    def post_process(self, row: dict, context: Optional[dict] = None) -> dict:
-        """As needed, append or transform raw data to match expected structure."""
+    def post_process(self, row: dict, context: dict | None = None) -> dict | None:
+        """As needed, append or transform raw data to match expected structure.
+
+        Args:
+            row: An individual record from the stream.
+            context: The stream context.
+
+        Returns:
+            The updated record dictionary, or ``None`` to skip the record.
+        """
         # TODO: Delete this method if not needed.
         return row

cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%}

@@ -1,16 +1,18 @@
 """Custom client handling, including {{ cookiecutter.source_name }}Stream base class."""
 
+from __future__ import annotations
+
 import requests
 from pathlib import Path
-from typing import Any, Dict, Optional, Union, List, Iterable
+from typing import Any, Iterable
 
 from singer_sdk.streams import Stream
 
 
 class {{ cookiecutter.source_name }}Stream(Stream):
     """Stream class for {{ cookiecutter.source_name }} streams."""
 
-    def get_records(self, context: Optional[dict]) -> Iterable[dict]:
+    def get_records(self, context: dict | None) -> Iterable[dict]:
         """Return a generator of record-type dictionary objects.
 
         The optional `context` argument is used to identify a specific slice of the