From 958f7699341affb76416c8003b7c1262bed650a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 14 Feb 2023 16:43:53 -0600 Subject: [PATCH] Fix files --- .github/workflows/cookiecutter-e2e.yml | 2 + .../{{cookiecutter.tap_id}}/mypy.ini | 6 - .../{{cookiecutter.tap_id}}/pyproject.toml | 10 +- .../{{cookiecutter.tap_id}}/tox.ini | 2 +- .../{{cookiecutter.library_name}}/__init__.py | 2 +- .../{{cookiecutter.library_name}}/tap.py | 45 +++--- ...kiecutter.stream_type %}client.py{%endif%} | 48 ++++-- ...kiecutter.stream_type %}client.py{%endif%} | 6 +- ...kiecutter.stream_type %}client.py{%endif%} | 146 ++++++++++++++---- ...iecutter.stream_type %}streams.py{%endif%} | 24 +-- ...kiecutter.stream_type %}client.py{%endif%} | 2 + ...hod in ('OAuth2', 'JWT')%}auth.py{%endif%} | 25 ++- .../{{cookiecutter.target_id}}/pyproject.toml | 3 +- e2e-tests/cookiecutters/test_cookiecutter.sh | 1 - poetry.lock | 58 +------ 15 files changed, 235 insertions(+), 145 deletions(-) delete mode 100644 cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 36dfc4d22..fcc527bdb 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -4,6 +4,7 @@ on: push: branches: [main] paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -21,6 +22,7 @@ jobs: matrix: include: - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } + - { cookiecutter: "tap-template", replay: "tap-rest-jwt-github.json", python-version: "3.9", os: "ubuntu-latest" } - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } steps: diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini deleted file mode 100644 index ba621de2b..000000000 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/mypy.ini +++ /dev/null @@ -1,6 +0,0 @@ -[mypy] -python_version = 3.9 -warn_unused_configs = True - -[mypy-backoff.*] -ignore_missing_imports = True diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml index c2bf70e58..784b49ab0 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml @@ -26,12 +26,16 @@ fs-s3fs = { version = "^1.1.1", optional = true} {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} requests = "^2.28.2" {%- endif %} +{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} +cached-property = "^1" # Remove after Python 3.7 support is dropped +{%- endif %} [tool.poetry.group.dev.dependencies] pytest = "^7.2.1" flake8 = "^5.0.4" +darglint = "^1.8.1" black = "^23.1.0" -pydocstyle = "^6.3.0" +pyupgrade = "^3.3.1" mypy = "^1.0.0" isort = "^5.11.5" {%- if cookiecutter.stream_type in ["REST", "GraphQL"] %} @@ -46,6 +50,10 @@ profile = "black" multi_line_output = 3 # Vertical Hanging Indent src_paths = "{{cookiecutter.library_name}}" +[tool.mypy] +python_version = "3.9" +warn_unused_configs = true + [build-system] requires = ["poetry-core>=1.0.8"] build-backend = "poetry.core.masonry.api" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini index e542be5ac..5f20123c6 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini @@ -40,11 +40,11 @@ commands = poetry run black --check --diff {{cookiecutter.library_name}}/ poetry run isort --check {{cookiecutter.library_name}} poetry run flake8 {{cookiecutter.library_name}} - poetry run pydocstyle {{cookiecutter.library_name}} # refer to mypy.ini for specific settings poetry run mypy {{cookiecutter.library_name}} --exclude='{{cookiecutter.library_name}}/tests' [flake8] +docstring-convention = google ignore = W503 max-line-length = 88 max-complexity = 10 diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py index 91adfcd78..b5c6e813f 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/__init__.py @@ -1 +1 @@ -"""Tap for {{ cookiecutter.source_name }}.""" \ No newline at end of file +"""Tap for {{ cookiecutter.source_name }}.""" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py index 2f2cbc06b..5f2832e60 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/tap.py @@ -1,31 +1,17 @@ """{{ cookiecutter.source_name }} tap class.""" -from typing import List +from __future__ import annotations -from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap, {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Stream +from singer_sdk import {{ 'SQL' if cookiecutter.stream_type == 'SQL' else '' }}Tap from singer_sdk import typing as th # JSON schema typing helpers {%- if cookiecutter.stream_type == "SQL" %} + from {{ cookiecutter.library_name }}.client import {{ cookiecutter.source_name }}Stream {%- else %} -# TODO: Import your custom stream types here: -from {{ cookiecutter.library_name }}.streams import ( - {{ cookiecutter.source_name }}Stream, -{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} - UsersStream, - GroupsStream, -{%- endif %} -) -{%- endif %} -{%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} -# TODO: Compile a list of custom stream types here -# OR rewrite discover_streams() below with your custom logic. -STREAM_TYPES = [ - {{ cookiecutter.source_name }}Stream, - UsersStream, - GroupsStream, -] +# TODO: Import your custom stream types here: +from {{ cookiecutter.library_name }} import streams {%- endif %} @@ -41,31 +27,38 @@ class Tap{{ cookiecutter.source_name }}({{ 'SQL' if cookiecutter.stream_type == th.StringType, required=True, secret=True, # Flag config as protected. - description="The token to authenticate against the API service" + description="The token to authenticate against the API service", ), th.Property( "project_ids", th.ArrayType(th.StringType), required=True, - description="Project IDs to replicate" + description="Project IDs to replicate", ), th.Property( "start_date", th.DateTimeType, - description="The earliest record date to sync" + description="The earliest record date to sync", ), th.Property( "api_url", th.StringType, default="https://api.mysample.com", - description="The url for the API service" + description="The url for the API service", ), ).to_dict() {%- if cookiecutter.stream_type in ("GraphQL", "REST", "Other") %} - def discover_streams(self) -> List[Stream]: - """Return a list of discovered streams.""" - return [stream_class(tap=self) for stream_class in STREAM_TYPES] + def discover_streams(self) -> list[streams.{{ cookiecutter.source_name }}Stream]: + """Return a list of discovered streams. + + Returns: + A list of discovered streams. + """ + return [ + streams.GroupsStream(self), + streams.UsersStream(self), + ] {%- endif %} diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} index 386481a74..32c59e832 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'GraphQL' == cookiecutter.stream_type %}client.py{%endif%} @@ -1,11 +1,14 @@ """GraphQL client handling, including {{ cookiecutter.source_name }}Stream base class.""" -import requests -from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable +from __future__ import annotations + +from typing import Iterable +import requests from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} + from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator {%- endif %} @@ -16,7 +19,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) # TODO: Set the API's base URL here: @property def url_base(self) -> str: - """Return the API URL root, configurable via tap settings.""" + """Return the API URL root, configurable via tap settings. + + Returns: + The base URL for all requests. + """ return self.config["api_url"] # Alternatively, use a static string for url_base: @@ -25,14 +32,22 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} @property def authenticator(self) -> {{ cookiecutter.source_name }}Authenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return {{ cookiecutter.source_name }}Authenticator.create_for_stream(self) {%- endif %} @property def http_headers(self) -> dict: - """Return the http headers needed.""" + """Return the http headers needed. + + Returns: + A dictionary of HTTP headers. + """ headers = {} if "user_agent" in self.config: headers["User-Agent"] = self.config.get("user_agent") @@ -43,13 +58,28 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return headers def parse_response(self, response: requests.Response) -> Iterable[dict]: - """Parse the response and return an iterator of result records.""" + """Parse the response and return an iterator of result records. + + Args: + response: The HTTP ``requests.Response`` object. + + Yields: + Each record from the source. + """ # TODO: Parse response body and return a set of records. resp_json = response.json() for record in resp_json.get(""): yield record - def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: - """As needed, append or transform raw data to match expected structure.""" + def post_process(self, row: dict, context: dict | None = None) -> dict | None: + """As needed, append or transform raw data to match expected structure. + + Args: + row: An individual record from the stream. + context: The stream context. + + Returns: + The updated record dictionary, or ``None`` to skip the record. + """ # TODO: Delete this method if not needed. return row diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} index 8d6b118f6..0bf718c75 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'Other' == cookiecutter.stream_type %}client.py{%endif%} @@ -1,8 +1,10 @@ """Custom client handling, including {{ cookiecutter.source_name }}Stream base class.""" +from __future__ import annotations + import requests from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable +from typing import Any, Iterable from singer_sdk.streams import Stream @@ -10,7 +12,7 @@ from singer_sdk.streams import Stream class {{ cookiecutter.source_name }}Stream(Stream): """Stream class for {{ cookiecutter.source_name }} streams.""" - def get_records(self, context: Optional[dict]) -> Iterable[dict]: + def get_records(self, context: dict | None) -> Iterable[dict]: """Return a generator of record-type dictionary objects. The optional `context` argument is used to identify a specific slice of the diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} index 17141d4c5..52bb62a13 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'REST' == cookiecutter.stream_type %}client.py{%endif%} @@ -1,27 +1,46 @@ """REST client handling, including {{ cookiecutter.source_name }}Stream base class.""" -import requests -from pathlib import Path -from typing import Any, Dict, Iterable, Optional +from __future__ import annotations -{%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} -from memoization import cached -{%- endif %} +{% if cookiecutter.auth_method in ("OAuth2", "JWT") -%} +import sys +{% endif -%} +from pathlib import Path +from typing import Any, Callable, Iterable +import requests +{% if cookiecutter.auth_method == "API Key" -%} +from singer_sdk.authenticators import APIKeyAuthenticator from singer_sdk.helpers.jsonpath import extract_jsonpath from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream -{%- if cookiecutter.auth_method == "API Key" %} -from singer_sdk.authenticators import APIKeyAuthenticator -{%- elif cookiecutter.auth_method == "Bearer Token" %} + +{% elif cookiecutter.auth_method == "Bearer Token" -%} from singer_sdk.authenticators import BearerTokenAuthenticator -{%- elif cookiecutter.auth_method == "Basic Auth" %} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + +{% elif cookiecutter.auth_method == "Basic Auth" -%} from singer_sdk.authenticators import BasicAuthenticator -{%- elif cookiecutter.auth_method in ("OAuth2", "JWT") %} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream + +{% elif cookiecutter.auth_method in ("OAuth2", "JWT") -%} +from singer_sdk.helpers.jsonpath import extract_jsonpath +from singer_sdk.streams import {{ cookiecutter.stream_type }}Stream from {{ cookiecutter.library_name }}.auth import {{ cookiecutter.source_name }}Authenticator -{%- endif %} +{% endif -%} + +{%- if cookiecutter.auth_method in ("OAuth2", "JWT") -%} +if sys.version_info >= (3, 8): + from functools import cached_property +else: + from cached_property import cached_property + +{% endif -%} +_Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") @@ -42,29 +61,40 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) {%- if cookiecutter.auth_method in ("OAuth2", "JWT") %} - @property - @cached - def authenticator(self) -> {{ cookiecutter.source_name }}Authenticator: - """Return a new authenticator object.""" + @cached_property + def authenticator(self) -> _Auth: + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return {{ cookiecutter.source_name }}Authenticator.create_for_stream(self) {%- elif cookiecutter.auth_method == "API Key" %} @property def authenticator(self) -> APIKeyAuthenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return APIKeyAuthenticator.create_for_stream( self, key="x-api-key", value=self.config.get("api_key", ""), - location="header" + location="header", ) {%- elif cookiecutter.auth_method == "Bearer Token" %} @property def authenticator(self) -> BearerTokenAuthenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return BearerTokenAuthenticator.create_for_stream( self, token=self.config.get("api_key") @@ -74,7 +104,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) @property def authenticator(self) -> BasicAuthenticator: - """Return a new authenticator object.""" + """Return a new authenticator object. + + Returns: + An authenticator instance. + """ return BasicAuthenticator.create_for_stream( self, username=self.config.get("username"), @@ -85,7 +119,11 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) @property def http_headers(self) -> dict: - """Return the http headers needed.""" + """Return the http headers needed. + + Returns: + A dictionary of HTTP headers. + """ headers = {} if "user_agent" in self.config: headers["User-Agent"] = self.config.get("user_agent") @@ -96,9 +134,19 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return headers def get_next_page_token( - self, response: requests.Response, previous_token: Optional[Any] - ) -> Optional[Any]: - """Return a token for identifying next page or None if no more pages.""" + self, + response: requests.Response, + previous_token: Any | None, + ) -> Any | None: + """Return a token for identifying next page or None if no more pages. + + Args: + response: The HTTP ``requests.Response`` object. + previous_token: The previous page token value. + + Returns: + The next pagination token. + """ # TODO: If pagination is required, return a token which can be used to get the # next page. If this is the final page, return "None" to end the # pagination loop. @@ -114,9 +162,19 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return next_page_token def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] - ) -> Dict[str, Any]: - """Return a dictionary of values to be used in URL parameterization.""" + self, + context: dict | None, + next_page_token: Any | None, + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization. + + Args: + context: The stream context. + next_page_token: The next page index or value. + + Returns: + A dictionary of URL query parameters. + """ params: dict = {} if next_page_token: params["page"] = next_page_token @@ -126,21 +184,45 @@ class {{ cookiecutter.source_name }}Stream({{ cookiecutter.stream_type }}Stream) return params def prepare_request_payload( - self, context: Optional[dict], next_page_token: Optional[Any] - ) -> Optional[dict]: + self, + context: dict | None, + next_page_token: Any | None, + ) -> dict | None: """Prepare the data payload for the REST API request. By default, no payload will be sent (return None). + + Args: + context: The stream context. + next_page_token: The next page index or value. + + Returns: + A dictionary with the JSON body for a POST requests. """ # TODO: Delete this method if no payload is required. (Most REST APIs.) return None def parse_response(self, response: requests.Response) -> Iterable[dict]: - """Parse the response and return an iterator of result records.""" + """Parse the response and return an iterator of result records. + + Args: + response: The HTTP ``requests.Response`` object. + + Yields: + Each record from the source. + """ # TODO: Parse response body and return a set of records. yield from extract_jsonpath(self.records_jsonpath, input=response.json()) - def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: - """As needed, append or transform raw data to match expected structure.""" + def post_process(self, row: dict, context: dict | None = None) -> dict | None: + """As needed, append or transform raw data to match expected structure. + + Args: + row: An individual record from the stream. + context: The stream context. + + Returns: + The updated record dictionary, or ``None`` to skip the record. + """ # TODO: Delete this method if not needed. return row diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} index d085e061e..5ab485498 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' != cookiecutter.stream_type %}streams.py{%endif%} @@ -1,5 +1,7 @@ """Stream type classes for {{ cookiecutter.tap_id }}.""" +from __future__ import annotations + from pathlib import Path from singer_sdk import typing as th # JSON Schema typing helpers @@ -17,6 +19,7 @@ SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") class UsersStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "users" # Optionally, you may also use `schema_filepath` in place of `schema`: # schema_filepath = SCHEMAS_DIR / "users.json" @@ -25,17 +28,17 @@ class UsersStream({{ cookiecutter.source_name }}Stream): th.Property( "id", th.StringType, - description="The user's system ID" + description="The user's system ID", ), th.Property( "age", th.IntegerType, - description="The user's age in years" + description="The user's age in years", ), th.Property( "email", th.StringType, - description="The user's email address" + description="The user's email address", ), th.Property( "address", @@ -45,10 +48,10 @@ class UsersStream({{ cookiecutter.source_name }}Stream): th.Property( "state", th.StringType, - description="State name in ISO 3166-2 format" + description="State name in ISO 3166-2 format", ), th.Property("zip", th.StringType), - ) + ), ), ).to_dict() primary_keys = ["id"] @@ -71,6 +74,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream): class GroupsStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "groups" schema = th.PropertiesList( th.Property("name", th.StringType), @@ -95,6 +99,7 @@ class GroupsStream({{ cookiecutter.source_name }}Stream): class UsersStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "users" {%- if cookiecutter.stream_type == "REST" %} path = "/users" @@ -108,24 +113,24 @@ class UsersStream({{ cookiecutter.source_name }}Stream): th.Property( "id", th.StringType, - description="The user's system ID" + description="The user's system ID", ), th.Property( "age", th.IntegerType, - description="The user's age in years" + description="The user's age in years", ), th.Property( "email", th.StringType, - description="The user's email address" + description="The user's email address", ), th.Property("street", th.StringType), th.Property("city", th.StringType), th.Property( "state", th.StringType, - description="State name in ISO 3166-2 format" + description="State name in ISO 3166-2 format", ), th.Property("zip", th.StringType), ).to_dict() @@ -133,6 +138,7 @@ class UsersStream({{ cookiecutter.source_name }}Stream): class GroupsStream({{ cookiecutter.source_name }}Stream): """Define custom stream.""" + name = "groups" {%- if cookiecutter.stream_type == "REST" %} path = "/groups" diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} index 6217b9015..f22e4dd93 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if 'SQL' == cookiecutter.stream_type %}client.py{%endif%} @@ -3,6 +3,8 @@ This includes {{ cookiecutter.source_name }}Stream and {{ cookiecutter.source_name }}Connector. """ +from __future__ import annotations + import sqlalchemy from singer_sdk import SQLConnector, SQLStream diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} index f4ff1b16c..d1d87ca3d 100644 --- a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} +++ b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/{{cookiecutter.library_name}}/{%if cookiecutter.auth_method in ('OAuth2', 'JWT')%}auth.py{%endif%} @@ -1,9 +1,13 @@ """{{ cookiecutter.source_name }} Authentication.""" -{% if cookiecutter.auth_method not in ("Basic Auth", "OAuth2", "JWT") %} +from __future__ import annotations + +{%- if cookiecutter.auth_method not in ("Basic Auth", "OAuth2", "JWT") %} + # TODO: Delete this file or add custom authentication logic as needed. {%- elif cookiecutter.auth_method == "OAuth2" %} + from singer_sdk.authenticators import OAuthAuthenticator, SingletonMeta @@ -27,12 +31,21 @@ class {{ cookiecutter.source_name }}Authenticator(OAuthAuthenticator, metaclass= @classmethod def create_for_stream(cls, stream) -> "{{ cookiecutter.source_name }}Authenticator": + """Instantiate an authenticator for a specific Singer stream. + + Args: + stream: The Singer stream instance. + + Returns: + A new authenticator. + """ return cls( stream=stream, auth_endpoint="TODO: OAuth Endpoint URL", oauth_scopes="TODO: OAuth Scopes", ) {%- elif cookiecutter.auth_method == "JWT" %} + from singer_sdk.authenticators import OAuthJWTAuthenticator @@ -41,9 +54,17 @@ class {{ cookiecutter.source_name }}Authenticator(OAuthJWTAuthenticator): @classmethod def create_for_stream(cls, stream) -> "{{ cookiecutter.source_name }}Authenticator": + """Instantiate an authenticator for a specific Singer stream. + + Args: + stream: The Singer stream instance. + + Returns: + A new authenticator. + """ return cls( stream=stream, auth_endpoint="TODO: OAuth Endpoint URL", oauth_scopes="TODO: OAuth Scopes", ) -{% endif %} +{%- endif %} diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml index 052519280..68879ac6a 100644 --- a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml +++ b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml @@ -30,8 +30,9 @@ requests = "^2.28.2" [tool.poetry.dev-dependencies] pytest = "^7.2.1" flake8 = "^5.0.4" +darglint = "^1.8.1" black = "^23.1.0" -pydocstyle = "^6.3.0" +pyupgrade = "^3.3.1" mypy = "^1.0.0" isort = "^5.11.5" {%- if cookiecutter.serialization_method != "SQL" %} diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh index 307cce49b..8ab387f50 100644 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ b/e2e-tests/cookiecutters/test_cookiecutter.sh @@ -48,7 +48,6 @@ LIBRARY_NAME=$(ls * -d | egrep "tap|target") poetry run black $LIBRARY_NAME && poetry run isort $LIBRARY_NAME && poetry run flake8 $LIBRARY_NAME && - poetry run pydocstyle $LIBRARY_NAME && poetry run mypy $LIBRARY_NAME ## diff --git a/poetry.lock b/poetry.lock index 1f68326ec..144e5d6ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -566,8 +566,6 @@ files = [ {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, @@ -1051,7 +1049,6 @@ category = "main" optional = true python-versions = "*" files = [ - {file = "livereload-2.6.3-py2.py3-none-any.whl", hash = "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"}, {file = "livereload-2.6.3.tar.gz", hash = "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869"}, ] @@ -1632,7 +1629,7 @@ files = [ cffi = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5)", "sphinx_rtd_theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] @@ -1814,13 +1811,6 @@ files = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2256,46 +2246,6 @@ category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "SQLAlchemy-1.4.46-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:7001f16a9a8e06488c3c7154827c48455d1c1507d7228d43e781afbc8ceccf6d"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c7a46639ba058d320c9f53a81db38119a74b8a7a1884df44d09fbe807d028aaf"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27m-win32.whl", hash = "sha256:c04144a24103135ea0315d459431ac196fe96f55d3213bfd6d39d0247775c854"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27m-win_amd64.whl", hash = "sha256:7b81b1030c42b003fc10ddd17825571603117f848814a344d305262d370e7c34"}, - {file = "SQLAlchemy-1.4.46-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:939f9a018d2ad04036746e15d119c0428b1e557470361aa798e6e7d7f5875be0"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:b7f4b6aa6e87991ec7ce0e769689a977776db6704947e562102431474799a857"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dbf17ac9a61e7a3f1c7ca47237aac93cabd7f08ad92ac5b96d6f8dea4287fc1"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7f8267682eb41a0584cf66d8a697fef64b53281d01c93a503e1344197f2e01fe"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64cb0ad8a190bc22d2112001cfecdec45baffdf41871de777239da6a28ed74b6"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-win32.whl", hash = "sha256:5f752676fc126edc1c4af0ec2e4d2adca48ddfae5de46bb40adbd3f903eb2120"}, - {file = "SQLAlchemy-1.4.46-cp310-cp310-win_amd64.whl", hash = "sha256:31de1e2c45e67a5ec1ecca6ec26aefc299dd5151e355eb5199cd9516b57340be"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d68e1762997bfebf9e5cf2a9fd0bcf9ca2fdd8136ce7b24bbd3bbfa4328f3e4a"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d112b0f3c1bc5ff70554a97344625ef621c1bfe02a73c5d97cac91f8cd7a41e"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69fac0a7054d86b997af12dc23f581cf0b25fb1c7d1fed43257dee3af32d3d6d"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-win32.whl", hash = "sha256:887865924c3d6e9a473dc82b70977395301533b3030d0f020c38fd9eba5419f2"}, - {file = "SQLAlchemy-1.4.46-cp311-cp311-win_amd64.whl", hash = "sha256:984ee13543a346324319a1fb72b698e521506f6f22dc37d7752a329e9cd00a32"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:9167d4227b56591a4cc5524f1b79ccd7ea994f36e4c648ab42ca995d28ebbb96"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d61e9ecc849d8d44d7f80894ecff4abe347136e9d926560b818f6243409f3c86"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3ec187acf85984263299a3f15c34a6c0671f83565d86d10f43ace49881a82718"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9883f5fae4fd8e3f875adc2add69f8b945625811689a6c65866a35ee9c0aea23"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-win32.whl", hash = "sha256:535377e9b10aff5a045e3d9ada8a62d02058b422c0504ebdcf07930599890eb0"}, - {file = "SQLAlchemy-1.4.46-cp36-cp36m-win_amd64.whl", hash = "sha256:18cafdb27834fa03569d29f571df7115812a0e59fd6a3a03ccb0d33678ec8420"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:a1ad90c97029cc3ab4ffd57443a20fac21d2ec3c89532b084b073b3feb5abff3"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4847f4b1d822754e35707db913396a29d874ee77b9c3c3ef3f04d5a9a6209618"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c5a99282848b6cae0056b85da17392a26b2d39178394fc25700bcf967e06e97a"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4b1cc7835b39835c75cf7c20c926b42e97d074147c902a9ebb7cf2c840dc4e2"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-win32.whl", hash = "sha256:c522e496f9b9b70296a7675272ec21937ccfc15da664b74b9f58d98a641ce1b6"}, - {file = "SQLAlchemy-1.4.46-cp37-cp37m-win_amd64.whl", hash = "sha256:ae067ab639fa499f67ded52f5bc8e084f045d10b5ac7bb928ae4ca2b6c0429a5"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:e3c1808008124850115a3f7e793a975cfa5c8a26ceeeb9ff9cbb4485cac556df"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d164df3d83d204c69f840da30b292ac7dc54285096c6171245b8d7807185aa"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b33ffbdbbf5446cf36cd4cc530c9d9905d3c2fe56ed09e25c22c850cdb9fac92"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d94682732d1a0def5672471ba42a29ff5e21bb0aae0afa00bb10796fc1e28dd"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-win32.whl", hash = "sha256:f8cb80fe8d14307e4124f6fad64dfd87ab749c9d275f82b8b4ec84c84ecebdbe"}, - {file = "SQLAlchemy-1.4.46-cp38-cp38-win_amd64.whl", hash = "sha256:07e48cbcdda6b8bc7a59d6728bd3f5f574ffe03f2c9fb384239f3789c2d95c2e"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:1b1e5e96e2789d89f023d080bee432e2fef64d95857969e70d3cadec80bd26f0"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3714e5b33226131ac0da60d18995a102a17dddd42368b7bdd206737297823ad"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:955162ad1a931fe416eded6bb144ba891ccbf9b2e49dc7ded39274dd9c5affc5"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6e4cb5c63f705c9d546a054c60d326cbde7421421e2d2565ce3e2eee4e1a01f"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-win32.whl", hash = "sha256:51e1ba2884c6a2b8e19109dc08c71c49530006c1084156ecadfaadf5f9b8b053"}, - {file = "SQLAlchemy-1.4.46-cp39-cp39-win_amd64.whl", hash = "sha256:315676344e3558f1f80d02535f410e80ea4e8fddba31ec78fe390eff5fb8f466"}, {file = "SQLAlchemy-1.4.46.tar.gz", hash = "sha256:6913b8247d8a292ef8315162a51931e2b40ce91681f1b6f18f697045200c4a30"}, ] @@ -2305,7 +2255,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] @@ -2315,14 +2265,14 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlalchemy2-stubs"