From 07a63c2c17b146b52b7244d2d707d2df9012598d Mon Sep 17 00:00:00 2001 From: mjsqu Date: Tue, 6 Jun 2023 11:34:22 +1200 Subject: [PATCH 01/15] refactor(taps): Change `SQLStream.schema` into a cached property (#1745) * Change schema into a cached_property Schema is called at least once per record and does not change within a stream, so it should be cached for performance benefits. * Fix missing sys import * Changes for 3.7 compatibility ``` @property @lru_cache() ``` is a backwards compatible version of functools.cached_property for Python 3.7 https://stackoverflow.com/questions/4037481/caching-class-attributes-in-python * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add parentheses to lru_cache for py 3.7 * Fix schema return type mypy checks fail because the schema return type is lru_cache: ``` singer_sdk/streams/sql.py:82: error: Signature of "schema" incompatible with supertype "Stream" [override] singer_sdk/streams/sql.py:154: error: Argument "schema" to "get_selected_schema" has incompatible type "_lru_cache_wrapper[Dict[Any, Any]]"; expected "Dict[Any, Any]" [arg-type] Found 2 errors in 1 file (checked 59 source files) ``` The update should pass over the `_lru_cache_wrapper` type and return a `Dict` as expected * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix for mypy - Remove `lru_cache` - Set `_cached_schema` up as a class variable to hold the schema for each stream - Check for presence of `self._cached_schema` on the stream * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Re-add removed TYPE_CHECKING * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar R. M --- singer_sdk/streams/sql.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/singer_sdk/streams/sql.py b/singer_sdk/streams/sql.py index 3d3de5bbe..7a3fe3886 100644 --- a/singer_sdk/streams/sql.py +++ b/singer_sdk/streams/sql.py @@ -20,6 +20,7 @@ class SQLStream(Stream, metaclass=abc.ABCMeta): """Base class for SQLAlchemy-based streams.""" connector_class = SQLConnector + _cached_schema: dict | None = None def __init__( self, @@ -74,7 +75,7 @@ def metadata(self) -> MetadataMapping: """ return self._singer_catalog_entry.metadata - @property + @property # TODO: Investigate @cached_property after py > 3.7 def schema(self) -> dict: """Return metadata object (dict) as specified in the Singer spec. @@ -83,7 +84,13 @@ def schema(self) -> dict: Returns: The schema object. """ - return t.cast(dict, self._singer_catalog_entry.schema.to_dict()) + if not self._cached_schema: + self._cached_schema = t.cast( + dict, + self._singer_catalog_entry.schema.to_dict(), + ) + + return self._cached_schema @property def tap_stream_id(self) -> str: From 5267160fda096e07ba6ec86725fd98aa9cee9ae9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 22:16:23 -0600 Subject: [PATCH 02/15] chore: pre-commit autoupdate (#1751) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/python-jsonschema/check-jsonschema: 0.23.0 → 0.23.1](https://github.com/python-jsonschema/check-jsonschema/compare/0.23.0...0.23.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8d308ac62..3b4bcc315 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.0 + rev: 0.23.1 hooks: - id: check-dependabot - id: check-github-workflows From a64c66063998859c50f690c46b9de67c3c4cf33e Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 7 Jun 2023 04:56:39 -0600 Subject: [PATCH 03/15] chore: Add `Cookiecutter templates` to feature issue template (#1716) --- .github/ISSUE_TEMPLATE/feature.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index 0c084de5b..ad08d3e6a 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -20,6 +20,7 @@ body: - Targets (data type handling, batching, SQL object generation, etc.) - Configuration (settings parsing, validation, etc.) - CLI (options, error messages, logging, etc.) + - Cookiecutter templates - Other validations: required: true From cf5743901c1b87ad9691e0136b1b08fb42680a6e Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Wed, 7 Jun 2023 08:02:12 -0600 Subject: [PATCH 04/15] fix(taps): Always emit a STATE message at the start of the sync process (#1753) --- singer_sdk/tap_base.py | 4 ++- tests/core/test_parent_child.py | 28 +++++++++---------- tests/samples/test_tap_countries.py | 2 +- .../mapped_stream/aliased_stream.jsonl | 1 + .../changed_key_properties.jsonl | 1 + .../mapped_stream/drop_property.jsonl | 1 + .../drop_property_null_string.jsonl | 1 + .../snapshots/mapped_stream/flatten_all.jsonl | 1 + .../mapped_stream/flatten_depth_1.jsonl | 1 + .../mapped_stream/keep_all_fields.jsonl | 1 + .../mapped_stream/map_and_flatten.jsonl | 1 + tests/snapshots/mapped_stream/no_map.jsonl | 1 + .../mapped_stream/non_pk_passthrough.jsonl | 1 + .../mapped_stream/only_mapped_fields.jsonl | 1 + .../only_mapped_fields_null_string.jsonl | 1 + .../mapped_stream/sourced_stream_1.jsonl | 1 + .../sourced_stream_1_null_string.jsonl | 1 + .../mapped_stream/sourced_stream_2.jsonl | 1 + 18 files changed, 33 insertions(+), 16 deletions(-) diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index e6d999a02..23d8e815b 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -11,7 +11,7 @@ import click -from singer_sdk._singerlib import Catalog +from singer_sdk._singerlib import Catalog, StateMessage, write_message from singer_sdk.configuration._dict_config import merge_missing_config_jsonschema from singer_sdk.exceptions import AbortedSyncFailedException, AbortedSyncPausedException from singer_sdk.helpers import _state @@ -431,6 +431,8 @@ def sync_all(self) -> None: """Sync all streams.""" self._reset_state_progress_markers() self._set_compatible_replication_methods() + write_message(StateMessage(value=self.state)) + stream: Stream for stream in self.streams.values(): if not stream.selected and not stream.has_selected_descendents: diff --git a/tests/core/test_parent_child.py b/tests/core/test_parent_child.py index 5d7d4f2de..a91726bc3 100644 --- a/tests/core/test_parent_child.py +++ b/tests/core/test_parent_child.py @@ -103,19 +103,19 @@ def test_parent_context_fields_in_child(tap: MyTap): messages = _get_messages(tap) # Parent schema is emitted - assert messages[0] - assert messages[0]["type"] == SingerMessageType.SCHEMA - assert messages[0]["stream"] == parent_stream.name - assert messages[0]["schema"] == parent_stream.schema - - # Child schema is emitted assert messages[1] assert messages[1]["type"] == SingerMessageType.SCHEMA - assert messages[1]["stream"] == child_stream.name - assert messages[1]["schema"] == child_stream.schema + assert messages[1]["stream"] == parent_stream.name + assert messages[1]["schema"] == parent_stream.schema + + # Child schema is emitted + assert messages[2] + assert messages[2]["type"] == SingerMessageType.SCHEMA + assert messages[2]["stream"] == child_stream.name + assert messages[2]["schema"] == child_stream.schema # Child records are emitted - child_record_messages = messages[2:5] + child_record_messages = messages[3:6] assert child_record_messages assert all(msg["type"] == SingerMessageType.RECORD for msg in child_record_messages) assert all(msg["stream"] == child_stream.name for msg in child_record_messages) @@ -155,13 +155,13 @@ def test_child_deselected_parent(tap_with_deselected_parent: MyTap): messages = _get_messages(tap_with_deselected_parent) # First message is a schema for the child stream, not the parent - assert messages[0] - assert messages[0]["type"] == SingerMessageType.SCHEMA - assert messages[0]["stream"] == child_stream.name - assert messages[0]["schema"] == child_stream.schema + assert messages[1] + assert messages[1]["type"] == SingerMessageType.SCHEMA + assert messages[1]["stream"] == child_stream.name + assert messages[1]["schema"] == child_stream.schema # Child records are emitted - child_record_messages = messages[1:4] + child_record_messages = messages[2:5] assert child_record_messages assert all(msg["type"] == SingerMessageType.RECORD for msg in child_record_messages) assert all(msg["stream"] == child_stream.name for msg in child_record_messages) diff --git a/tests/samples/test_tap_countries.py b/tests/samples/test_tap_countries.py index 5e7a5cb98..f822f5c8b 100644 --- a/tests/samples/test_tap_countries.py +++ b/tests/samples/test_tap_countries.py @@ -135,4 +135,4 @@ def tally_messages(messages: list) -> t.Counter: assert counter["SCHEMA", "countries"] == 1 assert counter["BATCH", "countries"] == 1 - assert counter[("STATE",)] == 2 + assert counter[("STATE",)] == 3 diff --git a/tests/snapshots/mapped_stream/aliased_stream.jsonl b/tests/snapshots/mapped_stream/aliased_stream.jsonl index 49089dda9..46d5daffe 100644 --- a/tests/snapshots/mapped_stream/aliased_stream.jsonl +++ b/tests/snapshots/mapped_stream/aliased_stream.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "aliased_stream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "aliased_stream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "aliased_stream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/changed_key_properties.jsonl b/tests/snapshots/mapped_stream/changed_key_properties.jsonl index f62049ce8..c5168a45b 100644 --- a/tests/snapshots/mapped_stream/changed_key_properties.jsonl +++ b/tests/snapshots/mapped_stream/changed_key_properties.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}}}, "key_properties": ["email_hash"]} {"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/drop_property.jsonl b/tests/snapshots/mapped_stream/drop_property.jsonl index f4b0b6f71..8694f4736 100644 --- a/tests/snapshots/mapped_stream/drop_property.jsonl +++ b/tests/snapshots/mapped_stream/drop_property.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl index f4b0b6f71..8694f4736 100644 --- a/tests/snapshots/mapped_stream/drop_property_null_string.jsonl +++ b/tests/snapshots/mapped_stream/drop_property_null_string.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/flatten_all.jsonl b/tests/snapshots/mapped_stream/flatten_all.jsonl index 03e7af0cc..c54db1563 100644 --- a/tests/snapshots/mapped_stream/flatten_all.jsonl +++ b/tests/snapshots/mapped_stream/flatten_all.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub__num": {"type": ["integer", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub__num": 1}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub__num": 2}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl index 6765ccdef..275e3295c 100644 --- a/tests/snapshots/mapped_stream/flatten_depth_1.jsonl +++ b/tests/snapshots/mapped_stream/flatten_depth_1.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub": "{\"num\": 1}"}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub": "{\"num\": 2}"}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/keep_all_fields.jsonl b/tests/snapshots/mapped_stream/keep_all_fields.jsonl index 2be7c1105..13ddce438 100644 --- a/tests/snapshots/mapped_stream/keep_all_fields.jsonl +++ b/tests/snapshots/mapped_stream/keep_all_fields.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}, "email_hash": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}}, "email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}}, "email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/map_and_flatten.jsonl b/tests/snapshots/mapped_stream/map_and_flatten.jsonl index 40ecc7e40..bf2620184 100644 --- a/tests/snapshots/mapped_stream/map_and_flatten.jsonl +++ b/tests/snapshots/mapped_stream/map_and_flatten.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user__id": {"type": ["integer", "null"]}, "user__sub__num": {"type": ["integer", "null"]}, "email_hash": {"type": ["string", "null"]}}, "type": "object"}, "key_properties": ["email_hash"]} {"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user__id": 1, "user__sub__num": 1, "email_hash": "c160f8cc69a4f0bf2b0362752353d060"}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user__id": 2, "user__sub__num": 2, "email_hash": "4b9bb80620f03eb3719e0a061c14283d"}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/no_map.jsonl b/tests/snapshots/mapped_stream/no_map.jsonl index f2eeb77fd..019b1f9d9 100644 --- a/tests/snapshots/mapped_stream/no_map.jsonl +++ b/tests/snapshots/mapped_stream/no_map.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl b/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl index 4c2490c6c..0cbbf451a 100644 --- a/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl +++ b/tests/snapshots/mapped_stream/non_pk_passthrough.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"count": {"type": ["integer", "null"]}}}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"count": 21}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"count": 13}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/only_mapped_fields.jsonl b/tests/snapshots/mapped_stream/only_mapped_fields.jsonl index d5341c622..e53042958 100644 --- a/tests/snapshots/mapped_stream/only_mapped_fields.jsonl +++ b/tests/snapshots/mapped_stream/only_mapped_fields.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}, "fixed_count": {"type": ["integer", "null"]}}}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060", "fixed_count": 20}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d", "fixed_count": 12}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl b/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl index d5341c622..e53042958 100644 --- a/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl +++ b/tests/snapshots/mapped_stream/only_mapped_fields_null_string.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "mystream", "schema": {"type": "object", "properties": {"email_hash": {"type": ["string", "null"]}, "fixed_count": {"type": ["integer", "null"]}}}, "key_properties": []} {"type": "RECORD", "stream": "mystream", "record": {"email_hash": "c160f8cc69a4f0bf2b0362752353d060", "fixed_count": 20}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "mystream", "record": {"email_hash": "4b9bb80620f03eb3719e0a061c14283d", "fixed_count": 12}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl index 3d83ea30e..e63d03815 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "sourced_stream_1", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl index 3d83ea30e..e63d03815 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_1_null_string.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "sourced_stream_1", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "sourced_stream_1", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} diff --git a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl index 6a378a5ca..41cce23d7 100644 --- a/tests/snapshots/mapped_stream/sourced_stream_2.jsonl +++ b/tests/snapshots/mapped_stream/sourced_stream_2.jsonl @@ -1,3 +1,4 @@ +{"type": "STATE", "value": {}} {"type": "SCHEMA", "stream": "sourced_stream_2", "schema": {"properties": {"email": {"type": ["string", "null"]}, "count": {"type": ["integer", "null"]}, "user": {"properties": {"id": {"type": ["integer", "null"]}, "sub": {"properties": {"num": {"type": ["integer", "null"]}}, "type": ["object", "null"]}}, "type": ["object", "null"]}}, "type": "object"}, "key_properties": []} {"type": "RECORD", "stream": "sourced_stream_2", "record": {"email": "alice@example.com", "count": 21, "user": {"id": 1, "sub": {"num": 1}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} {"type": "RECORD", "stream": "sourced_stream_2", "record": {"email": "bob@example.com", "count": 13, "user": {"id": 2, "sub": {"num": 2}}}, "time_extracted": "2022-01-01T00:00:00+00:00"} From 6296599a6955d8239e04cfb7ccaa0963dcc0ae4b Mon Sep 17 00:00:00 2001 From: Ken Payne Date: Wed, 7 Jun 2023 16:08:51 +0100 Subject: [PATCH 05/15] fix: change runner scope to function for target tests (#1752) Co-authored-by: Edgar R. M --- samples/sample_target_csv/csv_target_sink.py | 1 + .../parquet_target_sink.py | 22 +- singer_sdk/testing/__init__.py | 3 +- singer_sdk/testing/factory.py | 487 ++++++++++++------ singer_sdk/testing/runners.py | 2 +- singer_sdk/testing/target_tests.py | 16 + tests/samples/test_target_parquet.py | 4 +- 7 files changed, 359 insertions(+), 176 deletions(-) diff --git a/samples/sample_target_csv/csv_target_sink.py b/samples/sample_target_csv/csv_target_sink.py index f06c3163d..4f02bf0bb 100644 --- a/samples/sample_target_csv/csv_target_sink.py +++ b/samples/sample_target_csv/csv_target_sink.py @@ -41,6 +41,7 @@ def process_batch(self, context: dict) -> None: delimiter="\t", quotechar='"', quoting=csv.QUOTE_NONNUMERIC, + escapechar="\\", ) for record in records_to_drain: if newfile and not records_written: diff --git a/samples/sample_target_parquet/parquet_target_sink.py b/samples/sample_target_parquet/parquet_target_sink.py index 40691c13c..e98dca2b1 100644 --- a/samples/sample_target_parquet/parquet_target_sink.py +++ b/samples/sample_target_parquet/parquet_target_sink.py @@ -36,7 +36,8 @@ def _json_schema_to_arrow_fields(schema: dict[str, t.Any]) -> pa.StructType: for name, property_schema in schema.get("properties", {}).items(): field = pa.field(name, _json_type_to_arrow_field(property_schema)) fields.append(field) - return fields + + return fields if fields else [pa.field("dummy", pa.string())] def _json_type_to_arrow_field( # noqa: PLR0911 @@ -99,22 +100,3 @@ def process_batch(self, context: dict) -> None: table = pa.Table.from_pylist(records_to_drain, schema=schema) writer.write_table(table) writer.close() - - @staticmethod - def translate_data_type(singer_type: str | dict) -> t.Any: - """Translate from singer_type to a native type.""" - if singer_type in ["decimal", "float", "double"]: - return pa.decimal128 - if singer_type in ["date-time"]: - return pa.datetime - if singer_type in ["date"]: - return pa.date64 - return pa.string - - def _get_parquet_schema(self) -> list[tuple[str, t.Any]]: - col_list: list[tuple[str, t.Any]] = [] - for prop in self.schema["properties"]: - col_list.append( - (prop["name"], self.translate_data_type(prop["type"])), - ) - return col_list diff --git a/singer_sdk/testing/__init__.py b/singer_sdk/testing/__init__.py index 3ca07d964..24ce4ac9f 100644 --- a/singer_sdk/testing/__init__.py +++ b/singer_sdk/testing/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations from .config import SuiteConfig -from .factory import get_tap_test_class, get_target_test_class, get_test_class +from .factory import get_tap_test_class, get_target_test_class from .legacy import ( _get_tap_catalog, _select_all, @@ -19,7 +19,6 @@ __all__ = [ "get_tap_test_class", "get_target_test_class", - "get_test_class", "_get_tap_catalog", "_select_all", "get_standard_tap_tests", diff --git a/singer_sdk/testing/factory.py b/singer_sdk/testing/factory.py index 32c9e6469..99c9058c4 100644 --- a/singer_sdk/testing/factory.py +++ b/singer_sdk/testing/factory.py @@ -19,134 +19,331 @@ from singer_sdk import Tap, Target -def get_test_class( # noqa: C901 - test_runner: TapTestRunner | TargetTestRunner, - test_suites: list, - suite_config: SuiteConfig | None, -) -> object: - """Construct a valid pytest test class from given suites. - - Args: - test_runner: A Tap or Target test runner instance. - test_suites: A list of Test Suits to apply. - suite_config: SuiteConfig instance to pass to tests. - - Returns: - A test class usable by pytest. - """ - - class BaseTestClass: - """Base test class.""" - - params: dict = {} - param_ids: dict = {} - - @pytest.fixture - def config(self) -> SuiteConfig: - return suite_config or SuiteConfig() - - @pytest.fixture - def resource(self) -> t.Any: # noqa: ANN401, PT004 - yield # noqa: PT022 - - @pytest.fixture(scope="class") - def runner(self) -> TapTestRunner | TargetTestRunner: - # Populate runner class with cached records for use in tests - test_runner.sync_all() - return test_runner - - for suite in test_suites: - # make sure given runner is of type TapTestRunner - expected_runner_class = ( - TapTestRunner - if suite.kind in {"tap", "tap_stream", "tap_stream_attribute"} - else TargetTestRunner +class BaseTestClass: + """Base test class.""" + + params: dict = {} + param_ids: dict = {} + + +class TapTestClassFactory: + """Factory for Tap Test Classes.""" + + def __init__( + self, + tap_class: type[Tap], + *, + config: dict | None = None, + ): + """Initialize TapTestClassFactory. + + Args: + tap_class: Tap class to be tested. + config: Tap configuration for testing. + """ + self.tap_class = tap_class + self.config = config + + def new_test_class( + self, + *, + include_tap_tests: bool = True, + include_stream_tests: bool = True, + include_stream_attribute_tests: bool = True, + custom_suites: list | None = None, + suite_config: SuiteConfig | None = None, + **kwargs: t.Any, + ) -> type[BaseTestClass]: + """Get a new test class. + + Args: + include_tap_tests: Include tap tests in the test class. + include_stream_tests: Include stream tests in the test class. + include_stream_attribute_tests: + Include stream attribute tests in the test class. + custom_suites: List of custom test suites to include in the test class. + suite_config: SuiteConfig instance to be used when instantiating tests. + kwargs: Default arguments to be passed to tap on create. + + Returns: + A new test class. + """ + # compile test suites + suites = custom_suites or [] + if include_tap_tests: + suites.append(tap_tests) + if include_stream_tests: + suites.append(tap_stream_tests) + if include_stream_attribute_tests: + suites.append(tap_stream_attribute_tests) + + # set default values + if "parse_env_config" not in kwargs: + kwargs["parse_env_config"] = True + + # create singleton test runner + test_runner = TapTestRunner( + tap_class=self.tap_class, + config=self.config, + suite_config=suite_config, + **kwargs, ) - assert isinstance(test_runner, expected_runner_class), ( - f"Test suite of kind {suite.kind} passed, " - f"but test runner if of type {type(test_runner)}." + + empty_test_class = self._get_empty_test_class( + test_runner=test_runner, + suite_config=suite_config, ) - test_runner = t.cast( - expected_runner_class, # type: ignore[valid-type] - test_runner, + return self._annotate_test_class( + empty_test_class=empty_test_class, + test_suites=suites, + test_runner=test_runner, ) - if suite.kind in {"tap", "target"}: - for test_class in suite.tests: - test = test_class() - test_name = f"test_{suite.kind}_{test.name}" - setattr(BaseTestClass, f"test_{suite.kind}_{test.name}", test.run) - - if suite.kind in {"tap_stream", "tap_stream_attribute"}: - streams = list(test_runner.new_tap().streams.values()) - - if suite.kind == "tap_stream": - params = [ - { - "stream": stream, - } - for stream in streams - ] - param_ids = [stream.name for stream in streams] - + def _get_empty_test_class( + self, + test_runner: TapTestRunner, + suite_config: SuiteConfig | None, + ) -> type[BaseTestClass]: + """Get an empty test class. + + Args: + test_runner: Test runner to be used in the test class. + suite_config: SuiteConfig instance to be used when instantiating tests. + + Returns: + An empty test class. + """ + + class TapTestClass(BaseTestClass): + """Tap Test Class.""" + + @pytest.fixture + def config(self) -> SuiteConfig: + return suite_config or SuiteConfig() + + @pytest.fixture + def resource(self) -> t.Any: # noqa: ANN401, PT004 + yield # noqa: PT022 + + @pytest.fixture(scope="class") + def runner(self) -> TapTestRunner | TargetTestRunner: + # Populate runner class with cached records for use in tests + test_runner.sync_all() + return test_runner + + return TapTestClass + + def _annotate_test_class( # noqa: C901 + self, + empty_test_class: type[BaseTestClass], + test_suites: list, + test_runner: TapTestRunner, + ) -> type[BaseTestClass]: + """Annotate test class with test methods. + + Args: + empty_test_class: Empty test class to be annotated. + test_suites: List of test suites to include in the test class. + test_runner: Test runner to be used in the test class. + + Returns: + An annotated test class. + """ + for suite in test_suites: + if suite.kind == "tap": for test_class in suite.tests: test = test_class() test_name = f"test_{suite.kind}_{test.name}" - setattr( - BaseTestClass, - test_name, - test.run, - ) - BaseTestClass.params[test_name] = params - BaseTestClass.param_ids[test_name] = param_ids - - if suite.kind == "tap_stream_attribute": - for test_class in suite.tests: - test = test_class() - test_name = f"test_{suite.kind}_{test.name}" - test_params = [] - test_ids = [] - for stream in streams: - test_params.extend( - [ - { - "stream": stream, - "attribute_name": property_name, - } - for property_name, property_schema in stream.schema[ - "properties" - ].items() - if test_class.evaluate( - stream=stream, - property_name=property_name, - property_schema=property_schema, - ) - ], - ) - test_ids.extend( - [ - f"{stream.name}.{property_name}" - for property_name, property_schema in stream.schema[ - "properties" - ].items() - if test_class.evaluate( - stream=stream, - property_name=property_name, - property_schema=property_schema, - ) - ], - ) - - if test_params: + setattr(empty_test_class, test_name, test.run) + + if suite.kind in {"tap_stream", "tap_stream_attribute"}: + streams = list(test_runner.new_tap().streams.values()) + + if suite.kind == "tap_stream": + params = [ + { + "stream": stream, + } + for stream in streams + ] + param_ids = [stream.name for stream in streams] + + for test_class in suite.tests: + test = test_class() + test_name = f"test_{suite.kind}_{test.name}" setattr( - BaseTestClass, + empty_test_class, test_name, test.run, ) - BaseTestClass.params[test_name] = test_params - BaseTestClass.param_ids[test_name] = test_ids + empty_test_class.params[test_name] = params + empty_test_class.param_ids[test_name] = param_ids + + if suite.kind == "tap_stream_attribute": + for test_class in suite.tests: + test = test_class() + test_name = f"test_{suite.kind}_{test.name}" + test_params = [] + test_ids = [] + for stream in streams: + test_params.extend( + [ + { + "stream": stream, + "attribute_name": property_name, + } + for property_name, property_schema in stream.schema[ + "properties" + ].items() + if test_class.evaluate( + stream=stream, + property_name=property_name, + property_schema=property_schema, + ) + ], + ) + test_ids.extend( + [ + f"{stream.name}.{property_name}" + for property_name, property_schema in stream.schema[ + "properties" + ].items() + if test_class.evaluate( + stream=stream, + property_name=property_name, + property_schema=property_schema, + ) + ], + ) + + if test_params: + setattr( + empty_test_class, + test_name, + test.run, + ) + empty_test_class.params[test_name] = test_params + empty_test_class.param_ids[test_name] = test_ids + + return empty_test_class + + +class TargetTestClassFactory: + """Factory for Target Test Classes.""" + + def __init__(self, target_class: type[Target], *, config: dict | None = None): + """Initialize TargetTestClassFactory. + + Args: + target_class: Target class to be tested. + config: Config to be used when instantiating tests. + """ + self.target_class = target_class + self.config = config + + def new_test_class( + self, + *, + custom_suites: list | None = None, + suite_config: SuiteConfig | None = None, + include_target_tests: bool = True, + **kwargs: t.Any, + ) -> type[BaseTestClass]: + """Get a new Target test class. + + Args: + custom_suites: List of custom test suites to include in the test class. + suite_config: SuiteConfig instance to be used when instantiating tests. + include_target_tests: Whether to include target tests in the test class. + kwargs: Keyword arguments to be passed to the Target on run. + + Returns: + A new Target test class. + """ + # compile test suites + suites = custom_suites or [] + if include_target_tests: + suites.append(target_tests) + + # set default values + if "parse_env_config" not in kwargs: + kwargs["parse_env_config"] = True + + empty_test_class = self._get_empty_test_class( + target_class=self.target_class, + config=self.config, + suite_config=suite_config, + **kwargs, + ) + return self._annotate_test_class( + empty_test_class=empty_test_class, + test_suites=suites, + ) + + def _get_empty_test_class( + self, + target_class: type[Target], + suite_config: SuiteConfig | None, + config: dict | None = None, + **kwargs: t.Any, + ) -> type[BaseTestClass]: + """Get an empty test class. + + Args: + target_class: Target class to be tested. + suite_config: SuiteConfig instance to be used when instantiating tests. + config: Config to be used when instantiating tests. + kwargs: Keyword arguments to be passed to the Target on run. + + Returns: + An empty test class. + """ + + class TargetTestClass(BaseTestClass): + """Target Test Class.""" + + @pytest.fixture + def config(self) -> SuiteConfig: + return suite_config or SuiteConfig() + + @pytest.fixture + def resource(self) -> t.Any: # noqa: ANN401, PT004 + yield # noqa: PT022 + + @pytest.fixture + def runner(self) -> TargetTestRunner: + # Instantiate new runner class and populate records for use in tests + return TargetTestRunner( + target_class=target_class, + config=config, + suite_config=suite_config, + **kwargs, + ) + + return TargetTestClass + + def _annotate_test_class( + self, + empty_test_class: type[BaseTestClass], + test_suites: list, + ) -> type[BaseTestClass]: + """Annotate test class with test methods. + + Args: + empty_test_class: Empty test class to be annotated. + test_suites: List of test suites to be included in the test class. + + Returns: + Annotated test class. + """ + for suite in test_suites: + if suite.kind == "target": + for test_class in suite.tests: + test = test_class() + test_name = f"test_{suite.kind}_{test.name}" + setattr(empty_test_class, test_name, test.run) - return BaseTestClass + return empty_test_class def get_tap_test_class( @@ -159,7 +356,7 @@ def get_tap_test_class( custom_suites: list | None = None, suite_config: SuiteConfig | None = None, **kwargs: t.Any, -) -> object: +) -> type[BaseTestClass]: """Get Tap Test Class. Args: @@ -175,27 +372,17 @@ def get_tap_test_class( Returns: A test class usable by pytest. """ - suites = custom_suites or [] - if include_tap_tests: - suites.append(tap_tests) - if include_stream_tests: - suites.append(tap_stream_tests) - if include_stream_attribute_tests: - suites.append(tap_stream_attribute_tests) - - # set default values - if "parse_env_config" not in kwargs: - kwargs["parse_env_config"] = True - - return get_test_class( - test_runner=TapTestRunner( - tap_class=tap_class, - config=config, - suite_config=suite_config, - **kwargs, - ), - test_suites=suites, + factory = TapTestClassFactory( + tap_class=tap_class, + config=config, + ) + return factory.new_test_class( + custom_suites=custom_suites, suite_config=suite_config, + include_tap_tests=include_tap_tests, + include_stream_tests=include_stream_tests, + include_stream_attribute_tests=include_stream_attribute_tests, + **kwargs, ) @@ -205,8 +392,9 @@ def get_target_test_class( config: dict | None = None, custom_suites: list | None = None, suite_config: SuiteConfig | None = None, + include_target_tests: bool = True, **kwargs: t.Any, -) -> object: +) -> type[BaseTestClass]: """Get Target Test Class. Args: @@ -214,24 +402,19 @@ def get_target_test_class( config: Config dict to use for testing. custom_suites: Custom test suites to add to standard tests. suite_config: SuiteConfig instance to pass to tests. + include_target_tests: Include standard target tests. kwargs: Keyword arguments to pass to the TapRunner. Returns: A test class usable by pytest. """ - suites = custom_suites or [] - suites.append(target_tests) - - # set default values - if "parse_env_config" not in kwargs: - kwargs["parse_env_config"] = True - - return get_test_class( - test_runner=TargetTestRunner( - target_class=target_class, - config=config, - **kwargs, - ), - test_suites=suites, + factory = TargetTestClassFactory( + target_class=target_class, + config=config, + ) + return factory.new_test_class( + custom_suites=custom_suites, suite_config=suite_config, + include_target_tests=include_target_tests, + **kwargs, ) diff --git a/singer_sdk/testing/runners.py b/singer_sdk/testing/runners.py index c32461d6b..16f2ba905 100644 --- a/singer_sdk/testing/runners.py +++ b/singer_sdk/testing/runners.py @@ -242,7 +242,7 @@ def target_input(self) -> t.IO[str]: if self.input_io: self._input = self.input_io elif self.input_filepath: - self._input = Path(self.input_filepath).open() + self._input = Path(self.input_filepath).open(encoding="utf8") return t.cast(t.IO[str], self._input) @target_input.setter diff --git a/singer_sdk/testing/target_tests.py b/singer_sdk/testing/target_tests.py index 50e48622a..8b582ed12 100644 --- a/singer_sdk/testing/target_tests.py +++ b/singer_sdk/testing/target_tests.py @@ -2,6 +2,8 @@ from __future__ import annotations +import pytest + from .templates import TargetFileTestTemplate, TargetTestTemplate @@ -52,6 +54,13 @@ class TargetInvalidSchemaTest(TargetFileTestTemplate): name = "invalid_schema" + def test(self) -> None: + """Run test.""" + # TODO: the SDK should raise a better error than Exception in this case + # https://github.com/meltano/sdk/issues/1755 + with pytest.raises(Exception): # noqa: PT011, B017 + super().test() + class TargetMultipleStateMessages(TargetFileTestTemplate): """Test Target correctly relays multiple received State messages (checkpoints).""" @@ -86,6 +95,13 @@ class TargetRecordBeforeSchemaTest(TargetFileTestTemplate): name = "record_before_schema" + def test(self) -> None: + """Run test.""" + # TODO: the SDK should raise a better error than KeyError in this case + # https://github.com/meltano/sdk/issues/1755 + with pytest.raises(KeyError): + super().test() + class TargetRecordMissingKeyProperty(TargetFileTestTemplate): """Test Target handles record missing key property.""" diff --git a/tests/samples/test_target_parquet.py b/tests/samples/test_target_parquet.py index 5c984679b..7f53036a4 100644 --- a/tests/samples/test_target_parquet.py +++ b/tests/samples/test_target_parquet.py @@ -12,7 +12,9 @@ SAMPLE_FILEPATH = Path(f".output/test_{uuid.uuid4()}/") SAMPLE_FILENAME = SAMPLE_FILEPATH / "testfile.parquet" -SAMPLE_CONFIG = {"filepath": str(SAMPLE_FILENAME)} +SAMPLE_CONFIG = { + "filepath": str(SAMPLE_FILENAME), +} StandardTests = get_target_test_class( target_class=SampleTargetParquet, From 3ceb28a97a77a5e22163fc887fe16ff129c2dc28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 9 Jun 2023 14:46:36 +0000 Subject: [PATCH 06/15] chore(deps-dev): Bump requests-mock from 1.10.0 to 1.11.0 (#1758) --- poetry.lock | 143 +++------------------------------------------------- 1 file changed, 7 insertions(+), 136 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5f6934978..eaad584e1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "appdirs" version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = false python-versions = "*" files = [ @@ -28,7 +26,6 @@ files = [ name = "argcomplete" version = "3.0.8" description = "Bash tab completion for argparse" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -46,7 +43,6 @@ test = ["coverage", "mypy", "pexpect", "ruff", "wheel"] name = "arrow" version = "1.2.3" description = "Better dates & times for Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -62,7 +58,6 @@ typing-extensions = {version = "*", markers = "python_version < \"3.8\""} name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -84,7 +79,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.12.1" description = "Internationalization utilities" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -99,7 +93,6 @@ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} name = "backoff" version = "2.2.1" description = "Function decoration for backoff and retry" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -111,7 +104,6 @@ files = [ name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" -category = "main" optional = true python-versions = ">=3.6.0" files = [ @@ -130,7 +122,6 @@ lxml = ["lxml"] name = "binaryornot" version = "0.4.4" description = "Ultra-lightweight pure Python package to check if a file is binary or text." -category = "dev" optional = false python-versions = "*" files = [ @@ -145,7 +136,6 @@ chardet = ">=3.0.2" name = "black" version = "23.3.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -196,7 +186,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "boto3" version = "1.26.129" description = "The AWS SDK for Python" -category = "main" optional = true python-versions = ">= 3.7" files = [ @@ -216,7 +205,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.129" description = "Low-level, data-driven core of boto 3." -category = "main" optional = true python-versions = ">= 3.7" files = [ @@ -236,7 +224,6 @@ crt = ["awscrt (==0.16.9)"] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -248,7 +235,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -325,7 +311,6 @@ pycparser = "*" name = "chardet" version = "5.1.0" description = "Universal encoding detector for Python 3" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -337,7 +322,6 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -422,7 +406,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -438,7 +421,6 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -450,7 +432,6 @@ files = [ name = "commitizen" version = "3.2.1" description = "Python commitizen client tool" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -476,7 +457,6 @@ typing-extensions = {version = ">=4.0.1,<5.0.0", markers = "python_version < \"3 name = "commitizen-version-bump" version = "0.1.0" description = "Commitizen customized for Meltano projects (https://commitizen-tools.github.io/commitizen/customization)" -category = "dev" optional = false python-versions = "^3.7" files = [] @@ -496,7 +476,6 @@ resolved_reference = "2ac24303b30441773d95357d5c2801275211ce5f" name = "cookiecutter" version = "2.1.1" description = "A command-line utility that creates projects from project templates, e.g. creating a Python package project from a Python package project template." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -517,7 +496,6 @@ requests = ">=2.23.0" name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -593,7 +571,6 @@ toml = ["tomli"] name = "cryptography" version = "41.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -635,7 +612,6 @@ test-randomorder = ["pytest-randomly"] name = "darglint" version = "1.8.1" description = "A utility for ensuring Google-style docstrings stay up to date with the source code." -category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -647,7 +623,6 @@ files = [ name = "decli" version = "0.6.0" description = "Minimal, easy-to-use, declarative cli tool" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -659,7 +634,6 @@ files = [ name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -671,7 +645,6 @@ files = [ name = "deprecated" version = "1.2.13" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -689,7 +662,6 @@ dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version name = "docutils" version = "0.19" description = "Docutils -- Python Documentation Utilities" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -701,7 +673,6 @@ files = [ name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -716,7 +687,6 @@ test = ["pytest (>=6)"] name = "flake8" version = "3.9.2" description = "the modular source code checker: pep8 pyflakes and co" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -734,7 +704,6 @@ pyflakes = ">=2.3.0,<2.4.0" name = "flake8-annotations" version = "2.9.1" description = "Flake8 Type Annotation Checks" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -751,7 +720,6 @@ typed-ast = {version = ">=1.4,<2.0", markers = "python_version < \"3.8\""} name = "flake8-docstrings" version = "1.7.0" description = "Extension for flake8 which uses pydocstyle to check docstrings" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -767,7 +735,6 @@ pydocstyle = ">=2.1" name = "freezegun" version = "1.2.2" description = "Let your Python tests travel through time" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -782,7 +749,6 @@ python-dateutil = ">=2.7" name = "fs" version = "2.4.16" description = "Python's filesystem abstraction layer" -category = "main" optional = false python-versions = "*" files = [ @@ -802,7 +768,6 @@ scandir = ["scandir (>=1.5,<2.0)"] name = "fs-s3fs" version = "1.1.1" description = "Amazon S3 filesystem for PyFilesystem2" -category = "main" optional = true python-versions = "*" files = [ @@ -819,7 +784,6 @@ six = ">=1.10,<2.0" name = "furo" version = "2023.3.27" description = "A clean customisable Sphinx documentation theme." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -837,7 +801,6 @@ sphinx-basic-ng = "*" name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -911,7 +874,6 @@ test = ["objgraph", "psutil"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -923,7 +885,6 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" -category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -935,7 +896,6 @@ files = [ name = "importlib-metadata" version = "4.13.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -956,7 +916,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -975,7 +934,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "inflection" version = "0.5.1" description = "A port of Ruby on Rails inflector to Python" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -987,7 +945,6 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -999,7 +956,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1017,7 +973,6 @@ i18n = ["Babel (>=2.7)"] name = "jinja2-time" version = "0.2.0" description = "Jinja2 Extension for Dates and Times" -category = "dev" optional = false python-versions = "*" files = [ @@ -1033,7 +988,6 @@ jinja2 = "*" name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1045,7 +999,6 @@ files = [ name = "joblib" version = "1.2.0" description = "Lightweight pipelining with Python functions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1057,7 +1010,6 @@ files = [ name = "jsonpath-ng" version = "1.5.3" description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -category = "main" optional = false python-versions = "*" files = [ @@ -1075,7 +1027,6 @@ six = "*" name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1099,7 +1050,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "livereload" version = "2.6.3" description = "Python LiveReload is an awesome tool for web developers" -category = "main" optional = true python-versions = "*" files = [ @@ -1115,7 +1065,6 @@ tornado = {version = "*", markers = "python_version > \"2.7\""} name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1141,7 +1090,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1201,7 +1149,6 @@ files = [ name = "mccabe" version = "0.6.1" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = "*" files = [ @@ -1213,7 +1160,6 @@ files = [ name = "mdit-py-plugins" version = "0.3.5" description = "Collection of plugins for markdown-it-py" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1233,7 +1179,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1245,7 +1190,6 @@ files = [ name = "memoization" version = "0.4.0" description = "A powerful caching library for Python, with TTL support and multiple algorithm options. (https://github.com/lonelyenvoy/python-memoization)" -category = "main" optional = false python-versions = ">=3, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" files = [ @@ -1256,7 +1200,6 @@ files = [ name = "mypy" version = "1.3.0" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1304,7 +1247,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1316,7 +1258,6 @@ files = [ name = "myst-parser" version = "1.0.0" description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1344,7 +1285,6 @@ testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4, name = "numpy" version = "1.21.6" description = "NumPy is the fundamental package for array computing with Python." -category = "dev" optional = false python-versions = ">=3.7,<3.11" files = [ @@ -1385,7 +1325,6 @@ files = [ name = "numpy" version = "1.24.3" description = "Fundamental package for array computing in Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1423,7 +1362,6 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1435,7 +1373,6 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1447,7 +1384,6 @@ files = [ name = "pendulum" version = "2.1.2" description = "Python datetimes made easy" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1482,7 +1418,6 @@ pytzdata = ">=2020.1" name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1494,7 +1429,6 @@ files = [ name = "platformdirs" version = "3.5.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1513,7 +1447,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1532,7 +1465,6 @@ testing = ["pytest", "pytest-benchmark"] name = "ply" version = "3.11" description = "Python Lex & Yacc" -category = "main" optional = false python-versions = "*" files = [ @@ -1544,7 +1476,6 @@ files = [ name = "prompt-toolkit" version = "3.0.38" description = "Library for building powerful interactive command lines in Python" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -1559,7 +1490,6 @@ wcwidth = "*" name = "pyarrow" version = "12.0.0" description = "Python library for Apache Arrow" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1597,7 +1527,6 @@ numpy = ">=1.16.6" name = "pycodestyle" version = "2.7.0" description = "Python style guide checker" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1609,7 +1538,6 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1621,7 +1549,6 @@ files = [ name = "pydocstyle" version = "6.3.0" description = "Python docstring style checker" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1640,7 +1567,6 @@ toml = ["tomli (>=1.2.3)"] name = "pyflakes" version = "2.3.1" description = "passive checker of Python programs" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1652,7 +1578,6 @@ files = [ name = "pygithub" version = "1.58.1" description = "Use the full Github API v3" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1670,7 +1595,6 @@ requests = ">=2.14.0" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1685,7 +1609,6 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1707,7 +1630,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pynacl" version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1734,7 +1656,6 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1771,7 +1692,6 @@ files = [ name = "pytest" version = "7.3.1" description = "pytest: simple powerful testing with Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1795,7 +1715,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-durations" version = "1.2.0" description = "Pytest plugin reporting fixtures and test functions execution time." -category = "main" optional = true python-versions = ">=3.6.2" files = [ @@ -1810,7 +1729,6 @@ pytest = ">=4.6" name = "pytest-snapshot" version = "0.9.0" description = "A plugin for snapshot testing with pytest." -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -1825,7 +1743,6 @@ pytest = ">=3.0.0" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1840,7 +1757,6 @@ six = ">=1.5" name = "python-dotenv" version = "0.21.1" description = "Read key-value pairs from a .env file and set them as environment variables" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1855,7 +1771,6 @@ cli = ["click (>=5.0)"] name = "python-slugify" version = "8.0.1" description = "A Python slugify application that also handles Unicode" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1873,7 +1788,6 @@ unidecode = ["Unidecode (>=1.1.1)"] name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -1885,7 +1799,6 @@ files = [ name = "pytzdata" version = "2020.1" description = "The Olson timezone database for Python." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1897,7 +1810,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1947,7 +1859,6 @@ files = [ name = "questionary" version = "1.10.0" description = "Python library to build pretty command line user prompts ⭐️" -category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1965,7 +1876,6 @@ docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphin name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1985,14 +1895,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-mock" -version = "1.10.0" +version = "1.11.0" description = "Mock out responses from the requests package" -category = "dev" optional = false python-versions = "*" files = [ - {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, - {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, ] [package.dependencies] @@ -2001,13 +1910,12 @@ six = "*" [package.extras] fixture = ["fixtures"] -test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] [[package]] name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "main" optional = true python-versions = ">= 3.7" files = [ @@ -2025,7 +1933,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "setuptools" version = "67.7.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2042,7 +1949,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "simplejson" version = "3.19.1" description = "Simple, fast, extensible JSON encoder/decoder for Python" -category = "main" optional = false python-versions = ">=2.5, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2137,7 +2043,6 @@ files = [ name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2149,7 +2054,6 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -category = "main" optional = false python-versions = "*" files = [ @@ -2161,7 +2065,6 @@ files = [ name = "soupsieve" version = "2.4.1" description = "A modern CSS selector implementation for Beautiful Soup." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2173,7 +2076,6 @@ files = [ name = "sphinx" version = "5.3.0" description = "Python documentation generator" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2209,7 +2111,6 @@ test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] name = "sphinx-autobuild" version = "2021.3.14" description = "Rebuild Sphinx documentation on changes, with live-reload in the browser." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2229,7 +2130,6 @@ test = ["pytest", "pytest-cov"] name = "sphinx-basic-ng" version = "1.0.0b1" description = "A modern skeleton for Sphinx themes." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2247,7 +2147,6 @@ docs = ["furo", "ipython", "myst-parser", "sphinx-copybutton", "sphinx-inline-ta name = "sphinx-copybutton" version = "0.5.2" description = "Add a copy button to each of your code cells." -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2266,7 +2165,6 @@ rtd = ["ipython", "myst-nb", "sphinx", "sphinx-book-theme", "sphinx-examples"] name = "sphinx-reredirects" version = "0.1.2" description = "Handles redirects for moved pages in Sphinx documentation projects" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2281,7 +2179,6 @@ sphinx = "*" name = "sphinxcontrib-applehelp" version = "1.0.2" description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2297,7 +2194,6 @@ test = ["pytest"] name = "sphinxcontrib-devhelp" version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2313,7 +2209,6 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2329,7 +2224,6 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2344,7 +2238,6 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-qthelp" version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2360,7 +2253,6 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2376,7 +2268,6 @@ test = ["pytest"] name = "sqlalchemy" version = "1.4.48" description = "Database Abstraction Library" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -2424,7 +2315,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"} importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [package.extras] @@ -2452,7 +2343,6 @@ sqlcipher = ["sqlcipher3-binary"] name = "sqlalchemy2-stubs" version = "0.0.2a34" description = "Typing Stubs for SQLAlchemy 1.4" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2467,7 +2357,6 @@ typing-extensions = ">=3.7.4" name = "termcolor" version = "2.3.0" description = "ANSI color formatting for output in terminal" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2482,7 +2371,6 @@ tests = ["pytest", "pytest-cov"] name = "text-unidecode" version = "1.3" description = "The most basic Text::Unidecode port" -category = "dev" optional = false python-versions = "*" files = [ @@ -2494,7 +2382,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2506,7 +2393,6 @@ files = [ name = "tomlkit" version = "0.11.8" description = "Style preserving TOML library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2518,7 +2404,6 @@ files = [ name = "tornado" version = "6.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "main" optional = true python-versions = ">= 3.7" files = [ @@ -2539,7 +2424,6 @@ files = [ name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2573,7 +2457,6 @@ files = [ name = "types-jsonschema" version = "4.17.0.8" description = "Typing stubs for jsonschema" -category = "dev" optional = false python-versions = "*" files = [ @@ -2585,7 +2468,6 @@ files = [ name = "types-python-dateutil" version = "2.8.19.13" description = "Typing stubs for python-dateutil" -category = "dev" optional = false python-versions = "*" files = [ @@ -2597,7 +2479,6 @@ files = [ name = "types-pytz" version = "2023.3.0.0" description = "Typing stubs for pytz" -category = "dev" optional = false python-versions = "*" files = [ @@ -2609,7 +2490,6 @@ files = [ name = "types-pyyaml" version = "6.0.12.10" description = "Typing stubs for PyYAML" -category = "dev" optional = false python-versions = "*" files = [ @@ -2621,7 +2501,6 @@ files = [ name = "types-requests" version = "2.31.0.1" description = "Typing stubs for requests" -category = "dev" optional = false python-versions = "*" files = [ @@ -2636,7 +2515,6 @@ types-urllib3 = "*" name = "types-simplejson" version = "3.19.0.1" description = "Typing stubs for simplejson" -category = "dev" optional = false python-versions = "*" files = [ @@ -2648,7 +2526,6 @@ files = [ name = "types-urllib3" version = "1.26.25.12" description = "Typing stubs for urllib3" -category = "dev" optional = false python-versions = "*" files = [ @@ -2660,7 +2537,6 @@ files = [ name = "typing-extensions" version = "4.6.3" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2672,7 +2548,6 @@ files = [ name = "urllib3" version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2689,7 +2564,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "dev" optional = false python-versions = "*" files = [ @@ -2701,7 +2575,6 @@ files = [ name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2786,7 +2659,6 @@ files = [ name = "xdoctest" version = "1.1.1" description = "A rewrite of the builtin doctest module" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2814,7 +2686,6 @@ tests-strict = ["codecov (==2.0.15)", "pytest (==4.6.0)", "pytest (==4.6.0)", "p name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2827,7 +2698,7 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -docs = ["sphinx", "furo", "sphinx-copybutton", "myst-parser", "sphinx-autobuild", "sphinx-reredirects"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-autobuild", "sphinx-copybutton", "sphinx-reredirects"] s3 = ["fs-s3fs"] testing = ["pytest", "pytest-durations"] From 0cf280be1378caaadde45cad00292f644bed88dd Mon Sep 17 00:00:00 2001 From: Ken Payne Date: Fri, 16 Jun 2023 12:29:05 +0100 Subject: [PATCH 07/15] chore: add tests to issue template options (#1764) --- .github/ISSUE_TEMPLATE/feature.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index ad08d3e6a..a5afd5974 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -16,8 +16,8 @@ body: label: Feature scope description: Functionality this new feature would impact options: - - Taps (catalog, state, stream maps, etc.) - - Targets (data type handling, batching, SQL object generation, etc.) + - Taps (catalog, state, stream maps, tests, etc.) + - Targets (data type handling, batching, SQL object generation, tests, etc.) - Configuration (settings parsing, validation, etc.) - CLI (options, error messages, logging, etc.) - Cookiecutter templates From b49804265844d88eafafb80163d971cc1a7054eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:23:16 +0000 Subject: [PATCH 08/15] chore(deps): Bump pytest from 7.3.1 to 7.3.2 (#1766) --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index eaad584e1..7fab94625 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1690,13 +1690,13 @@ files = [ [[package]] name = "pytest" -version = "7.3.1" +version = "7.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, + {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, + {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, ] [package.dependencies] @@ -1709,7 +1709,7 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-durations" From d7a1e79f9d0c27f9356b5d5b0859b1a05d00e32e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:10:40 +0000 Subject: [PATCH 09/15] chore(deps): Bump actions/checkout from 3.5.2 to 3.5.3 (#1759) --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/cookiecutter-e2e.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 6 +++--- .github/workflows/version_bump.yml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0dbb8bc22..75b4dc276 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 766857ac0..7adca51d7 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Upgrade pip env: diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index dc7c94bbd..8048f9cad 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: GitHub dependency vulnerability check if: ${{ github.event_name == 'pull_request_target' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 35d1db076..f34a699f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Set up Python uses: actions/setup-python@v4.6.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed83c627f..150901c93 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Install Poetry env: @@ -91,7 +91,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Install Poetry env: @@ -133,7 +133,7 @@ jobs: needs: tests steps: - name: Check out the repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Install Poetry run: | diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml index 1889fcf22..47649408e 100644 --- a/.github/workflows/version_bump.yml +++ b/.github/workflows/version_bump.yml @@ -35,7 +35,7 @@ jobs: pull-requests: write # to create and update PRs steps: - - uses: actions/checkout@v3.5.2 + - uses: actions/checkout@v3.5.3 with: fetch-depth: 0 From e8772158aa7b4a0f87fb280d40bba68f8c0e72e3 Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Mon, 19 Jun 2023 10:49:12 -0400 Subject: [PATCH 10/15] fix: Add descriptions for `batch_config` properties (#1771) add descriptions for batch_config properties --- singer_sdk/helpers/capabilities.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/singer_sdk/helpers/capabilities.py b/singer_sdk/helpers/capabilities.py index 00a420daf..55aff8ce3 100644 --- a/singer_sdk/helpers/capabilities.py +++ b/singer_sdk/helpers/capabilities.py @@ -59,11 +59,17 @@ "encoding", description="Specifies the format and compression of the batch files.", wrapped=ObjectType( - Property("format", StringType, allowed_values=["jsonl"]), + Property( + "format", + StringType, + allowed_values=["jsonl"], + description="Format to use for batch files.", + ), Property( "compression", StringType, allowed_values=["gzip", "none"], + description="Compression format to use for batch files.", ), ), ), @@ -71,8 +77,16 @@ "storage", description="Defines the storage layer to use when writing batch files", wrapped=ObjectType( - Property("root", StringType), - Property("prefix", StringType), + Property( + "root", + StringType, + description="Root path to use when writing batch files.", + ), + Property( + "prefix", + StringType, + description="Prefix to use when writing batch files.", + ), ), ), ), From 7e8997f9fdd1185984db99cfaeab64b278b2252e Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Mon, 19 Jun 2023 16:21:50 -0400 Subject: [PATCH 11/15] fix: Add tests for SQL type conversion from JSON schemas (#1775) * add tests for sql type parsing * fix anyof sqltype parsing test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup precommit errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update imports in tests --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- singer_sdk/typing.py | 4 +++- tests/core/test_typing.py | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index f2766944c..1e70f6201 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -825,7 +825,9 @@ def _jsonschema_type_check(jsonschema_type: dict, type_check: tuple[str]) -> boo if jsonschema_type.get("type") in type_check: # noqa: PLR5501 return True - if any(t in type_check for t in jsonschema_type.get("anyOf", ())): + if any( + _jsonschema_type_check(t, type_check) for t in jsonschema_type.get("anyOf", ()) + ): return True return False diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 854669e5b..b2cf9c691 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -4,7 +4,9 @@ import datetime import logging -import typing as t + +import pytest +import sqlalchemy from singer_sdk.helpers._typing import ( TypeConformanceLevel, @@ -17,11 +19,9 @@ PropertiesList, Property, StringType, + to_sql_type, ) -if t.TYPE_CHECKING: - import pytest - logger = logging.getLogger("log") @@ -292,3 +292,29 @@ def test_conform_primitives(): assert _conform_primitive_property(None, {"type": "boolean"}) is None assert _conform_primitive_property(0, {"type": "boolean"}) is False assert _conform_primitive_property(1, {"type": "boolean"}) is True + + +@pytest.mark.parametrize( + "jsonschema_type,expected", + [ + ({"type": ["string", "null"]}, sqlalchemy.types.VARCHAR), + ({"type": ["integer", "null"]}, sqlalchemy.types.INTEGER), + ({"type": ["number", "null"]}, sqlalchemy.types.DECIMAL), + ({"type": ["boolean", "null"]}, sqlalchemy.types.BOOLEAN), + ({"type": "object", "properties": {}}, sqlalchemy.types.VARCHAR), + ({"type": "array"}, sqlalchemy.types.VARCHAR), + ({"format": "date", "type": ["string", "null"]}, sqlalchemy.types.DATE), + ({"format": "time", "type": ["string", "null"]}, sqlalchemy.types.TIME), + ( + {"format": "date-time", "type": ["string", "null"]}, + sqlalchemy.types.DATETIME, + ), + ( + {"anyOf": [{"type": "string", "format": "date-time"}, {"type": "null"}]}, + sqlalchemy.types.DATETIME, + ), + ({"anyOf": [{"type": "integer"}, {"type": "null"}]}, sqlalchemy.types.INTEGER), + ], +) +def test_to_sql_type(jsonschema_type, expected): + assert isinstance(to_sql_type(jsonschema_type), expected) From e2be08152ecd8709979571479646f2d262f8ac6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:48:02 -0600 Subject: [PATCH 12/15] chore: pre-commit autoupdate (#1760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: pre-commit autoupdate updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.270 → v0.0.272](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.270...v0.0.272) * Use sequences instead of sets to iterate --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar R. M Co-authored-by: Edgar Ramírez Mondragón --- .pre-commit-config.yaml | 2 +- singer_sdk/sinks/core.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b4bcc315..03fb5f67d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: - id: check-readthedocs - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.272 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index 0c6351779..fbcbbb295 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -254,17 +254,17 @@ def _add_sdc_metadata_to_schema(self) -> None: https://sdk.meltano.com/en/latest/implementation/record_metadata.html """ properties_dict = self.schema["properties"] - for col in { + for col in ( "_sdc_extracted_at", "_sdc_received_at", "_sdc_batched_at", "_sdc_deleted_at", - }: + ): properties_dict[col] = { "type": ["null", "string"], "format": "date-time", } - for col in {"_sdc_sequence", "_sdc_table_version"}: + for col in ("_sdc_sequence", "_sdc_table_version"): properties_dict[col] = {"type": ["null", "integer"]} def _remove_sdc_metadata_from_schema(self) -> None: @@ -274,14 +274,14 @@ def _remove_sdc_metadata_from_schema(self) -> None: https://sdk.meltano.com/en/latest/implementation/record_metadata.html """ properties_dict = self.schema["properties"] - for col in { + for col in ( "_sdc_extracted_at", "_sdc_received_at", "_sdc_batched_at", "_sdc_deleted_at", "_sdc_sequence", "_sdc_table_version", - }: + ): properties_dict.pop(col, None) def _remove_sdc_metadata_from_record(self, record: dict) -> None: From 56c794fd599c501e794ad964b95b2459f3b5dc52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:57:15 -0600 Subject: [PATCH 13/15] chore(deps-dev): Bump pyarrow from 12.0.0 to 12.0.1 (#1762) Bumps [pyarrow](https://github.com/apache/arrow) from 12.0.0 to 12.0.1. - [Commits](https://github.com/apache/arrow/compare/go/v12.0.0...go/v12.0.1) --- updated-dependencies: - dependency-name: pyarrow dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7fab94625..363130529 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1488,36 +1488,36 @@ wcwidth = "*" [[package]] name = "pyarrow" -version = "12.0.0" +version = "12.0.1" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.7" files = [ - {file = "pyarrow-12.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:3b97649c8a9a09e1d8dc76513054f1331bd9ece78ee39365e6bf6bc7503c1e94"}, - {file = "pyarrow-12.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc4ea634dacb03936f50fcf59574a8e727f90c17c24527e488d8ceb52ae284de"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d568acfca3faa565d663e53ee34173be8e23a95f78f2abfdad198010ec8f745"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b50bb9a82dca38a002d7cbd802a16b1af0f8c50ed2ec94a319f5f2afc047ee9"}, - {file = "pyarrow-12.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3d1733b1ea086b3c101427d0e57e2be3eb964686e83c2363862a887bb5c41fa8"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:a7cd32fe77f967fe08228bc100433273020e58dd6caced12627bcc0a7675a513"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:92fb031e6777847f5c9b01eaa5aa0c9033e853ee80117dce895f116d8b0c3ca3"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:280289ebfd4ac3570f6b776515baa01e4dcbf17122c401e4b7170a27c4be63fd"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:272f147d4f8387bec95f17bb58dcfc7bc7278bb93e01cb7b08a0e93a8921e18e"}, - {file = "pyarrow-12.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:0846ace49998825eda4722f8d7f83fa05601c832549c9087ea49d6d5397d8cec"}, - {file = "pyarrow-12.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:993287136369aca60005ee7d64130f9466489c4f7425f5c284315b0a5401ccd9"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7b6a765ee4f88efd7d8348d9a1f804487d60799d0428b6ddf3344eaef37282"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1c4fce253d5bdc8d62f11cfa3da5b0b34b562c04ce84abb8bd7447e63c2b327"}, - {file = "pyarrow-12.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e6be4d85707fc8e7a221c8ab86a40449ce62559ce25c94321df7c8500245888f"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ea830d9f66bfb82d30b5794642f83dd0e4a718846462d22328981e9eb149cba8"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7b5b9f60d9ef756db59bec8d90e4576b7df57861e6a3d6a8bf99538f68ca15b3"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99e559d27db36ad3a33868a475f03e3129430fc065accc839ef4daa12c6dab6"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b0810864a593b89877120972d1f7af1d1c9389876dbed92b962ed81492d3ffc"}, - {file = "pyarrow-12.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:23a77d97f4d101ddfe81b9c2ee03a177f0e590a7e68af15eafa06e8f3cf05976"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2cc63e746221cddb9001f7281dee95fd658085dd5b717b076950e1ccc607059c"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8c26912607e26c2991826bbaf3cf2b9c8c3e17566598c193b492f058b40d3a4"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d8b90efc290e99a81d06015f3a46601c259ecc81ffb6d8ce288c91bd1b868c9"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2466be046b81863be24db370dffd30a2e7894b4f9823fb60ef0a733c31ac6256"}, - {file = "pyarrow-12.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:0e36425b1c1cbf5447718b3f1751bf86c58f2b3ad299f996cd9b1aa040967656"}, - {file = "pyarrow-12.0.0.tar.gz", hash = "sha256:19c812d303610ab5d664b7b1de4051ae23565f9f94d04cbea9e50569746ae1ee"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, + {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, + {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, + {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, + {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, + {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, + {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, + {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, ] [package.dependencies] From 5a5f43a4fb43f55412464fa816cde70f4198607b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 23:14:25 +0000 Subject: [PATCH 14/15] chore(deps): Bump pre-commit from 3.3.2 to 3.3.3 in /.github/workflows (#1761) --- .github/workflows/constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt index 6596d2c0e..01217f3a3 100644 --- a/.github/workflows/constraints.txt +++ b/.github/workflows/constraints.txt @@ -1,5 +1,5 @@ pip==23.1.2 poetry==1.5.1 -pre-commit==3.3.2 +pre-commit==3.3.3 nox==2023.4.22 nox-poetry==1.0.2 From 3e7f5b3d121295de0bea22fc2ce9b56b882b7432 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 19 Jun 2023 17:59:09 -0600 Subject: [PATCH 15/15] fix: Force stream selection in tests (#1698) Co-authored-by: Ken Payne --- singer_sdk/streams/core.py | 11 +++++++++- singer_sdk/tap_base.py | 4 ++++ tests/samples/test_tap_countries.py | 22 +++++++++++++++++++ .../countries_write_schemas | 2 ++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/snapshots/countries_write_schemas/countries_write_schemas diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index 31e16dc8b..a414123dd 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -269,7 +269,6 @@ def get_starting_timestamp(self, context: dict | None) -> datetime.datetime | No return t.cast(datetime.datetime, pendulum.parse(value)) - @final @property def selected(self) -> bool: """Check if stream is selected. @@ -279,6 +278,16 @@ def selected(self) -> bool: """ return self.mask.get((), True) + @selected.setter + def selected(self, value: bool | None) -> None: + """Set stream selection. + + Args: + value: True if the stream is selected. + """ + self.metadata.root.selected = value + self._mask = self.metadata.resolve_selection() + @final @property def has_selected_descendents(self) -> bool: diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py index 23d8e815b..150f0ee81 100644 --- a/singer_sdk/tap_base.py +++ b/singer_sdk/tap_base.py @@ -246,6 +246,9 @@ def run_sync_dry_run( # Initialize streams' record limits before beginning the sync test. stream.ABORT_AT_RECORD_COUNT = dry_run_record_limit + # Force selection of streams. + stream.selected = True + for stream in streams: if stream.parent_stream_type: self.logger.debug( @@ -267,6 +270,7 @@ def run_sync_dry_run( def write_schemas(self) -> None: """Write a SCHEMA message for all known streams to STDOUT.""" for stream in self.streams.values(): + stream.selected = True stream._write_schema_message() # Stream detection: diff --git a/tests/samples/test_tap_countries.py b/tests/samples/test_tap_countries.py index f822f5c8b..e1910d0cb 100644 --- a/tests/samples/test_tap_countries.py +++ b/tests/samples/test_tap_countries.py @@ -9,6 +9,9 @@ import typing as t from contextlib import redirect_stdout +import pytest +from click.testing import CliRunner + from samples.sample_tap_countries.countries_tap import SampleTapCountries from singer_sdk.helpers._catalog import ( get_selected_schema, @@ -17,6 +20,11 @@ from singer_sdk.testing import get_tap_test_class from singer_sdk.testing.config import SuiteConfig +if t.TYPE_CHECKING: + from pathlib import Path + + from pytest_snapshot.plugin import Snapshot + SAMPLE_CONFIG = {} SAMPLE_CONFIG_BAD = {"not": "correct"} @@ -136,3 +144,17 @@ def tally_messages(messages: list) -> t.Counter: assert counter["BATCH", "countries"] == 1 assert counter[("STATE",)] == 3 + + +@pytest.mark.snapshot() +def test_write_schema( + snapshot: Snapshot, + snapshot_dir: Path, +): + snapshot.snapshot_dir = snapshot_dir.joinpath("countries_write_schemas") + + runner = CliRunner(mix_stderr=False) + result = runner.invoke(SampleTapCountries.cli, ["--test", "schema"]) + + snapshot_name = "countries_write_schemas" + snapshot.assert_match(result.stdout, snapshot_name) diff --git a/tests/snapshots/countries_write_schemas/countries_write_schemas b/tests/snapshots/countries_write_schemas/countries_write_schemas new file mode 100644 index 000000000..b0808ce23 --- /dev/null +++ b/tests/snapshots/countries_write_schemas/countries_write_schemas @@ -0,0 +1,2 @@ +{"type": "SCHEMA", "stream": "continents", "schema": {"properties": {"code": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}}, "type": "object"}, "key_properties": ["code"]} +{"type": "SCHEMA", "stream": "countries", "schema": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}, "native": {"type": ["string", "null"]}, "phone": {"type": ["string", "null"]}, "capital": {"type": ["string", "null"]}, "currency": {"type": ["string", "null"]}, "emoji": {"type": ["string", "null"]}, "continent": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": ["object", "null"]}, "languages": {"items": {"properties": {"code": {"type": ["string", "null"]}, "name": {"type": ["string", "null"]}}, "type": "object"}, "type": ["array", "null"]}}, "type": "object"}, "key_properties": ["code"]}