From 29c6bf35a6cf4e82b9409ab9b63d8c7039c801e8 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Mon, 17 Jul 2023 16:18:22 -0600 Subject: [PATCH 1/4] chore: Add stale bot config (#1846) --- .github/stale.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/stale.yml diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 000000000..6263a6249 --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,7 @@ +# This config file extends the shared Meltano GitHub org stale bot config: +# https://github.com/meltano/.github/blob/main/.github/stale.yml + +_extends: .github + +# In most cases, this file should not be updated. +# Updates to the stale bot config should be shared by all Meltano GitHub repositories. From 041080881520a03f52c49e28e2cad589c8faaeb4 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Jul 2023 11:19:06 -0600 Subject: [PATCH 2/4] docs: Fix stale `pip_url` example that uses shell script workaround for editable installation (#1852) --- docs/cli_commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 97bf0df9a..d12d245f8 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -161,7 +161,7 @@ plugins: extractors: - name: my-tap namespace: my_tap - executable: ./my-tap.sh + executable: -e . capabilities: - state - catalog From 4c5331353b5a553b5083358d6df5085e455cc800 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 11:40:06 -0600 Subject: [PATCH 3/4] chore: pre-commit autoupdate (#1851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3cde0b46..3c8a18afe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,14 +36,14 @@ repos: )$ - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.2 + rev: 0.23.3 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.277 + rev: v0.0.278 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --show-fixes] diff --git a/pyproject.toml b/pyproject.toml index d4a73d704..bac2a0f7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,8 @@ pytest11 = { callable = "singer_sdk:testing.pytest_plugin", extras = ["testing"] [tool.ruff] exclude = [ "cookiecutter/*", + "singer_sdk/helpers/_simpleeval.py", + "tests/core/test_simpleeval.py", ] ignore = [ "ANN101", # Missing type annotation for `self` in method From e6553866950d1104a282c3eced76958c6b834bdc Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Jul 2023 17:40:10 -0600 Subject: [PATCH 4/4] fix(targets): Check against the unconformed key properties when validating record keys (#1853) --- singer_sdk/sinks/core.py | 7 +++++-- tests/conftest.py | 4 ++++ tests/core/test_target_base.py | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/singer_sdk/sinks/core.py b/singer_sdk/sinks/core.py index ec43c4060..9928aa6f2 100644 --- a/singer_sdk/sinks/core.py +++ b/singer_sdk/sinks/core.py @@ -215,6 +215,9 @@ def datetime_error_treatment(self) -> DatetimeErrorTreatmentEnum: def key_properties(self) -> list[str]: """Return key properties. + Override this method to return a list of key properties in a format that is + compatible with the target. + Returns: A list of stream key properties. """ @@ -331,10 +334,10 @@ def _singer_validate_message(self, record: dict) -> None: Raises: MissingKeyPropertiesError: If record is missing one or more key properties. """ - if not all(key_property in record for key_property in self.key_properties): + if any(key_property not in record for key_property in self._key_properties): msg = ( f"Record is missing one or more key_properties. \n" - f"Key Properties: {self.key_properties}, " + f"Key Properties: {self._key_properties}, " f"Record Keys: {list(record.keys())}" ) raise MissingKeyPropertiesError( diff --git a/tests/conftest.py b/tests/conftest.py index 142e76fe1..cb201c9a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,6 +86,10 @@ def process_batch(self, context: dict) -> None: self.target.records_written.extend(context["records"]) self.target.num_batches_processed += 1 + @property + def key_properties(self) -> list[str]: + return [key.upper() for key in super().key_properties] + class TargetMock(Target): """A mock Target class.""" diff --git a/tests/core/test_target_base.py b/tests/core/test_target_base.py index 778fab722..1fd6b9a93 100644 --- a/tests/core/test_target_base.py +++ b/tests/core/test_target_base.py @@ -2,6 +2,9 @@ import copy +import pytest + +from singer_sdk.exceptions import MissingKeyPropertiesError from tests.conftest import BatchSinkMock, TargetMock @@ -28,3 +31,25 @@ def test_get_sink(): key_properties=key_properties, ) assert sink_returned == sink + + +def test_validate_record(): + target = TargetMock() + sink = BatchSinkMock( + target=target, + stream_name="test", + schema={ + "properties": { + "id": {"type": ["integer"]}, + "name": {"type": ["string"]}, + }, + }, + key_properties=["id"], + ) + + # Test valid record + sink._singer_validate_message({"id": 1, "name": "test"}) + + # Test invalid record + with pytest.raises(MissingKeyPropertiesError): + sink._singer_validate_message({"name": "test"})