Merge branch 'main' into 1086-feat-built-in-handling-default-target-s…

…chedule
meltano · Nov 16, 2022 · 3e599fa · 3e599fa
2 parents 601c5e2 + 61ed7b4
commit 3e599fa
Show file tree

Hide file tree

Showing 18 changed files with 257 additions and 201 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.13.1"
+      placeholder: "0.14.0"
     validations:
       required: true
   - type: dropdown

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,5 +1,5 @@
 pip==22.3.1
 poetry==1.2.2
-virtualenv==20.16.6
+virtualenv==20.16.7
 nox==2022.8.7
 nox-poetry==1.0.2
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -57,7 +57,7 @@ repos:
     files: 'singer_sdk/.*'
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.2.0
+  rev: v3.2.2
   hooks:
   - id: pyupgrade
     args: [--py37-plus]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
+## v0.14.0 (2022-11-16)
+
+### ✨ New
+
+- [#1175](https://github.com/meltano/sdk/issues/1175) Add `datetime` functions to simpleeval env in stream maps -- _**Thanks @qbatten!**_
+
+### 🐛 Fixes
+
+- [#1182](https://github.com/meltano/sdk/issues/1182) Update `SQLConnector` import for SQL target cookiecutter -- _**Thanks @radbrt!**_
+- [#1168](https://github.com/meltano/sdk/issues/1168) `SQLConnector.table_exists()` to use separate `table_name` and `schema_name` instead of fully qualified name -- _**Thanks @BuzzCutNorman!**_
+- [#1164](https://github.com/meltano/sdk/issues/1164) Write a valid final state message at the end of each stream sync -- _**Thanks @laurentS!**_
+
+### ⚙️ Under the Hood
+
+- [#1114](https://github.com/meltano/sdk/issues/1114) Make DDL overridable for column `ADD`, `ALTER`, and `RENAME` operations
+
 ## v0.13.1 (2022-11-08)
 
 ### 🐛 Fixes

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -12,7 +12,7 @@ license = "Apache 2.0"
 [tool.poetry.dependencies]
 python = "<3.11,>=3.7.1"
 requests = "^2.25.1"
-singer-sdk = "^0.13.1"
+singer-sdk = "^0.14.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml
@@ -12,7 +12,7 @@ license = "Apache 2.0"
 [tool.poetry.dependencies]
 python = "<3.11,>=3.7.1"
 requests = "^2.25.1"
-singer-sdk = "^0.13.1"
+singer-sdk = "^0.14.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"

diff --git a/...ecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/sinks.py b/...ecutter/target-template/{{cookiecutter.target_id}}/{{cookiecutter.library_name}}/sinks.py
@@ -12,10 +12,13 @@
 
 {%- set sinkclass = sinkclass_mapping[cookiecutter.serialization_method] %}
 
-from singer_sdk.sinks import {% if sinkclass == "SQLSink" %}SQLConnector, {% endif %}{{ sinkclass }}
+from singer_sdk.sinks import {{ sinkclass }}
 
 {%- if sinkclass == "SQLSink" %}
+from singer_sdk.connectors import SQLConnector
+{% endif %}
 
+{%- if sinkclass == "SQLSink" %}
 
 class {{ cookiecutter.destination_name }}Connector(SQLConnector):
     """The connector for {{ cookiecutter.destination_name }}.

diff --git a/docs/conf.py b/docs/conf.py
@@ -25,7 +25,7 @@
 author = "Meltano Core Team and Contributors"
 
 # The full version, including alpha/beta/rc tags
-release = "0.13.1"
+release = "0.14.0"
 
 
 # -- General configuration ---------------------------------------------------

diff --git a/docs/stream_maps.md b/docs/stream_maps.md
@@ -160,6 +160,8 @@ can be referenced directly by mapping expressions.
     of the hash's hex digest.
   - This is defined by the SDK internally with native python:
     `hashlib.md5(<input>.encode("utf-8")).hexdigest()`.
+- `datetime` - This is the datetime module object from the Python standard library. You can access
+    datetime.datetime, datetime.timedelta, etc.
 
 #### Built-in Variable Names
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "singer-sdk"
-version = "0.13.1"
+version = "0.14.0"
 description = "A framework for building Singer taps"
 authors = ["Meltano Team and Contributors"]
 maintainers = ["Meltano Team and Contributors"]
@@ -89,18 +89,18 @@ debugging = [
 commitizen-version-bump = { git = "https://github.com/meltano/commitizen-version-bump.git", branch = "main" }
 pytest = "^7.2.0"
 xdoctest = "^1.1.0"
-mypy = "^0.990"
+mypy = "^0.991"
 cookiecutter = "^2.1.1"
 PyYAML = "^6.0"
 pyarrow = {version = "^10.0.0", optional = true, python = "<3.11"}
 freezegun = "^1.2.2"
-viztracer = {version = "^0.15.5", optional = true, python = "<3.11"}
+viztracer = {version = "^0.15.6", optional = true, python = "<3.11"}
 requests-mock = "^1.10.0"
 sqlalchemy2-stubs = {version = "^0.0.2a29", allow-prereleases = true}
 types-python-dateutil = "^2.8.19"
 types-pytz = "^2022.6.0.1"
 types-requests = "^2.28.11"
-types-simplejson = "^3.17.7"
+types-simplejson = "^3.18.0"
 types-PyYAML = "^6.0.12"
 coverage = {extras = ["toml"], version = "^6.5"}
 
@@ -129,7 +129,7 @@ markers = [
 
 [tool.commitizen]
 name = "cz_version_bump"
-version = "0.13.1"
+version = "0.14.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 version_files = [
     "docs/conf.py",

diff --git a/singer_sdk/_singerlib/messages.py b/singer_sdk/_singerlib/messages.py
@@ -79,6 +79,26 @@ class RecordMessage(Message):
     time_extracted: datetime | None = None
     """The time the record was extracted."""
 
+    def to_dict(self) -> dict[str, t.Any]:
+        """Return a dictionary representation of the message.
+
+        This overrides the default conversion logic, since it uses unnecessary
+        deep copying and is very slow.
+
+        Returns:
+            A dictionary with the defined message fields.
+        """
+        result: dict[str, t.Any] = {
+            "type": "RECORD",
+            "stream": self.stream,
+            "record": self.record,
+        }
+        if self.version is not None:
+            result["version"] = self.version
+        if self.time_extracted is not None:
+            result["time_extracted"] = self.time_extracted
+        return result
+
     def __post_init__(self) -> None:
         """Post-init processing.
 

diff --git a/singer_sdk/connectors/sql.py b/singer_sdk/connectors/sql.py
@@ -477,9 +477,11 @@ def table_exists(self, full_table_name: str) -> bool:
         Returns:
             True if table exists, False if not, None if unsure or undetectable.
         """
+        _, schema_name, table_name = self.parse_full_table_name(full_table_name)
+
         return cast(
             bool,
-            sqlalchemy.inspect(self._engine).has_table(full_table_name),
+            sqlalchemy.inspect(self._engine).has_table(table_name, schema_name),
         )
 
     def schema_exists(self, schema_name: str) -> bool:

diff --git a/singer_sdk/mapper.py b/singer_sdk/mapper.py
@@ -7,6 +7,7 @@
 
 import abc
 import copy
+import datetime
 import hashlib
 import logging
 from typing import Any, Callable
@@ -292,6 +293,7 @@ def functions(self) -> dict[str, Callable]:
         """
         funcs: dict[str, Any] = simpleeval.DEFAULT_FUNCTIONS.copy()
         funcs["md5"] = md5
+        funcs["datetime"] = datetime
         return funcs
 
     def _eval(

diff --git a/singer_sdk/tap_base.py b/singer_sdk/tap_base.py
@@ -380,6 +380,7 @@ def sync_all(self) -> None:
 
             stream.sync()
             stream.finalize_state_progress_markers()
+            stream._write_state_message()
 
         # this second loop is needed for all streams to print out their costs
         # including child streams which are otherwise skipped in the loop above

diff --git a/singer_sdk/target_base.py b/singer_sdk/target_base.py
@@ -43,7 +43,7 @@ class Target(PluginBase, SingerReader, metaclass=abc.ABCMeta):
     object for that record.
     """
 
-    _MAX_RECORD_AGE_IN_MINUTES: float = 30.0
+    _MAX_RECORD_AGE_IN_MINUTES: float = 5.0
 
     # Default class to use for creating new sink objects.
     # Required if `Target.get_sink_class()` is not defined.

diff --git a/tests/core/test_countries_sync.py b/tests/core/test_countries_sync.py
@@ -130,4 +130,4 @@ def tally_messages(messages: list) -> Counter:
     assert counter["SCHEMA", "countries"] == 1
     assert counter["BATCH", "countries"] == 1
 
-    assert counter[("STATE",)] == 2
+    assert counter[("STATE",)] == 4
diff --git a/tests/core/test_mapper.py b/tests/core/test_mapper.py
@@ -36,6 +36,7 @@ def sample_catalog_dict() -> dict:
         Property("name", StringType),
         Property("owner_email", StringType),
         Property("description", StringType),
+        Property("description", StringType),
     ).to_dict()
     foobars_schema = PropertiesList(
         Property("the", StringType),
@@ -70,21 +71,25 @@ def sample_stream():
                 "name": "tap-something",
                 "owner_email": "[email protected]",
                 "description": "Comment A",
+                "create_date": "2019-01-01",
             },
             {
                 "name": "my-tap-something",
                 "owner_email": "[email protected]",
                 "description": "Comment B",
+                "create_date": "2020-01-01",
             },
             {
                 "name": "target-something",
                 "owner_email": "[email protected]",
                 "description": "Comment C",
+                "create_date": "2021-01-01",
             },
             {
                 "name": "not-atap",
                 "owner_email": "[email protected]",
                 "description": "Comment D",
+                "create_date": "2022-01-01",
             },
         ],
         "foobars": [
@@ -107,6 +112,7 @@ def transform_stream_maps():
             "email_hash": "md5(config['hash_seed'] + owner_email)",
             "description": "'[masked]'",
             "description2": "str('[masked]')",
+            "create_year": "int(datetime.date.fromisoformat(create_date).year)",
             "int_test": "int('0')",
             "__else__": None,
         },
@@ -125,6 +131,7 @@ def transformed_result(stream_map_config):
                 ),
                 "description": "[masked]",
                 "description2": "[masked]",
+                "create_year": 2019,
                 "int_test": 0,
             },
             {
@@ -135,6 +142,7 @@ def transformed_result(stream_map_config):
                 ),
                 "description": "[masked]",
                 "description2": "[masked]",
+                "create_year": 2020,
                 "int_test": 0,
             },
             {
@@ -145,6 +153,7 @@ def transformed_result(stream_map_config):
                 ),
                 "description": "[masked]",
                 "description2": "[masked]",
+                "create_year": 2021,
                 "int_test": 0,
             },
             {
@@ -155,6 +164,7 @@ def transformed_result(stream_map_config):
                 ),
                 "description": "[masked]",
                 "description2": "[masked]",
+                "create_year": 2022,
                 "int_test": 0,
             },
         ],
@@ -174,6 +184,7 @@ def transformed_schemas():
             Property("email_hash", StringType),
             Property("description", StringType),
             Property("description2", StringType),
+            Property("create_year", IntegerType),
             Property("int_test", IntegerType),
         ).to_dict(),
         "foobars": PropertiesList(