From 254735cfa0bfdf7529745e0b34140f8ea5495208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 3 Jan 2023 16:19:08 -0600 Subject: [PATCH] Rename `ConformanceLevel` to `TypeConformanceLevel` Also document the field. --- singer_sdk/helpers/_typing.py | 12 ++++++------ singer_sdk/streams/core.py | 25 ++++++++++++++++++++++--- tests/core/test_record_typing.py | 4 ++-- tests/core/test_typing.py | 22 +++++++++++----------- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/singer_sdk/helpers/_typing.py b/singer_sdk/helpers/_typing.py index d231063585..80b975652a 100644 --- a/singer_sdk/helpers/_typing.py +++ b/singer_sdk/helpers/_typing.py @@ -284,7 +284,7 @@ def _warn_unmapped_properties( ) -class ConformanceLevel(Enum): +class TypeConformanceLevel(Enum): """Used to configure how data is conformed to json compatible types. Before outputting data as JSON, it is conformed to types that are valid in json, @@ -316,7 +316,7 @@ def conform_record_data_types( # noqa: C901 stream_name: str, record: Dict[str, Any], schema: dict, - level: ConformanceLevel, + level: TypeConformanceLevel, logger: logging.Logger, ) -> Dict[str, Any]: """Translate values in record dictionary to singer-compatible data types. @@ -335,7 +335,7 @@ def conform_record_data_types( # noqa: C901 def _conform_record_data_types( input_object: Dict[str, Any], schema: dict, - level: ConformanceLevel, + level: TypeConformanceLevel, parent: Optional[str], ) -> Tuple[Dict[str, Any], List[str]]: # noqa: C901 """Translate values in record dictionary to singer-compatible data types. @@ -354,7 +354,7 @@ def _conform_record_data_types( output_object: Dict[str, Any] = {} unmapped_properties: List[str] = [] - if level == ConformanceLevel.NONE: + if level == TypeConformanceLevel.NONE: return input_object, unmapped_properties for property_name, elem in input_object.items(): @@ -367,7 +367,7 @@ def _conform_record_data_types( property_schema = schema["properties"][property_name] if isinstance(elem, list) and is_uniform_list(property_schema): - if level == ConformanceLevel.RECURSIVE: + if level == TypeConformanceLevel.RECURSIVE: item_schema = property_schema["items"] output = [] for item in elem: @@ -390,7 +390,7 @@ def _conform_record_data_types( and is_object_type(property_schema) and "properties" in property_schema ): - if level == ConformanceLevel.RECURSIVE: + if level == TypeConformanceLevel.RECURSIVE: ( output_object[property_name], sub_unmapped_properties, diff --git a/singer_sdk/streams/core.py b/singer_sdk/streams/core.py index a7414be91b..1a19633897 100644 --- a/singer_sdk/streams/core.py +++ b/singer_sdk/streams/core.py @@ -40,7 +40,7 @@ write_starting_replication_value, ) from singer_sdk.helpers._typing import ( - ConformanceLevel, + TypeConformanceLevel, conform_record_data_types, is_datetime_type, ) @@ -81,12 +81,31 @@ def lazy_chunked_generator( class Stream(metaclass=abc.ABCMeta): """Abstract base class for tap streams.""" - STATE_MSG_FREQUENCY = 10000 # Number of records between state messages + STATE_MSG_FREQUENCY = 10000 + """Number of records between state messages.""" + _MAX_RECORDS_LIMIT: int | None = None - CONFORMANCE_LEVEL = ConformanceLevel.RECURSIVE + + TYPE_CONFORMANCE_LEVEL = TypeConformanceLevel.RECURSIVE + """Type conformance level for this stream. + + Field types in the schema are used to convert record field values to the correct + type. + + Available options are: + + - ``TypeConformanceLevel.NONE``: No conformance is performed. + - ``TypeConformanceLevel.RECURSIVE``: Conformance is performed recursively through + all nested levels in the record. + - ``TypeConformanceLevel.ROOT``: Conformance is performed only on the root level. + """ # Used for nested stream relationships parent_stream_type: type[Stream] | None = None + """Parent stream type for this stream. If this stream is a child stream, this should + be set to the parent stream class. + """ + ignore_parent_replication_key: bool = False # Internal API cost aggregator diff --git a/tests/core/test_record_typing.py b/tests/core/test_record_typing.py index 4f972e7340..27d2ada9bd 100644 --- a/tests/core/test_record_typing.py +++ b/tests/core/test_record_typing.py @@ -10,7 +10,7 @@ import pytest from singer_sdk.helpers._typing import ( - ConformanceLevel, + TypeConformanceLevel, conform_record_data_types, get_datelike_property_type, to_json_compatible, @@ -67,7 +67,7 @@ def test_conform_record_data_types( with caplog.at_level(logging.INFO, logger=logger.name): actual = conform_record_data_types( - stream_name, record, schema, ConformanceLevel.RECURSIVE, logger + stream_name, record, schema, TypeConformanceLevel.RECURSIVE, logger ) if ignore_props_message: assert ignore_props_message in caplog.text diff --git a/tests/core/test_typing.py b/tests/core/test_typing.py index 99a306b290..2980c4cec9 100644 --- a/tests/core/test_typing.py +++ b/tests/core/test_typing.py @@ -6,7 +6,7 @@ import pytest from singer_sdk.helpers._typing import ( - ConformanceLevel, + TypeConformanceLevel, _conform_primitive_property, conform_record_data_types, ) @@ -38,7 +38,7 @@ def test_simple_schema_conforms_types(): } actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output @@ -55,7 +55,7 @@ def test_primitive_arrays_are_conformed(): expected_output = {"list": [True, False]} actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output @@ -80,7 +80,7 @@ def test_only_root_fields_are_conformed_for_root_level(): } actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.ROOT_ONLY, logger + "test_stream", record, schema, TypeConformanceLevel.ROOT_ONLY, logger ) assert actual_output == expected_output @@ -99,7 +99,7 @@ def test_no_fields_are_conformed_for_none_level(): } actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.NONE, logger + "test_stream", record, schema, TypeConformanceLevel.NONE, logger ) assert actual_output == record @@ -114,7 +114,7 @@ def test_object_arrays_are_conformed(): expected_output = {"list": [{"value": True}, {"value": False}]} actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output @@ -138,7 +138,7 @@ def test_mixed_arrays_are_conformed(): expected_output = {"list": [{"value": True}, False]} actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output @@ -153,7 +153,7 @@ def test_nested_objects_are_conformed(): expected_output = {"object": {"value": True}} actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output @@ -169,7 +169,7 @@ def test_simple_schema_removes_types(caplog: pytest.LogCaptureFixture): with caplog.at_level(logging.WARNING): actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output assert caplog.records[0].message == ( @@ -189,7 +189,7 @@ def test_nested_objects_remove_types(caplog: pytest.LogCaptureFixture): with caplog.at_level(logging.WARNING): actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output assert caplog.records[0].message == ( @@ -209,7 +209,7 @@ def test_object_arrays_remove_types(caplog: pytest.LogCaptureFixture): with caplog.at_level(logging.WARNING): actual_output = conform_record_data_types( - "test_stream", record, schema, ConformanceLevel.RECURSIVE, logger + "test_stream", record, schema, TypeConformanceLevel.RECURSIVE, logger ) assert actual_output == expected_output assert caplog.records[0].message == (