From 78393d80d4e1b896a3c45f124ad8d29b0ab4b97f Mon Sep 17 00:00:00 2001 From: Ivo Reumkens Date: Fri, 11 Nov 2022 17:06:19 +0100 Subject: [PATCH] Add timestamp as option to DateTime Fields (#2022) * Add timestamp as option to DateTime Fields * Load dates from timestamp timezone independant * Updated versionchanged info * Invert datetime aware if check * Update changelog Co-authored-by: Steven Loria --- AUTHORS.rst | 1 + CHANGELOG.rst | 10 ++++++++ src/marshmallow/fields.py | 27 +++++++++++++-------- src/marshmallow/utils.py | 28 ++++++++++++++++++++++ tests/test_deserialization.py | 44 +++++++++++++++++++++++++++++++++++ tests/test_serialization.py | 32 +++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 10 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index f4aec1678..a0a928a9a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -170,3 +170,4 @@ Contributors (chronological) - Isira Seneviratne `@Isira-Seneviratne `_ - Karthikeyan Singaravelan `@tirkarthi `_ - Marco Satti `@marcosatti `_ +- Ivo Reumkens `@vanHoi `_ diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ec2a084b1..8e9188188 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,16 @@ Changelog --------- +3.19.0 (unreleased) +******************* + +Features: + +- Add ``timestamp`` and ``timestamp_ms`` formats to `fields.DateTime` + (:issue:`612`). + Thanks :user:`vgavro` for the suggestion and thanks :user:`vanHoi` for + the PR. + 3.18.0 (2022-09-15) ******************* diff --git a/src/marshmallow/fields.py b/src/marshmallow/fields.py index f153b5f9f..b53562a3c 100644 --- a/src/marshmallow/fields.py +++ b/src/marshmallow/fields.py @@ -1218,11 +1218,14 @@ class DateTime(Field): Example: ``'2014-12-22T03:12:58.019077+00:00'`` :param format: Either ``"rfc"`` (for RFC822), ``"iso"`` (for ISO8601), - or a date format string. If `None`, defaults to "iso". + ``"timestamp"``, ``"timestamp_ms"`` (for a POSIX timestamp) or a date format string. + If `None`, defaults to "iso". :param kwargs: The same keyword arguments that :class:`Field` receives. .. versionchanged:: 3.0.0rc9 Does not modify timezone information on (de)serialization. + .. versionchanged:: 3.19 + Add timestamp as a format. """ SERIALIZATION_FUNCS = { @@ -1230,13 +1233,17 @@ class DateTime(Field): "iso8601": utils.isoformat, "rfc": utils.rfcformat, "rfc822": utils.rfcformat, - } # type: typing.Dict[str, typing.Callable[[typing.Any], str]] + "timestamp": utils.timestamp, + "timestamp_ms": utils.timestamp_ms, + } # type: typing.Dict[str, typing.Callable[[typing.Any], str | float]] DESERIALIZATION_FUNCS = { "iso": utils.from_iso_datetime, "iso8601": utils.from_iso_datetime, "rfc": utils.from_rfc, "rfc822": utils.from_rfc, + "timestamp": utils.from_timestamp, + "timestamp_ms": utils.from_timestamp_ms, } # type: typing.Dict[str, typing.Callable[[str], typing.Any]] DEFAULT_FORMAT = "iso" @@ -1252,7 +1259,7 @@ class DateTime(Field): "format": '"{input}" cannot be formatted as a {obj_type}.', } - def __init__(self, format: str | None = None, **kwargs): + def __init__(self, format: str | None = None, **kwargs) -> None: super().__init__(**kwargs) # Allow this to be None. It may be set later in the ``_serialize`` # or ``_deserialize`` methods. This allows a Schema to dynamically set the @@ -1267,7 +1274,7 @@ def _bind_to_schema(self, field_name, schema): or self.DEFAULT_FORMAT ) - def _serialize(self, value, attr, obj, **kwargs): + def _serialize(self, value, attr, obj, **kwargs) -> str | float | None: if value is None: return None data_format = self.format or self.DEFAULT_FORMAT @@ -1277,7 +1284,7 @@ def _serialize(self, value, attr, obj, **kwargs): else: return value.strftime(data_format) - def _deserialize(self, value, attr, data, **kwargs): + def _deserialize(self, value, attr, data, **kwargs) -> dt.datetime: if not value: # Falsy values, e.g. '', None, [] are not valid raise self.make_error("invalid", input=value, obj_type=self.OBJ_TYPE) data_format = self.format or self.DEFAULT_FORMAT @@ -1298,7 +1305,7 @@ def _deserialize(self, value, attr, data, **kwargs): ) from error @staticmethod - def _make_object_from_format(value, data_format): + def _make_object_from_format(value, data_format) -> dt.datetime: return dt.datetime.strptime(value, data_format) @@ -1323,11 +1330,11 @@ def __init__( *, timezone: dt.timezone | None = None, **kwargs, - ): + ) -> None: super().__init__(format=format, **kwargs) self.timezone = timezone - def _deserialize(self, value, attr, data, **kwargs): + def _deserialize(self, value, attr, data, **kwargs) -> dt.datetime: ret = super()._deserialize(value, attr, data, **kwargs) if is_aware(ret): if self.timezone is None: @@ -1360,11 +1367,11 @@ def __init__( *, default_timezone: dt.tzinfo | None = None, **kwargs, - ): + ) -> None: super().__init__(format=format, **kwargs) self.default_timezone = default_timezone - def _deserialize(self, value, attr, data, **kwargs): + def _deserialize(self, value, attr, data, **kwargs) -> dt.datetime: ret = super()._deserialize(value, attr, data, **kwargs) if not is_aware(ret): if self.default_timezone is None: diff --git a/src/marshmallow/utils.py b/src/marshmallow/utils.py index fd6513352..7a16ff6d1 100644 --- a/src/marshmallow/utils.py +++ b/src/marshmallow/utils.py @@ -190,6 +190,34 @@ def from_iso_date(value): return dt.date(**kw) +def from_timestamp(value: typing.Any) -> dt.datetime: + value = float(value) + if value < 0: + raise ValueError("Not a valid POSIX timestamp") + + # Load a timestamp with utc as timezone to prevent using system timezone. + # Then set timezone to None, to let the Field handle adding timezone info. + return dt.datetime.fromtimestamp(value, tz=dt.timezone.utc).replace(tzinfo=None) + + +def from_timestamp_ms(value: typing.Any) -> dt.datetime: + value = float(value) + return from_timestamp(value / 1000) + + +def timestamp( + value: dt.datetime, +) -> float: + if not is_aware(value): + # When a date is naive, use UTC as zone info to prevent using system timezone. + value = value.replace(tzinfo=dt.timezone.utc) + return value.timestamp() + + +def timestamp_ms(value: dt.datetime) -> float: + return timestamp(value) * 1000 + + def isoformat(datetime: dt.datetime) -> str: """Return the ISO8601-formatted representation of a datetime object. diff --git a/tests/test_deserialization.py b/tests/test_deserialization.py index c80a2581f..32aa09912 100644 --- a/tests/test_deserialization.py +++ b/tests/test_deserialization.py @@ -526,6 +526,50 @@ def test_iso_datetime_field_deserialization(self, fmt, value, expected, aware): else: assert field.deserialize(value) == expected + @pytest.mark.parametrize( + ("fmt", "value", "expected"), + [ + ("timestamp", 1384043025, dt.datetime(2013, 11, 10, 0, 23, 45)), + ("timestamp", "1384043025", dt.datetime(2013, 11, 10, 0, 23, 45)), + ("timestamp", 1384043025, dt.datetime(2013, 11, 10, 0, 23, 45)), + ("timestamp", 1384043025.12, dt.datetime(2013, 11, 10, 0, 23, 45, 120000)), + ( + "timestamp", + 1384043025.123456, + dt.datetime(2013, 11, 10, 0, 23, 45, 123456), + ), + ("timestamp", 1, dt.datetime(1970, 1, 1, 0, 0, 1)), + ("timestamp_ms", 1384043025000, dt.datetime(2013, 11, 10, 0, 23, 45)), + ("timestamp_ms", 1000, dt.datetime(1970, 1, 1, 0, 0, 1)), + ], + ) + def test_timestamp_field_deserialization(self, fmt, value, expected): + field = fields.DateTime(format=fmt) + assert field.deserialize(value) == expected + + # By default, a datetime from a timestamp is never aware. + field = fields.NaiveDateTime(format=fmt) + assert field.deserialize(value) == expected + + field = fields.AwareDateTime(format=fmt) + with pytest.raises(ValidationError, match="Not a valid aware datetime."): + field.deserialize(value) + + # But it can be added by providing a default. + field = fields.AwareDateTime(format=fmt, default_timezone=central) + expected_aware = expected.replace(tzinfo=central) + assert field.deserialize(value) == expected_aware + + @pytest.mark.parametrize("fmt", ["timestamp", "timestamp_ms"]) + @pytest.mark.parametrize( + "in_value", + ["", "!@#", 0, -1, dt.datetime(2013, 11, 10, 1, 23, 45)], + ) + def test_invalid_timestamp_field_deserialization(self, fmt, in_value): + field = fields.DateTime(format="timestamp") + with pytest.raises(ValidationError, match="Not a valid datetime."): + field.deserialize(in_value) + @pytest.mark.parametrize( ("fmt", "timezone", "value", "expected"), [ diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 51671ddf6..56f0773ef 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -579,6 +579,38 @@ def test_datetime_field_rfc822(self, fmt, value, expected): field = fields.DateTime(format=fmt) assert field.serialize("d", {"d": value}) == expected + @pytest.mark.parametrize( + ("fmt", "value", "expected"), + [ + ("timestamp", dt.datetime(1970, 1, 1), 0), + ("timestamp", dt.datetime(2013, 11, 10, 0, 23, 45), 1384043025), + ( + "timestamp", + dt.datetime(2013, 11, 10, 0, 23, 45, tzinfo=dt.timezone.utc), + 1384043025, + ), + ( + "timestamp", + central.localize(dt.datetime(2013, 11, 10, 0, 23, 45), is_dst=False), + 1384064625, + ), + ("timestamp_ms", dt.datetime(2013, 11, 10, 0, 23, 45), 1384043025000), + ( + "timestamp_ms", + dt.datetime(2013, 11, 10, 0, 23, 45, tzinfo=dt.timezone.utc), + 1384043025000, + ), + ( + "timestamp_ms", + central.localize(dt.datetime(2013, 11, 10, 0, 23, 45), is_dst=False), + 1384064625000, + ), + ], + ) + def test_datetime_field_timestamp(self, fmt, value, expected): + field = fields.DateTime(format=fmt) + assert field.serialize("d", {"d": value}) == expected + @pytest.mark.parametrize("fmt", ["iso", "iso8601", None]) @pytest.mark.parametrize( ("value", "expected"),