From 0b6bd68d87deca520fc6a24121f99e1616b7a8d3 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Fri, 24 Feb 2023 05:31:12 -0500 Subject: [PATCH 1/2] Allow string dates to stay strings --- src/fides/api/ops/util/cache.py | 11 ++++------- tests/ops/util/test_cache.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/fides/api/ops/util/cache.py b/src/fides/api/ops/util/cache.py index 63be1bdafb..530ce6eaac 100644 --- a/src/fides/api/ops/util/cache.py +++ b/src/fides/api/ops/util/cache.py @@ -21,6 +21,7 @@ _connection = None ENCODED_BYTES_PREFIX = "quote_encoded_" +ENCODED_DATE_PREFIX = "date_encoded_" ENCODED_MONGO_OBJECT_ID_PREFIX = "encoded_object_id_" @@ -31,7 +32,7 @@ def default(self, o: Any) -> Any: # pylint: disable=too-many-return-statements if isinstance(o, bytes): return f"{ENCODED_BYTES_PREFIX}{quote(o)}" if isinstance(o, (datetime, date)): - return o.isoformat() + return f"{ENCODED_DATE_PREFIX}{o.isoformat()}" if isinstance(o, ObjectId): return f"{ENCODED_MONGO_OBJECT_ID_PREFIX}{str(o)}" if isinstance(o, object): @@ -47,17 +48,13 @@ def default(self, o: Any) -> Any: # pylint: disable=too-many-return-statements def _custom_decoder(json_dict: Dict[str, Any]) -> Dict[str, Any]: for k, v in json_dict.items(): - try: - json_dict[k] = datetime.fromisoformat(v) - continue - except (TypeError, ValueError): - pass - if isinstance(v, str): # The mongodb objectids couldn't be directly json encoded so they are converted # to strings and prefixed with encoded_object_id in order to find during decodeint. if v.startswith(ENCODED_MONGO_OBJECT_ID_PREFIX): json_dict[k] = ObjectId(v[18:]) + if v.startswith(ENCODED_DATE_PREFIX): + json_dict[k] = datetime.fromisoformat(v[13:]) # The bytes from secrets couldn't be directly json encoded so it is url # encode and prefixed with quite_encoded in order to find during decodeint. elif v.startswith(ENCODED_BYTES_PREFIX): diff --git a/tests/ops/util/test_cache.py b/tests/ops/util/test_cache.py index 33c0c8319e..99f2a4ad14 100644 --- a/tests/ops/util/test_cache.py +++ b/tests/ops/util/test_cache.py @@ -10,6 +10,7 @@ from fides.api.ops.util.cache import ( ENCODED_BYTES_PREFIX, + ENCODED_DATE_PREFIX, ENCODED_MONGO_OBJECT_ID_PREFIX, FidesopsRedis, ) @@ -121,7 +122,15 @@ def __init__(self): (b"some value", f'"{ENCODED_BYTES_PREFIX}some%20value"'), ( datetime(2023, 2, 14, 20, 58), - f'"{datetime(2023, 2, 14, 20, 58).isoformat()}"', + f'"{ENCODED_DATE_PREFIX}{datetime(2023, 2, 14, 20, 58).isoformat()}"', + ), + ( + {"a": datetime(2023, 2, 14, 20, 58)}, + f'{{"a": "{ENCODED_DATE_PREFIX}{datetime(2023, 2, 14, 20, 58).isoformat()}"}}', + ), + ( + {"a": {"b": datetime(2023, 2, 14, 20, 58)}}, + f'{{"a": {{"b": "{ENCODED_DATE_PREFIX}{datetime(2023, 2, 14, 20, 58).isoformat()}"}}}}', ), ({"a": "b"}, '{"a": "b"}'), ({"a": {"b": "c"}}, '{"a": {"b": "c"}}'), @@ -152,16 +161,28 @@ class TestCustomDecoder: [ (f'{{"a": "{ENCODED_BYTES_PREFIX}some%20value"}}', {"a": b"some value"}), ( - f'{{"a": "{datetime(2023, 2, 17, 14, 5).isoformat()}"}}', + f'{{"a": "{ENCODED_MONGO_OBJECT_ID_PREFIX}507f191e810c19729de860ea"}}', + {"a": ObjectId("507f191e810c19729de860ea")}, + ), + ( + f'{{"a": "{ENCODED_DATE_PREFIX}{datetime(2023, 2, 17, 14, 5).isoformat()}"}}', {"a": datetime(2023, 2, 17, 14, 5)}, ), ( - f'{{"a": "{ENCODED_MONGO_OBJECT_ID_PREFIX}507f191e810c19729de860ea"}}', - {"a": ObjectId("507f191e810c19729de860ea")}, + f'{{"a": {{"b": {{"c": "{ENCODED_DATE_PREFIX}{datetime(2023, 2, 17, 14, 5).isoformat()}"}}}}}}', + {"a": {"b": {"c": datetime(2023, 2, 17, 14, 5)}}}, + ), + ( + '{"birthday": "2001-11-08"}', + {"birthday": "2001-11-08"}, + ), + ( + '{"a": {"b": {"birthday": "2001-11-08"}}}', + {"a": {"b": {"birthday": "2001-11-08"}}}, ), ], ) - def test_decode_bytes(self, value, expected): + def test_cache_decode(self, value, expected): cache = FidesopsRedis() assert cache.decode_obj(value) == expected From 850c742dfc8d6e1f5ccf824bb99f0dbcbf2c06c7 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Fri, 24 Feb 2023 07:19:13 -0500 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04aabd9631..7460607edd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The types of changes are: * Fix support for "redis.user" setting when authenticating to the Redis cache [#2666](https://github.com/ethyca/fides/pull/2666) * Fix error with the classify dataset feature flag not writing the dataset to the server [#2675](https://github.com/ethyca/fides/pull/2675) +* Allow string dates to stay strings in cache decoding [#2695](https://github.com/ethyca/fides/pull/2695) * Admin UI * Remove Identifiability (Data Qualifier) from taxonomy editor [2684](https://github.com/ethyca/fides/pull/2684)