Skip to content

Commit

Permalink
IOTData: Simplify delta calculation (getmoto#8423)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bblommers
bblommers authored Dec 22, 2024
1 parent ab5ab27 commit 1e4364d
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 48 deletions.
66 changes: 24 additions & 42 deletions moto/iotdata/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
import time
from typing import Any, Dict, List, Optional, Tuple

import jsondiff

from moto.core.base_backend import BackendDict, BaseBackend
from moto.core.common_models import BaseModel
from moto.core.utils import merge_dicts
Expand Down Expand Up @@ -77,51 +75,35 @@ def parse_payload(cls, desired: Any, reported: Any) -> Any: # type: ignore[misc
elif reported is None and desired:
delta = desired
elif desired and reported:
delta = jsondiff.diff(reported, desired, syntax="symmetric")
delta.pop(jsondiff.add, None)
delta.pop(jsondiff.delete, None)
delta.pop(jsondiff.replace, None)
cls._resolve_nested_deltas(desired, reported, delta)
delta = cls._compute_delta_dict(desired, reported)
else:
delta = None
return delta

@classmethod
def _resolve_nested_deltas(cls, desired: Any, reported: Any, delta: Any) -> None: # type: ignore[misc]
for key, value in delta.items():
if isinstance(value, dict):
# delta = {insert: [(0, val1),, ], delete: [(0, val2),..] }
if isinstance(reported.get(key), list):
# Delta is a dict, but were supposed to have a list
# Explicitly insert/delete from the original list
list_delta = reported[key].copy()
if jsondiff.delete in value:
for idx, _ in sorted(value[jsondiff.delete], reverse=True):
del list_delta[idx]
if jsondiff.insert in value:
for new_val in sorted(value[jsondiff.insert], reverse=True):
list_delta.insert(*new_val)
delta[key] = list_delta
if isinstance(reported.get(key), dict):
# Delta is a dict, exactly what we're expecting
# Just delete any unknown symbols
value.pop(jsondiff.add, None)
value.pop(jsondiff.delete, None)
value.pop(jsondiff.replace, None)
elif isinstance(value, list):
# delta = [v1, v2]
# If the actual value is a string/bool/int and our delta is a list,
# that means that the delta reports both values - reported and desired
# But we only want to show the desired value here
# (Note that bool is a type of int, so we don't have to explicitly mention it)
if isinstance(desired.get(key), (str, int, float)):
delta[key] = desired[key]

# Resolve nested deltas
if isinstance(delta[key], dict):
cls._resolve_nested_deltas(
desired.get(key), reported.get(key), delta[key]
)
def _compute_delta_dict(cls, desired: Any, reported: Any) -> Dict[str, Any]: # type: ignore[misc]
delta = {}
for key, value in desired.items():
delta_value = cls._compute_delta(reported.get(key), value)

if delta_value is not None:
delta[key] = delta_value
return delta

@classmethod
def _compute_delta(cls, reported_value: Any, desired_value: Any) -> Any: # type: ignore[misc]
if reported_value == desired_value:
return None

if isinstance(desired_value, dict) and isinstance(reported_value, dict):
return cls._compute_delta_dict(desired_value, reported_value)

# Types are different, or
# Both types are intrinsic values (str, int, etc), or
# Both types are lists:
#
# Just return the desired value
return desired_value

def _create_metadata_from_state(self, state: Any, ts: Any) -> Any:
"""
Expand Down
7 changes: 1 addition & 6 deletions setup.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ all =
jsonschema
openapi-spec-validator>=0.5.0
pyparsing>=3.0.7
jsondiff>=1.1.2
py-partiql-parser==0.5.6
aws-xray-sdk!=0.96,>=0.93
setuptools
Expand All @@ -70,7 +69,6 @@ proxy =
cfn-lint>=0.40.0
openapi-spec-validator>=0.5.0
pyparsing>=3.0.7
jsondiff>=1.1.2
py-partiql-parser==0.5.6
aws-xray-sdk!=0.96,>=0.93
setuptools
Expand All @@ -85,7 +83,6 @@ server =
cfn-lint>=0.40.0
openapi-spec-validator>=0.5.0
pyparsing>=3.0.7
jsondiff>=1.1.2
py-partiql-parser==0.5.6
aws-xray-sdk!=0.96,>=0.93
setuptools
Expand Down Expand Up @@ -120,7 +117,6 @@ cloudformation =
cfn-lint>=0.40.0
openapi-spec-validator>=0.5.0
pyparsing>=3.0.7
jsondiff>=1.1.2
py-partiql-parser==0.5.6
aws-xray-sdk!=0.96,>=0.93
setuptools
Expand Down Expand Up @@ -173,7 +169,7 @@ guardduty =
iam =
inspector2 =
iot =
iotdata = jsondiff>=1.1.2
iotdata =
ivs =
kinesis =
kinesisvideo =
Expand Down Expand Up @@ -209,7 +205,6 @@ resourcegroupstaggingapi =
cfn-lint>=0.40.0
openapi-spec-validator>=0.5.0
pyparsing>=3.0.7
jsondiff>=1.1.2
py-partiql-parser==0.5.6
route53 =
route53resolver =
Expand Down
41 changes: 41 additions & 0 deletions tests/test_iotdata/test_iotdata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None:
@pytest.mark.parametrize(
"desired,initial_delta,reported,delta_after_report",
[
# Boolean flip
(
{"desired": {"online": True}},
{"desired": {"online": True}, "delta": {"online": True}},
Expand All @@ -215,14 +216,34 @@ def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None:
"reported": {"online": False, "enabled": True},
},
),
# No data
({}, {}, {"reported": {"online": False}}, {"reported": {"online": False}}),
# Missing data
({}, {}, {"reported": {"online": None}}, {}),
(
{"desired": {}},
{},
{"reported": {"online": False}},
{"reported": {"online": False}},
),
# Missing key
(
{"desired": {"enabled": True}},
{"desired": {"enabled": True}, "delta": {"enabled": True}},
{"reported": {}},
{"desired": {"enabled": True}, "delta": {"enabled": True}},
),
# Changed key
(
{"desired": {"enabled": True}},
{"desired": {"enabled": True}, "delta": {"enabled": True}},
{"reported": {"online": True}},
{
"desired": {"enabled": True},
"reported": {"online": True},
"delta": {"enabled": True},
},
),
# Remove from list
(
{"reported": {"list": ["value_1", "value_2"]}},
Expand Down Expand Up @@ -278,6 +299,26 @@ def test_delete_field_from_device_shadow(name: Optional[str] = None) -> None:
"reported": {"a": {"b": {"c": "d", "e": "f"}}},
},
),
(
{"reported": {"a1": {"b1": {"c": "d"}}}},
{"reported": {"a1": {"b1": {"c": "d"}}}},
{"desired": {"a1": {"b1": {"c": "d"}}, "a2": {"b2": "sth"}}},
{
"delta": {"a2": {"b2": "sth"}},
"desired": {"a1": {"b1": {"c": "d"}}, "a2": {"b2": "sth"}},
"reported": {"a1": {"b1": {"c": "d"}}},
},
),
(
{"reported": {"a": {"b1": {"c": "d"}}}},
{"reported": {"a": {"b1": {"c": "d"}}}},
{"desired": {"a": {"b1": {"c": "d"}, "b2": "sth"}}},
{
"delta": {"a": {"b2": "sth"}},
"desired": {"a": {"b1": {"c": "d"}, "b2": "sth"}},
"reported": {"a": {"b1": {"c": "d"}}},
},
),
],
)
def test_delta_calculation(
Expand Down

0 comments on commit 1e4364d

Please sign in to comment.