From 331c0bac1cb6a8c905341a74f5dad960f8a7a8d5 Mon Sep 17 00:00:00 2001
From: Andreas Motl <andreas.motl@crate.io>
Date: Thu, 26 Sep 2024 01:35:24 +0200
Subject: [PATCH] DynamoDB CDC: Make `MODIFY` operation also propagate deleted
 attributes

---
 CHANGES.md                      | 1 +
 pyproject.toml                  | 6 +++---
 tests/io/dynamodb/test_relay.py | 2 ++
 tests/io/test_processor.py      | 2 ++
 4 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 79382c76..c68228a9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -3,6 +3,7 @@
 
 ## Unreleased
 - MongoDB: Configure `MongoDBCrateDBConverter` after updating to commons-codec 0.0.18
+- DynamoDB CDC: Fix `MODIFY` operation to also propagate deleted attributes
 
 ## 2024/09/22 v0.0.25
 - Table Loader: Improved conditional handling of "transformation" parameter
diff --git a/pyproject.toml b/pyproject.toml
index d29cd26b..427e37ed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -143,7 +143,7 @@ docs = [
 ]
 dynamodb = [
   "boto3",
-  "commons-codec>=0.0.14",
+  "commons-codec>=0.0.19",
 ]
 full = [
   "cratedb-toolkit[cfr,cloud,datasets,io,service]",
@@ -163,11 +163,11 @@ io = [
 kinesis = [
   "aiobotocore<2.16",
   "async-kinesis<1.2",
-  "commons-codec>=0.0.14",
+  "commons-codec>=0.0.19",
   "lorrystream[carabas]>=0.0.6",
 ]
 mongodb = [
-  "commons-codec[mongodb,zyp]>=0.0.18",
+  "commons-codec[mongodb,zyp]>=0.0.19",
   "cratedb-toolkit[io]",
   "orjson<4,>=3.3.1",
   "pymongo<4.9,>=3.10.1",
diff --git a/tests/io/dynamodb/test_relay.py b/tests/io/dynamodb/test_relay.py
index ac30a60a..4b60a10a 100644
--- a/tests/io/dynamodb/test_relay.py
+++ b/tests/io/dynamodb/test_relay.py
@@ -55,6 +55,7 @@ def test_kinesis_earliest_dynamodb_cdc_insert_update(caplog, cratedb, dynamodb):
     assert cratedb.database.count_records(table_name) == 1
     results = cratedb.database.run_sql(f"SELECT * FROM {table_name}", records=True)  # noqa: S608
     assert results[0]["data"]["list_of_objects"] == [{"foo": "bar"}, {"baz": "qux"}]
+    assert "tombstone" not in results[0]["data"]
 
 
 def test_kinesis_latest_dynamodb_cdc_insert_update(caplog, cratedb, dynamodb):
@@ -104,3 +105,4 @@ def test_kinesis_latest_dynamodb_cdc_insert_update(caplog, cratedb, dynamodb):
     assert cratedb.database.count_records(table_name) == 1
     results = cratedb.database.run_sql(f"SELECT * FROM {table_name}", records=True)  # noqa: S608
     assert results[0]["data"]["list_of_objects"] == [{"foo": "bar"}, {"baz": "qux"}]
+    assert "tombstone" not in results[0]["data"]
diff --git a/tests/io/test_processor.py b/tests/io/test_processor.py
index 3439380b..e30d0e01 100644
--- a/tests/io/test_processor.py
+++ b/tests/io/test_processor.py
@@ -34,6 +34,8 @@
                     "test2": {"N": 2},
                 }
             },
+            # An item that should be removed by the subsequent MODIFY operation.
+            "tombstone": {"S": "foo"},
         },
         "SizeBytes": 156,
         "ApproximateCreationDateTimePrecision": "MICROSECOND",