TDL-15168: Use unique_line_item_id for invoice updates' lines value instead of id

tap_stripe/__init__.py

@@ -705,6 +705,27 @@ def sync_sub_stream(sub_stream_name, parent_obj, updates=False):
             obj_ad_dict = sub_stream_obj.to_dict_recursive()
 
             if sub_stream_name == "invoice_line_items":
+                # we will get "unique_id" for default API versions older than "2019-12-03"
+                if updates and obj_ad_dict.get("unique_id"):
+                    # get unique_id
+                    unique_id = obj_ad_dict.get("unique_id")
+                    # if type is invoiceitem, update 'id' field with 'unique_id'
+                    if obj_ad_dict.get("type") == "invoiceitem":
+                        # get invoice_item id
+                        invoice_item_id = obj_ad_dict.get("id")
+                        obj_ad_dict["id"] = unique_id
+                        # update 'invoice_item' with 'id' if not present
+                        if not obj_ad_dict.get("invoice_item"):
+                            obj_ad_dict["invoice_item"] = invoice_item_id
+                    # if type is subscription, update 'id' field with 'unique_id'
+                    if obj_ad_dict.get("type") == "subscription":
+                        # get subscription id
+                        subscription_id = obj_ad_dict.get("id")
+                        obj_ad_dict["id"] = unique_id
+                        # update 'subscription' with 'id' if not present
+                        if not obj_ad_dict.get("subscription"):
+                            obj_ad_dict["subscription"] = subscription_id
+
                 # Synthetic addition of a key to the record we sync
                 obj_ad_dict["invoice"] = parent_obj.id
             elif sub_stream_name == "payout_transactions":

tests/test_all_fields.py

@@ -165,6 +165,28 @@
     'payment_intents': set()
 }
 
+# we have observed that the SDK object creation returns some new fields intermittently
+SCHEMA_MISSING_FIELDS = {
+    'customers': {
+        'test_clock'
+    },
+    'subscriptions': {
+        'test_clock',
+    },
+    'products':set(),
+    'invoice_items':{
+        'test_clock',
+    },
+    'payouts':set(),
+    'charges': set(),
+    'subscription_items': set(),
+    'plans': set(),
+    'invoice_line_items': set(),
+    'invoices': {
+        'test_clock',
+    }
+}
+
 KNOWN_FAILING_FIELDS = {
     'coupons': {
         'percent_off', # BUG_9720 | Decimal('67') != Decimal('66.6') (value is changing in duplicate records)
@@ -254,7 +276,10 @@
 # As for the `price` field added in the schema, the API doc doesn't mention any
 # `trial_period_days` in the field, hence skipping the assertion error for the same.
 KNOWN_NESTED_MISSING_FIELDS = {
-    'subscription_items': {'price': 'recurring.trial_period_days'}
+    'subscription_items': {'price': 'recurring.trial_period_days'},
+    'charges': {'payment_method_details': 'card.mandate'},
+    'payment_intents': {'charges': 'payment_method_details.card.mandate',
+                        'payment_method_options': 'card.mandate_options'}
 }
 
 class ALlFieldsTest(BaseTapTest):
@@ -456,7 +481,6 @@ def all_fields_test(self, streams_to_test):
                 adjusted_actual_keys = actual_records_keys.union(  # BUG_12478
                     KNOWN_MISSING_FIELDS.get(stream, set())
                 ).union(SCHEMA_MISSING_FIELDS.get(stream, set()))
-
                 if stream == 'invoice_items':
                     adjusted_actual_keys = adjusted_actual_keys.union({'subscription_item'})  # BUG_13666
 
@@ -548,8 +572,22 @@ def all_fields_test(self, streams_to_test):
                                         f"AssertionError({failure_1})")
 
                                     nested_key = KNOWN_NESTED_MISSING_FIELDS.get(stream, {})
-                                    if self.find_nested_key(nested_key, expected_field_value, field):
-                                        continue
+                                    # Check whether expected_field_value is list or not.
+                                    # If expected_field_value is list then loop through each item of list
+                                    if type(expected_field_value) == list:
+                                        is_fickle = True
+                                        for each_expected_field_value in expected_field_value:
+                                            if self.find_nested_key(nested_key, each_expected_field_value, field):
+                                                continue
+                                            else:
+                                                is_fickle = False
+                                                break
+
+                                        if is_fickle:
+                                            continue
+                                    else:
+                                        if self.find_nested_key(nested_key, expected_field_value, field):
+                                            continue
 
                                     if field in KNOWN_FAILING_FIELDS[stream] or field in FIELDS_TO_NOT_CHECK[stream]:
                                         continue # skip the following wokaround

tests/unittests/test_invoice_line_item_id.py

@@ -0,0 +1,283 @@
+import unittest
+from unittest import mock
+import tap_stripe
+
+# mock invoice line items
+class MockLines:
+    def __init__(self, data):
+        self.data = data
+
+    def to_dict_recursive(self):
+        return self.data
+
+# mock invoice
+class MockInvoice:
+    def __init__(self, lines):
+        self.lines = lines
+        self.id = "inv_testinvoice"
+
+# mock transform function
+def transform(*args, **kwargs):
+    # return the data with was passed for transformation in the argument
+    return args[0]
+
+@mock.patch("singer.Transformer.transform")
+@mock.patch("tap_stripe.Context.get_catalog_entry")
+@mock.patch("tap_stripe.Context.new_counts")
+@mock.patch("tap_stripe.Context.updated_counts")
+class InvoiceLineItemId(unittest.TestCase):
+    """
+        Test cases to verify the invoice line items 'id' is used as expected when syncing 'event updates'
+    """
+
+    def test_no_events_updates(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify no data should be changed when function is not called with 'event updates'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "ii_testinvoiceitem",
+                "object": "line_item",
+                "invoice_item": "ii_testinvoiceitem",
+                "subscription": "sub_testsubscription",
+                "type": "invoiceitem",
+                "unique_id": "il_testlineitem"
+            })
+        ]
+
+        # function call when 'updates=False'
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), False)
+
+        # expected data
+        expected_record = {
+            "id": "ii_testinvoiceitem",
+            "object": "line_item",
+            "invoice_item": "ii_testinvoiceitem",
+            "subscription": "sub_testsubscription",
+            "type": "invoiceitem",
+            "unique_id": "il_testlineitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for transform function
+        args, kwargs = mocked_transform.call_args
+        # verify the data is not changed as function was not called with updates
+        self.assertEqual(expected_record, args[0])
+
+    def test_no_unique_id(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify no data should be changed when invoice line item data does not contain 'unique_id'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "ii_testinvoiceitem",
+                "object": "line_item",
+                "invoice_item": "ii_testinvoiceitem",
+                "subscription": "sub_testsubscription",
+                "type": "invoiceitem"
+            })
+        ]
+
+        # function call
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), True)
+
+        # expected data
+        expected_record = {
+            "id": "ii_testinvoiceitem",
+            "object": "line_item",
+            "invoice_item": "ii_testinvoiceitem",
+            "subscription": "sub_testsubscription",
+            "type": "invoiceitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for transform function
+        args, kwargs = mocked_transform.call_args
+        # verify the data is not changed as not 'unique_id' is present
+        self.assertEqual(expected_record, args[0])
+
+    def test_no_updates_and_unique_id(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify no data should be changed when invoice line item data
+            does not contain 'unique_id' and function is not called with 'event updates'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "ii_testinvoiceitem",
+                "object": "line_item",
+                "invoice_item": "ii_testinvoiceitem",
+                "subscription": "sub_testsubscription",
+                "type": "invoiceitem"
+            })
+        ]
+
+        # function call with 'updates=False'
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), False)
+
+        # expected data
+        expected_record = {
+            "id": "ii_testinvoiceitem",
+            "object": "line_item",
+            "invoice_item": "ii_testinvoiceitem",
+            "subscription": "sub_testsubscription",
+            "type": "invoiceitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for tranform function
+        args, kwargs = mocked_transform.call_args
+        # verify the data is not changed as the function was not called with updates and not unique_id is present
+        self.assertEqual(expected_record, args[0])
+
+    def test_invoiceitem_with_invoice_item(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify 'unique_id' is used as 'id' value when invoice line item type is 'invoiceitem'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "ii_testinvoiceitem",
+                "object": "line_item",
+                "invoice_item": "ii_testinvoiceitem",
+                "subscription": "sub_testsubscription",
+                "type": "invoiceitem",
+                "unique_id": "il_testlineitem"
+            })
+        ]
+
+        # function call with updates
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), True)
+
+        # expected data
+        expected_record = {
+            "id": "il_testlineitem",
+            "object": "line_item",
+            "invoice_item": "ii_testinvoiceitem",
+            "subscription": "sub_testsubscription",
+            "type": "invoiceitem",
+            "unique_id": "il_testlineitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for transform function
+        args, kwargs = mocked_transform.call_args
+        # verify the unique_id's value is used as 'id'
+        self.assertEqual(expected_record, args[0])
+
+    def test_invoiceitem_without_invoice_item(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify 'unique_id' is used as 'id' and 'invoice_item' field
+            contains 'id' value when invoice line item type is 'invoiceitem'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "ii_testinvoiceitem",
+                "object": "line_item",
+                "invoice_item": None,
+                "subscription": "sub_testsubscription",
+                "type": "invoiceitem",
+                "unique_id": "il_testlineitem"
+            })
+        ]
+
+        # function call with updates
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), True)
+
+        # expected data
+        expected_record = {
+            "id": "il_testlineitem",
+            "object": "line_item",
+            "invoice_item": "ii_testinvoiceitem",
+            "subscription": "sub_testsubscription",
+            "type": "invoiceitem",
+            "unique_id": "il_testlineitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for transform function
+        args, kwargs = mocked_transform.call_args
+        # verify the unique_id's value is used as 'id' and id's value is used as 'invoice_item' value
+        self.assertEqual(expected_record, args[0])
+
+    def test_subscription_without_subscription(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify 'unique_id' is used as 'id' and 'subscription' field
+            contains the 'id' value when invoice line item type is 'subscription'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "sub_testsubscription",
+                "object": "line_item",
+                "subscription": None,
+                "type": "subscription",
+                "unique_id": "il_testlineitem",
+                "unique_line_item_id": "sli_testsublineitem"
+            })
+        ]
+
+        # function call with updates
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), True)
+
+        # expected data
+        expected_record = {
+            "id": "il_testlineitem",
+            "object": "line_item",
+            "subscription": "sub_testsubscription",
+            "type": "subscription",
+            "unique_id": "il_testlineitem",
+            "unique_line_item_id": "sli_testsublineitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for transform function
+        args, kwargs = mocked_transform.call_args
+        # verify the unique_id's value is used as 'id' and id's value is used as 'subscription' value
+        self.assertEqual(expected_record, args[0])
+
+    def test_subscription_with_subscription(self, mocked_new_counts, mocked_updated_counts, mocked_get_catalog_entry, mocked_transform):
+        """
+            Test case to verify 'unique_id' is used as 'id' and 'subscription'
+            field is not updated when invoice line item type is 'subscription'
+        """
+        # mock transform
+        mocked_transform.side_effect = transform
+        # create line items dummy data
+        lines = [
+            MockLines({
+                "id": "sli_1KJvqbDcBSxinnbLvE4qMiJV",
+                "object": "line_item",
+                "subscription": "sub_testsubscription",
+                "type": "subscription",
+                "unique_id": "il_testlineitem",
+                "unique_line_item_id": "sli_testsublineitem"
+            })
+        ]
+
+        # function call with updates
+        tap_stripe.sync_sub_stream("invoice_line_items", MockInvoice(lines), True)
+
+        # expected data
+        expected_record = {
+            "id": "il_testlineitem",
+            "object": "line_item",
+            "subscription": "sub_testsubscription",
+            "type": "subscription",
+            "unique_id": "il_testlineitem",
+            "unique_line_item_id": "sli_testsublineitem",
+            "invoice": "inv_testinvoice"
+        }
+        # get args for transform function
+        args, kwargs = mocked_transform.call_args
+        # verify the unique_id's value is used as 'id'
+        self.assertEqual(expected_record, args[0])