TDL-15168: Use unique_line_item_id for invoice updates' lines value instead of id

tap_stripe/__init__.py

@@ -705,6 +705,42 @@ def sync_sub_stream(sub_stream_name, parent_obj, updates=False):
             obj_ad_dict = sub_stream_obj.to_dict_recursive()
 
             if sub_stream_name == "invoice_line_items":
+                # we will get "unique_id" for default API versions older than "2019-12-03"
+                # ie. for API version older than "2019-12-03", the value in the field
+                # "unique_id" is moved to "id" field in the newer API version
+                # For example:
+                #   OLDER API VERSION
+                #     {
+                #         "id": "ii_testinvoiceitem",
+                #         "object": "line_item",
+                #         "invoice_item": "ii_testinvoiceitem",
+                #         "subscription": "sub_testsubscription",
+                #         "type": "invoiceitem",
+                #         "unique_id": "il_testlineitem"
+                #     }
+
+                #   NEWER API VERSION
+                #     {
+                #         "id": "il_testlineitem",
+                #         "object": "line_item",
+                #         "invoice_item": "ii_testinvoiceitem",
+                #         "subscription": "sub_testsubscription",
+                #         "type": "invoiceitem",
+                #     }
+                if updates and obj_ad_dict.get("unique_id"):
+                    # get "unique_id"
+                    object_unique_id = obj_ad_dict.get("unique_id")
+                    # get "id"
+                    object_id = obj_ad_dict.get("id")
+                    # update "id" field with "unique_id" value
+                    obj_ad_dict["id"] = object_unique_id
+                    # if type is invoiceitem, update 'invoice_item' field with 'id' if not present
+                    if obj_ad_dict.get("type") == "invoiceitem" and not obj_ad_dict.get("invoice_item"):
+                        obj_ad_dict["invoice_item"] = object_id
+                    # if type is subscription, update 'subscription' field with 'id' if not present
+                    elif obj_ad_dict.get("type") == "subscription" and not obj_ad_dict.get("subscription"):
+                        obj_ad_dict["subscription"] = object_id
+
                 # Synthetic addition of a key to the record we sync
                 obj_ad_dict["invoice"] = parent_obj.id
             elif sub_stream_name == "payout_transactions":

tests/test_all_fields.py

@@ -45,18 +45,28 @@
     },
     'subscriptions': {
         'test_clock',
+        'description',
+        'application'
+    },
+    'products':{
+        'default_price'
     },
-    'products':set(),
     'invoice_items':{
         'test_clock',
     },
     'payouts':set(),
-    'charges': set(),
+    'charges': {
+        'failure_balance_transaction'
+    },
     'subscription_items': set(),
     'plans': set(),
     'invoice_line_items': set(),
     'invoices': {
         'test_clock',
+        'application'
+    },
+    'payment_intents': {
+        'amount_details'
     }
 }
 
@@ -182,16 +192,28 @@
         'plan', # BUG_12478 | missing subfields
     },
     'payouts': set(),
-    'charges': set(),
+    'charges': {
+        # missing subfield ['card.mandate']
+        'payment_method_details'
+    },
     'subscription_items': {
         # BUG_12478 | missing subfields on plan ['statement_description', 'statement_descriptor', 'name']
         'plan',
+        # missing subfield ['recurring.trial_period_days']
+        'price'
     },
     'invoices': {
         'plans', # BUG_12478 | missing subfields
     },
     'plans': set(),
-    'payment_intents':set(),
+    'payment_intents':{
+        # missing subfield ['payment_method_details.card.mandate']
+        'charges',
+        # missing subfield ['card.mandate_options']
+        'payment_method_options',
+        # missing subfield ['payment_method']
+        'last_payment_error'
+    },
     'invoice_line_items': set()
     # 'invoice_line_items': { # TODO This is a test issue that prevents us from consistently passing
     #     'unique_line_item_id',
@@ -251,12 +273,6 @@
     },
 }
 
-# As for the `price` field added in the schema, the API doc doesn't mention any
-# `trial_period_days` in the field, hence skipping the assertion error for the same.
-KNOWN_NESTED_MISSING_FIELDS = {
-    'subscription_items': {'price': 'recurring.trial_period_days'}
-}
-
 class ALlFieldsTest(BaseTapTest):
     """Test tap sets a bookmark and respects it for the next sync of a stream"""
 
@@ -354,25 +370,6 @@ def test_run(self):
                 # run the test
                 self.all_fields_test(streams_to_test)
 
-    def find_nested_key(self, nested_key, actual_field_value, field):
-        '''
-        Find the nested key that is failing in the field and ignore the assertion error
-        gained from it, if any.
-        '''
-        for field_name, each_keys in nested_key.items():
-            # split the keys through `.`, for getting the nested keys
-            keys = each_keys.split('.')
-            temp_value = actual_field_value
-            if field == field_name:
-                for failing_key in keys:
-                    # if the failing key is not present in the actual key or not
-                    if not temp_value.get(failing_key, None):
-                        return False
-                    else:
-                        temp_value = temp_value.get(failing_key)
-                        if keys[-1] in temp_value:
-                            return True
-
     def all_fields_test(self, streams_to_test):
         """
         Verify that for each stream data is synced when all fields are selected.
@@ -547,10 +544,6 @@ def all_fields_test(self, streams_to_test):
                                     print(f"WARNING {base_err_msg} failed exact comparison.\n"
                                         f"AssertionError({failure_1})")
 
-                                    nested_key = KNOWN_NESTED_MISSING_FIELDS.get(stream, {})
-                                    if self.find_nested_key(nested_key, expected_field_value, field):
-                                        continue
-
                                     if field in KNOWN_FAILING_FIELDS[stream] or field in FIELDS_TO_NOT_CHECK[stream]:
                                         continue # skip the following wokaround