TDL-14354 log request_id

tap_stripe/__init__.py

@@ -8,6 +8,7 @@
 import stripe
 import stripe.error
 from stripe.stripe_object import StripeObject
+from stripe.api_requestor import APIRequestor
 from stripe.util import convert_to_stripe_object
 import singer
 from singer import utils, Transformer, metrics
@@ -392,6 +393,18 @@ def reduce_foreign_keys(rec, stream_name):
                     rec['lines'][k] = [li.to_dict_recursive() for li in val]
     return rec
 
+def new_request(self, method, url, params=None, headers=None):
+    '''The new request function to overwrite the request() function of the APIRequestor class of SDK.'''
+    rbody, rcode, rheaders, my_api_key = self.request_raw(
+        method.lower(), url, params, headers, is_streaming=False
+    )
+    resp = self.interpret_response(rbody, rcode, rheaders)
+    LOGGER.debug(f'request id : {resp.request_id}')
+    return resp, my_api_key
+
+# To log the request_id, we replaced the request() function of the APIRequestor
+# class o SDK, captured the response and logged the request_id
+APIRequestor.request = new_request
 
 def paginate(sdk_obj, filter_key, start_date, end_date, stream_name, request_args=None, limit=100):
     yield from sdk_obj.list(

tests/test_all_fields.py

@@ -45,18 +45,28 @@
     },
     'subscriptions': {
         'test_clock',
+        'application',
+        'description'
+    },
+    'products': {
+        'default_price'
     },
-    'products':set(),
     'invoice_items':{
         'test_clock',
     },
     'payouts':set(),
-    'charges': set(),
+    'charges': {
+        'failure_balance_transaction'
+    },
     'subscription_items': set(),
     'plans': set(),
     'invoice_line_items': set(),
     'invoices': {
         'test_clock',
+        'application'
+    },
+    'payment_intents': {
+        'amount_details'
     }
 }
 
@@ -182,16 +192,28 @@
         'plan', # BUG_12478 | missing subfields
     },
     'payouts': set(),
-    'charges': set(),
+    'charges': {
+        # missing subfield ['card.mandate']
+        'payment_method_details'
+    },
     'subscription_items': {
         # BUG_12478 | missing subfields on plan ['statement_description', 'statement_descriptor', 'name']
         'plan',
+        # missing subfields on price ['recurring.trial_period_days']
+        'price'
     },
     'invoices': {
         'plans', # BUG_12478 | missing subfields
     },
     'plans': set(),
-    'payment_intents':set(),
+    'payment_intents':{
+        # missing subfield ['payment_method_details.card.mandate']
+        'charges',
+        # missing subfield ['card.mandate_options']
+        'payment_method_options',
+        # missing subfield ['payment_method']
+        'last_payment_error'
+    },
     'invoice_line_items': set()
     # 'invoice_line_items': { # TODO This is a test issue that prevents us from consistently passing
     #     'unique_line_item_id',
@@ -218,6 +240,7 @@
     },
     'charges': {
         'status', # expect 'paid', get 'succeeded'
+        'receipt_url' # keeps changing with every request
     },
     'subscription_items': set(),
     'invoices': {
@@ -251,11 +274,6 @@
     },
 }
 
-# As for the `price` field added in the schema, the API doc doesn't mention any
-# `trial_period_days` in the field, hence skipping the assertion error for the same.
-KNOWN_NESTED_MISSING_FIELDS = {
-    'subscription_items': {'price': 'recurring.trial_period_days'}
-}
 
 class ALlFieldsTest(BaseTapTest):
     """Test tap sets a bookmark and respects it for the next sync of a stream"""
@@ -354,25 +372,6 @@ def test_run(self):
                 # run the test
                 self.all_fields_test(streams_to_test)
 
-    def find_nested_key(self, nested_key, actual_field_value, field):
-        '''
-        Find the nested key that is failing in the field and ignore the assertion error
-        gained from it, if any.
-        '''
-        for field_name, each_keys in nested_key.items():
-            # split the keys through `.`, for getting the nested keys
-            keys = each_keys.split('.')
-            temp_value = actual_field_value
-            if field == field_name:
-                for failing_key in keys:
-                    # if the failing key is not present in the actual key or not
-                    if not temp_value.get(failing_key, None):
-                        return False
-                    else:
-                        temp_value = temp_value.get(failing_key)
-                        if keys[-1] in temp_value:
-                            return True
-
     def all_fields_test(self, streams_to_test):
         """
         Verify that for each stream data is synced when all fields are selected.
@@ -547,10 +546,6 @@ def all_fields_test(self, streams_to_test):
                                     print(f"WARNING {base_err_msg} failed exact comparison.\n"
                                         f"AssertionError({failure_1})")
 
-                                    nested_key = KNOWN_NESTED_MISSING_FIELDS.get(stream, {})
-                                    if self.find_nested_key(nested_key, expected_field_value, field):
-                                        continue
-
                                     if field in KNOWN_FAILING_FIELDS[stream] or field in FIELDS_TO_NOT_CHECK[stream]:
                                         continue # skip the following wokaround
 

tests/unittests/test_log_request_id.py

@@ -0,0 +1,33 @@
+import unittest
+from unittest import mock
+from tap_stripe import new_request
+
+class MockRequest():
+    '''Mock Request object'''
+    def __init__(self, response):
+        self.last_response = response
+
+    def request_raw(self, method, url, params=None, supplied_headers=None, is_streaming=False):
+        return {}, {}, {}, {}
+
+    def interpret_response(self, rbody, rcode, rheaders):
+        return get_request_id()
+
+class MockResponse():
+    '''Mock response object which contains the request_id'''
+    def __init__(self, request_id):
+        self.request_id = request_id
+
+
+def get_request_id():
+    '''Return the MockRequest object which contains request_id'''
+    response = MockResponse('dummy_request_id')
+    return response
+
+class TestDebugLogger(unittest.TestCase):
+    @mock.patch('tap_stripe.LOGGER.debug')
+    def test_debug_logger(self, mock_debug):
+        '''Test that the debug is called with proper request id.'''
+        mock_request = MockRequest('url')
+        new_request(mock_request, 'GET', 'dummy_url')
+        mock_debug.assert_called_with('request id : dummy_request_id')