TDL-14058: Update SDK and api version

.circleci/config.yml

@@ -71,7 +71,8 @@ jobs:
           name: 'Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-stripe/bin/activate
-            nosetests tests/unittests
+            nosetests --with-coverage --cover-erase --cover-package=tap_stripe --cover-html-dir=htmlcov tests/unittests
+            coverage html
   run_integration_test:
     parameters:
       file:
@@ -91,7 +92,7 @@ jobs:
             source /usr/local/share/virtualenvs/tap-stripe/bin/activate
             source /usr/local/share/virtualenvs/tap-tester/bin/activate
             source /usr/local/share/virtualenvs/dev_env.sh
-            pip install 'stripe==2.42.0'
+            pip install 'stripe==2.64.0'
             run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_<< parameters.file >>.py
       - slack/notify-on-failure:
           only_for_branches: master

setup.py

@@ -11,12 +11,13 @@
     py_modules=["tap_stripe"],
     install_requires=[
         "singer-python==5.5.1",
-        "stripe==2.10.1",
+        "stripe==2.64.0",
     ],
     extras_require={
         'test': [
             'pylint==2.7.2',
-            'nose==1.3.7'
+            'nose==1.3.7',
+            'coverage'
         ],
         'dev': [
             'ipdb',

tap_stripe/__init__.py

@@ -46,7 +46,7 @@
     'events': 'created',
     'customers': 'created',
     'plans': 'created',
-    'invoices': 'date',
+    'invoices': 'created',
     'invoice_items': 'date',
     'transfers': 'created',
     'coupons': 'created',
@@ -80,6 +80,17 @@
     # payouts - these are called transfers with an event type of payout.*
 }
 
+# Some fields are not available by default with latest API version so
+# retrive it by passing expand paramater in SDK object
+STREAM_TO_EXPAND_FIELDS = {
+    'customers': ['data.sources', 'data.subscriptions'],
+    'plans': ['data.tiers'],
+    'invoice_items': ['data.plan.tiers'],
+    'invoice_line_items': ['data.plan.tiers'],
+    'subscriptions': ['data.plan.tiers'],
+    'subscription_items': ['data.plan.tiers']
+}
+
 SUB_STREAMS = {
     'subscriptions': 'subscription_items',
     'invoices': 'invoice_line_items',
@@ -162,7 +173,7 @@ def configure_stripe_client():
     # https://github.com/stripe/stripe-python/tree/a9a8d754b73ad47bdece6ac4b4850822fa19db4e#usage
     stripe.api_key = Context.config.get('client_secret')
     # Override the Stripe API Version for consistent access
-    stripe.api_version = '2018-09-24'
+    stripe.api_version = '2020-08-27'
     # Allow ourselves to retry retriable network errors 5 times
     # https://github.com/stripe/stripe-python/tree/a9a8d754b73ad47bdece6ac4b4850822fa19db4e#configuring-automatic-retries
     stripe.max_network_retries = 15
@@ -177,7 +188,7 @@ def configure_stripe_client():
     account = stripe.Account.retrieve(Context.config.get('account_id'))
     msg = "Successfully connected to Stripe Account with display name" \
           + " `%s`"
-    LOGGER.info(msg, account.display_name)
+    LOGGER.info(msg, account.settings.dashboard.display_name)
 
 def unwrap_data_objects(rec):
     """
@@ -371,10 +382,11 @@ def reduce_foreign_keys(rec, stream_name):
     return rec
 
 
-def paginate(sdk_obj, filter_key, start_date, end_date, limit=100):
+def paginate(sdk_obj, filter_key, start_date, end_date, stream_name, limit=100):
     yield from sdk_obj.list(
         limit=limit,
         stripe_account=Context.config.get('account_id'),
+        expand=STREAM_TO_EXPAND_FIELDS.get(stream_name, []),
         # None passed to starting_after appears to retrieve
         # all of them so this should always be safe.
         **{filter_key + "[gte]": start_date,
@@ -391,6 +403,7 @@ def epoch_to_dt(epoch_ts):
     return datetime.fromtimestamp(epoch_ts)
 
 # pylint: disable=too-many-locals
+# pylint: disable=too-many-statements
 def sync_stream(stream_name):
     """
     Sync each stream, looking for newly created records. Updates are captured by events stream.
@@ -404,8 +417,16 @@ def sync_stream(stream_name):
     replication_key = metadata.get(stream_metadata, (), 'valid-replication-keys')[0]
     # Invoice Items bookmarks on `date`, but queries on `created`
     filter_key = 'created' if stream_name == 'invoice_items' else replication_key
-    stream_bookmark = singer.get_bookmark(Context.state, stream_name, replication_key) or \
-        int(utils.strptime_to_utc(Context.config["start_date"]).timestamp())
+
+    # Invoice was bookmarking on `date` but in latest API version, that field is deprecated and replication key changed to `created`
+    # kept `date` in bookmarking as it as to respect bookmark of active connection too
+    if stream_name == 'invoices':
+        stream_bookmark = singer.get_bookmark(Context.state, stream_name, 'date') or \
+            int(utils.strptime_to_utc(Context.config["start_date"]).timestamp())
+    else:
+        stream_bookmark = singer.get_bookmark(Context.state, stream_name, replication_key) or \
+            int(utils.strptime_to_utc(Context.config["start_date"]).timestamp())
+
     bookmark = stream_bookmark
 
     # if this stream has a sub_stream, compare the bookmark
@@ -414,8 +435,16 @@ def sync_stream(stream_name):
     # If there is a sub-stream and its selected, get its bookmark (or the start date if no bookmark)
     should_sync_sub_stream = sub_stream_name and Context.is_selected(sub_stream_name)
     if should_sync_sub_stream:
-        sub_stream_bookmark = singer.get_bookmark(Context.state, sub_stream_name, replication_key) \
-            or int(utils.strptime_to_utc(Context.config["start_date"]).timestamp())
+
+        # Invoices's replication key changed from `date` to `created` in latest API version.
+        # Invoice line Items write bookmark with Invoice's replication key but it changed to `created`
+        # so kept `date` in bookmarking as it as to respect bookmark of active connection too.
+        if sub_stream_name == "invoice_line_items":
+            sub_stream_bookmark = singer.get_bookmark(Context.state, sub_stream_name, 'date') \
+                or int(utils.strptime_to_utc(Context.config["start_date"]).timestamp())
+        else:
+            sub_stream_bookmark = singer.get_bookmark(Context.state, sub_stream_name, replication_key) \
+                or int(utils.strptime_to_utc(Context.config["start_date"]).timestamp())
 
         # if there is a sub stream, set bookmark to sub stream's bookmark
         # since we know it must be earlier than the stream's bookmark
@@ -454,7 +483,7 @@ def sync_stream(stream_name):
                 stop_window = end_time
 
             for stream_obj in paginate(STREAM_SDK_OBJECTS[stream_name]['sdk_object'],
-                                       filter_key, start_window, stop_window):
+                                       filter_key, start_window, stop_window, stream_name):
 
                 # get the replication key value from the object
                 rec = unwrap_data_objects(stream_obj.to_dict_recursive())
@@ -488,18 +517,37 @@ def sync_stream(stream_name):
             # Update stream/sub-streams bookmarks as stop window
             if stop_window > stream_bookmark:
                 stream_bookmark = stop_window
-                singer.write_bookmark(Context.state,
-                                      stream_name,
-                                      replication_key,
-                                      stream_bookmark)
+                # Invoice was bookmarking on `date` but in latest API version,
+                # that field is deprecated and replication key changed to `created`
+                # kept `date` in bookmarking as it as to respect bookmark of active connection too.
+                if stream_name == "invoices":
+                    singer.write_bookmark(Context.state,
+                                          stream_name,
+                                          'date',
+                                          stream_bookmark)
+                else:
+                    singer.write_bookmark(Context.state,
+                                          stream_name,
+                                          replication_key,
+                                          stream_bookmark)
 
             # the sub stream bookmarks on its parent
             if should_sync_sub_stream and stop_window > sub_stream_bookmark:
                 sub_stream_bookmark = stop_window
-                singer.write_bookmark(Context.state,
-                                      sub_stream_name,
-                                      replication_key,
-                                      sub_stream_bookmark)
+
+                # Invoices's replication key changed from `date` to `created` in latest API version.
+                # Invoice line Items write bookmark with Invoice's replication key but it changed to `created`
+                # so kept `date` in bookmarking as it as to respect bookmark of active connection too.
+                if sub_stream_name == "invoice_line_items":
+                    singer.write_bookmark(Context.state,
+                                          sub_stream_name,
+                                          'date',
+                                          sub_stream_bookmark)
+                else:
+                    singer.write_bookmark(Context.state,
+                                          sub_stream_name,
+                                          replication_key,
+                                          sub_stream_bookmark)
 
             singer.write_state(Context.state)
 

tap_stripe/schemas/invoices.json

@@ -707,6 +707,12 @@
         "integer"
       ]
     },
+    "application_fee_amount": {
+      "type": [
+        "null",
+        "integer"
+      ]
+    },
     "lines": {
       "type": [
         "null",

tests/base.py

@@ -8,6 +8,7 @@
 import decimal
 from datetime import datetime as dt
 from datetime import timezone as tz
+from dateutil import parser
 
 from tap_tester import connections, menagerie, runner
 
@@ -79,12 +80,7 @@ def expected_metadata(self):
             'events': default,
             'customers': default,
             'plans': default,
-            'invoices': {
-                self.AUTOMATIC_FIELDS: {"updated"},
-                self.REPLICATION_KEYS: {"date"},
-                self.PRIMARY_KEYS: {"id"},
-                self.REPLICATION_METHOD: self.INCREMENTAL,
-            },
+            'invoices': default,
             'invoice_items': {
                 self.AUTOMATIC_FIELDS: {"updated"},
                 self.REPLICATION_KEYS: {"date"},
@@ -314,15 +310,31 @@ def split_records_into_created_and_updated(self, records):
                                    'schema': batch['schema'],
                                    'key_names' : batch.get('key_names'),
                                    'table_version': batch.get('table_version')}
-            created[stream]['messages'] += [m for m in batch['messages']
-                                                if m['data'].get("updated") == m['data'].get(bookmark_key)]
+            # Bookmark key changed for `invoices` from `date` to `created` due to latest API change
+            # but for `invoices` stream, the `created` field have integer type(epoch format) from starting so
+            # converting `updated` to epoch for comparison.
+            if stream == "invoices":
+                created[stream]['messages'] += [m for m in batch['messages']
+                                                    if self.dt_to_ts(m['data'].get("updated")) == m['data'].get(bookmark_key)]
+            else:
+                created[stream]['messages'] += [m for m in batch['messages']
+                                                    if m['data'].get("updated") == m['data'].get(bookmark_key)]
+
             if stream not in updated:
                 updated[stream] = {'messages': [],
                                    'schema': batch['schema'],
                                    'key_names' : batch.get('key_names'),
                                    'table_version': batch.get('table_version')}
-            updated[stream]['messages'] += [m for m in batch['messages']
-                                                if m['data'].get("updated") != m['data'].get(bookmark_key)]
+
+            # Bookmark key changed for `invoices` from `date` to `created` due to latest API change
+            # but for `invoices` stream, the `created` field have integer type(epoch format) from starting so
+            # converting `updated` to epoch for comparison.
+            if stream == "invoices":
+                updated[stream]['messages'] += [m for m in batch['messages']
+                                                    if self.dt_to_ts(m['data'].get("updated")) != m['data'].get(bookmark_key)]
+            else:
+                updated[stream]['messages'] += [m for m in batch['messages']
+                                                    if m['data'].get("updated") != m['data'].get(bookmark_key)]
         return created, updated
 
     def select_all_streams_and_fields(self, conn_id, catalogs, select_all_fields: bool = True, exclude_streams=None):
@@ -523,3 +535,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.start_date = self.get_properties().get('start_date')
         self.maxDiff=None
+
+
+    def dt_to_ts(self, dtime):
+        return parser.parse(dtime).timestamp()