diff --git a/.circleci/config.yml b/.circleci/config.yml index fedd840e..e7aa66c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,7 +71,8 @@ jobs: name: 'Unit Tests' command: | source /usr/local/share/virtualenvs/tap-stripe/bin/activate - nosetests tests/unittests + nosetests --with-coverage --cover-erase --cover-package=tap_stripe --cover-html-dir=htmlcov tests/unittests + coverage html run_integration_test: parameters: file: @@ -91,7 +92,7 @@ jobs: source /usr/local/share/virtualenvs/tap-stripe/bin/activate source /usr/local/share/virtualenvs/tap-tester/bin/activate source /usr/local/share/virtualenvs/dev_env.sh - pip install 'stripe==2.42.0' + pip install 'stripe==2.64.0' run-test --tap=${CIRCLE_PROJECT_REPONAME} tests/test_<< parameters.file >>.py - slack/notify-on-failure: only_for_branches: master @@ -214,7 +215,7 @@ workflows: - tier-1-tap-user requires: - 'Testing bookmarks' -build_daily: + build_daily: <<: *commit_jobs triggers: - schedule: diff --git a/setup.py b/setup.py index 5fa0ad6f..e09fbbb4 100755 --- a/setup.py +++ b/setup.py @@ -11,12 +11,13 @@ py_modules=["tap_stripe"], install_requires=[ "singer-python==5.5.1", - "stripe==2.10.1", + "stripe==2.64.0", ], extras_require={ 'test': [ 'pylint==2.7.2', - 'nose==1.3.7' + 'nose==1.3.7', + 'coverage' ], 'dev': [ 'ipdb', diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index 47f83d30..c4f58fd8 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -8,6 +8,7 @@ import stripe import stripe.error from stripe.stripe_object import StripeObject +from stripe.util import convert_to_stripe_object import singer from singer import utils, Transformer, metrics from singer import metadata @@ -46,7 +47,7 @@ 'events': 'created', 'customers': 'created', 'plans': 'created', - 'invoices': 'date', + 'invoices': 'created', 'invoice_items': 'date', 'transfers': 'created', 'coupons': 'created', @@ -80,6 +81,20 @@ # payouts - these are called transfers with an event type of payout.* } +# Some fields are not available by default with latest API version so +# retrieve it by passing expand paramater in SDK object +STREAM_TO_EXPAND_FIELDS = { + # `tax_ids` field is not included in API response by default. To include it in the response, pass it in expand paramater. + # Reference: https://stripe.com/docs/api/customers/object#customer_object-tax_ids + + 'customers': ['data.sources', 'data.subscriptions', 'data.tax_ids'], + 'plans': ['data.tiers'], + 'invoice_items': ['data.plan.tiers'], + 'invoice_line_items': ['data.plan.tiers'], + 'subscriptions': ['data.plan.tiers'], + 'subscription_items': ['data.plan.tiers'] +} + SUB_STREAMS = { 'subscriptions': 'subscription_items', 'invoices': 'invoice_line_items', @@ -162,7 +177,7 @@ def configure_stripe_client(): # https://github.com/stripe/stripe-python/tree/a9a8d754b73ad47bdece6ac4b4850822fa19db4e#usage stripe.api_key = Context.config.get('client_secret') # Override the Stripe API Version for consistent access - stripe.api_version = '2018-09-24' + stripe.api_version = '2020-08-27' # Allow ourselves to retry retriable network errors 5 times # https://github.com/stripe/stripe-python/tree/a9a8d754b73ad47bdece6ac4b4850822fa19db4e#configuring-automatic-retries stripe.max_network_retries = 15 @@ -177,7 +192,7 @@ def configure_stripe_client(): account = stripe.Account.retrieve(Context.config.get('account_id')) msg = "Successfully connected to Stripe Account with display name" \ + " `%s`" - LOGGER.info(msg, account.display_name) + LOGGER.info(msg, account.settings.dashboard.display_name) def unwrap_data_objects(rec): """ @@ -371,10 +386,13 @@ def reduce_foreign_keys(rec, stream_name): return rec -def paginate(sdk_obj, filter_key, start_date, end_date, limit=100): +def paginate(sdk_obj, filter_key, start_date, end_date, stream_name, limit=100): yield from sdk_obj.list( limit=limit, stripe_account=Context.config.get('account_id'), + # Some fields are not available by default with latest API version so + # retrieve it by passing expand paramater in SDK object + expand=STREAM_TO_EXPAND_FIELDS.get(stream_name, []), # None passed to starting_after appears to retrieve # all of them so this should always be safe. **{filter_key + "[gte]": start_date, @@ -390,7 +408,60 @@ def dt_to_epoch(dt): def epoch_to_dt(epoch_ts): return datetime.fromtimestamp(epoch_ts) +def get_bookmark_for_stream(stream_name, replication_key): + """ + Returns bookmark for the streams from the state if found otherwise start_date + """ + # Invoices's replication key changed from `date` to `created` in latest API version. + # Invoice line Items write bookmark with Invoice's replication key but it changed to `created` + # so kept `date` in bookmarking for Invoices and Invoice line Items as it has to respect bookmark of active connection too. + if stream_name in ['invoices', 'invoice_line_items']: + stream_bookmark = singer.get_bookmark(Context.state, stream_name, 'date') or \ + int(utils.strptime_to_utc(Context.config["start_date"]).timestamp()) + else: + stream_bookmark = singer.get_bookmark(Context.state, stream_name, replication_key) or \ + int(utils.strptime_to_utc(Context.config["start_date"]).timestamp()) + return stream_bookmark + +def write_bookmark_for_stream(stream_name, replication_key, stream_bookmark): + """ + Write bookmark for the streams using replication key and bookmark value + """ + # Invoices's replication key changed from `date` to `created` in latest API version. + # Invoice line Items write bookmark with Invoice's replication key but it changed to `created` + # so kept `date` in bookmarking for Invoices and Invoice line Items as it has to respect bookmark of active connection too. + if stream_name in ['invoices', 'invoice_line_items']: + singer.write_bookmark(Context.state, + stream_name, + 'date', + stream_bookmark) + else: + singer.write_bookmark(Context.state, + stream_name, + replication_key, + stream_bookmark) + +def convert_dict_to_stripe_object(record): + """ + Convert field datatype of dict object to `stripe.stripe_object.StripeObject`. + Example: + record = {'id': 'dummy_id', 'tiers': [{"flat_amount": 4578"unit_amount": 7241350}]} + This function convert datatype of each record of 'tiers' field to `stripe.stripe_object.StripeObject`. + """ + # Loop through each fields of `record` object + for key, val in record.items(): + # Check type of field + if isinstance(val, list): + # Loop through each records of list + for index, field_val in enumerate(val): + if isinstance(field_val, dict): + # Convert datatype of dict to `stripe.stripe_object.StripeObject` + record[key][index] = convert_to_stripe_object(record[key][index]) + + return record + # pylint: disable=too-many-locals +# pylint: disable=too-many-statements def sync_stream(stream_name): """ Sync each stream, looking for newly created records. Updates are captured by events stream. @@ -404,8 +475,10 @@ def sync_stream(stream_name): replication_key = metadata.get(stream_metadata, (), 'valid-replication-keys')[0] # Invoice Items bookmarks on `date`, but queries on `created` filter_key = 'created' if stream_name == 'invoice_items' else replication_key - stream_bookmark = singer.get_bookmark(Context.state, stream_name, replication_key) or \ - int(utils.strptime_to_utc(Context.config["start_date"]).timestamp()) + + # Get bookmark for the stream + stream_bookmark = get_bookmark_for_stream(stream_name, replication_key) + bookmark = stream_bookmark # if this stream has a sub_stream, compare the bookmark @@ -414,8 +487,9 @@ def sync_stream(stream_name): # If there is a sub-stream and its selected, get its bookmark (or the start date if no bookmark) should_sync_sub_stream = sub_stream_name and Context.is_selected(sub_stream_name) if should_sync_sub_stream: - sub_stream_bookmark = singer.get_bookmark(Context.state, sub_stream_name, replication_key) \ - or int(utils.strptime_to_utc(Context.config["start_date"]).timestamp()) + + # Get bookmark for the sub_stream + sub_stream_bookmark = get_bookmark_for_stream(sub_stream_name, replication_key) # if there is a sub stream, set bookmark to sub stream's bookmark # since we know it must be earlier than the stream's bookmark @@ -454,10 +528,12 @@ def sync_stream(stream_name): stop_window = end_time for stream_obj in paginate(STREAM_SDK_OBJECTS[stream_name]['sdk_object'], - filter_key, start_window, stop_window): + filter_key, start_window, stop_window, stream_name): # get the replication key value from the object rec = unwrap_data_objects(stream_obj.to_dict_recursive()) + # convert field datatype of dict object to `stripe.stripe_object.StripeObject` + rec = convert_dict_to_stripe_object(rec) rec = reduce_foreign_keys(rec, stream_name) stream_obj_created = rec[replication_key] rec['updated'] = stream_obj_created @@ -488,18 +564,15 @@ def sync_stream(stream_name): # Update stream/sub-streams bookmarks as stop window if stop_window > stream_bookmark: stream_bookmark = stop_window - singer.write_bookmark(Context.state, - stream_name, - replication_key, - stream_bookmark) + # Write bookmark for the stream + write_bookmark_for_stream(stream_name, replication_key, stream_bookmark) # the sub stream bookmarks on its parent if should_sync_sub_stream and stop_window > sub_stream_bookmark: sub_stream_bookmark = stop_window - singer.write_bookmark(Context.state, - sub_stream_name, - replication_key, - sub_stream_bookmark) + + # Write bookmark for the sub_stream + write_bookmark_for_stream(sub_stream_name, replication_key, sub_stream_bookmark) singer.write_state(Context.state) diff --git a/tap_stripe/schemas/customers.json b/tap_stripe/schemas/customers.json index 83da121f..20357a96 100644 --- a/tap_stripe/schemas/customers.json +++ b/tap_stripe/schemas/customers.json @@ -209,6 +209,22 @@ } ] }, + "tax_ids": { + "anyOf": [ + { + "type": [ + "null", + "array" + ], + "items": { + "$ref": "shared/tax_ids.json#/" + } + }, + { + "$ref": "shared/tax_ids.json#/" + } + ] + }, "delinquent": { "type": [ "null", diff --git a/tap_stripe/schemas/invoice_line_items.json b/tap_stripe/schemas/invoice_line_items.json index 5999fce2..1ed5e831 100644 --- a/tap_stripe/schemas/invoice_line_items.json +++ b/tap_stripe/schemas/invoice_line_items.json @@ -169,6 +169,312 @@ } } }, + "tax_rates": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "active": { + "type": [ + "null", + "boolean" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "display_name": { + "type": [ + "null", + "string" + ] + }, + "inclusive": { + "type": [ + "null", + "boolean" + ] + }, + "jurisdiction": { + "type": [ + "null", + "string" + ] + }, + "livemode": { + "type": [ + "null", + "boolean" + ] + }, + "percentage": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "tax_type": { + "type": [ + "null", + "boolean" + ] + }, + "metadata": { + "type": [ + "null", + "object" + ], + "properties": {} + } + } + } + }, + "price": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "active": { + "type": [ + "null", + "boolean" + ] + }, + "billing_scheme": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "lookup_key": { + "type": [ + "null", + "string" + ] + }, + "nickname": { + "type": [ + "null", + "string" + ] + }, + "product": { + "type": [ + "null", + "string" + ] + }, + "recurring": { + "type": [ + "null", + "object" + ], + "properties": { + "aggregate_usage": { + "type": [ + "null", + "string" + ] + }, + "interval": { + "type": [ + "null", + "string" + ] + }, + "interval_count": { + "type": [ + "null", + "integer" + ] + }, + "usage_type": { + "type": [ + "null", + "string" + ] + } + } + }, + "tax_behavior": { + "type": [ + "null", + "string" + ] + }, + "tiers": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "flat_amount": { + "type": [ + "null", + "integer" + ] + }, + "flat_amount_decimal": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "unit_amount": { + "type": [ + "null", + "integer" + ] + }, + "unit_amount_decimal": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "up_to": { + "type": [ + "null", + "integer" + ] + } + } + } + }, + "tiers_mode": { + "type": [ + "null", + "string" + ] + }, + "transform_quantity": { + "type": [ + "null", + "object" + ], + "properties": { + "divide_by": { + "type": [ + "null", + "integer" + ] + }, + "round": { + "type": [ + "null", + "string" + ] + } + } + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "unit_amount": { + "type": [ + "null", + "integer" + ] + }, + "unit_amount_decimal": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "livemode": { + "type": [ + "null", + "boolean" + ] + }, + "metadata": { + "type": [ + "null", + "object" + ], + "properties": {} + } + } + }, "subscription": { "type": [ "null", diff --git a/tap_stripe/schemas/invoices.json b/tap_stripe/schemas/invoices.json index 627ec4d8..719dbe09 100644 --- a/tap_stripe/schemas/invoices.json +++ b/tap_stripe/schemas/invoices.json @@ -707,6 +707,12 @@ "integer" ] }, + "application_fee_amount": { + "type": [ + "null", + "integer" + ] + }, "lines": { "type": [ "null", @@ -775,6 +781,158 @@ "string" ] }, + "payment_settings": { + "type": [ + "null", + "object" + ], + "properties": { + "payment_method_options": { + "type": [ + "null", + "object" + ], + "properties": { + "acss_debit": { + "type": [ + "null", + "object" + ], + "properties": { + "mandate_options": { + "type": [ + "null", + "object" + ], + "properties": { + "transaction_type": { + "type": [ + "null", + "string" + ] + } + } + }, + "verification_method": { + "type": [ + "null", + "string" + ] + } + } + }, + "bancontact": { + "type": [ + "null", + "object" + ], + "properties": { + "preferred_language": { + "type": [ + "null", + "string" + ] + } + } + }, + "card": { + "type": [ + "null", + "object" + ], + "properties": { + "request_three_d_secure": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "payment_method_types": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "on_behalf_of": { + "type": [ + "null", + "string", + "object" + ], + "properties": {} + }, + "custom_fields": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "name": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "paid_out_of_band": { + "type": [ + "null", + "boolean" + ] + }, + "automatic_tax": { + "type": [ + "null", + "object" + ], + "properties": { + "enabled": { + "type": [ + "null", + "boolean" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + } + } + }, + "quote": { + "type": [ + "null", + "object", + "string" + ], + "properties": {} + }, "updated": { "type": [ "null", diff --git a/tap_stripe/schemas/products.json b/tap_stripe/schemas/products.json index 53e2cdc6..e94957bf 100644 --- a/tap_stripe/schemas/products.json +++ b/tap_stripe/schemas/products.json @@ -164,6 +164,12 @@ "null", "string" ] + }, + "tax_code": { + "type": [ + "null", + "string" + ] } } } diff --git a/tap_stripe/schemas/shared/discount.json b/tap_stripe/schemas/shared/discount.json index 09aad6c6..a9ad6941 100644 --- a/tap_stripe/schemas/shared/discount.json +++ b/tap_stripe/schemas/shared/discount.json @@ -106,7 +106,7 @@ "percent_off": { "type": [ "null", - "integer" + "number" ] }, "created": { @@ -142,6 +142,36 @@ "null", "string" ] + }, + "checkout_session": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "invoice": { + "type": [ + "null", + "string" + ] + }, + "invoice_item": { + "type": [ + "null", + "string" + ] + }, + "promotion_code": { + "type": [ + "null", + "string" + ] } } } diff --git a/tap_stripe/schemas/shared/tax_ids.json b/tap_stripe/schemas/shared/tax_ids.json new file mode 100644 index 00000000..e1fcb41e --- /dev/null +++ b/tap_stripe/schemas/shared/tax_ids.json @@ -0,0 +1,84 @@ +{ + "type": [ + "null", + "object" + ], + "properties":{ + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "customer": { + "type": [ + "null", + "string" + ] + }, + "livemode": { + "type": [ + "null", + "boolean" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "verification": { + "type": [ + "null", + "object" + ], + "properties": { + "status": { + "type": [ + "null", + "string" + ] + }, + "verified_address": { + "type": [ + "null", + "string" + ] + }, + "verified_name": { + "type": [ + "null", + "string" + ] + } + } + } + } +} + \ No newline at end of file diff --git a/tap_stripe/schemas/subscription_items.json b/tap_stripe/schemas/subscription_items.json index 86200939..65051792 100644 --- a/tap_stripe/schemas/subscription_items.json +++ b/tap_stripe/schemas/subscription_items.json @@ -26,6 +26,296 @@ "format": "date-time" }, "plan": {"$ref": "shared/plan.json#/"}, + "tax_rates": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "active": { + "type": [ + "null", + "boolean" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "display_name": { + "type": [ + "null", + "string" + ] + }, + "inclusive": { + "type": [ + "null", + "boolean" + ] + }, + "jurisdiction": { + "type": [ + "null", + "string" + ] + }, + "livemode": { + "type": [ + "null", + "boolean" + ] + }, + "percentage": { + "type": [ + "null", + "string" + ], + "format": "singer.decimal" + }, + "state": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "price": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "object": { + "type": [ + "null", + "string" + ] + }, + "active": { + "type": [ + "null", + "boolean" + ] + }, + "billing_scheme": { + "type": [ + "null", + "string" + ] + }, + "created": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "lookup_key": { + "type": [ + "null", + "string" + ] + }, + "nickname": { + "type": [ + "null", + "string" + ] + }, + "product": { + "type": [ + "null", + "string" + ] + }, + "recurring": { + "type": [ + "null", + "object" + ], + "properties": { + "aggregate_usage": { + "type": [ + "null", + "string" + ] + }, + "interval": { + "type": [ + "null", + "string" + ] + }, + "interval_count": { + "type": [ + "null", + "integer" + ] + }, + "usage_type": { + "type": [ + "null", + "string" + ] + } + } + }, + "tax_behavior": { + "type": [ + "null", + "string" + ] + }, + "tiers": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "flat_amount": { + "type": [ + "null", + "integer" + ] + }, + "flat_amount_decimal": { + "type": [ + "null", + "string" + ] + }, + "unit_amount": { + "type": [ + "null", + "integer" + ] + }, + "unit_amount_decimal": { + "type": [ + "null", + "string" + ] + }, + "up_to": { + "type": [ + "null", + "integer" + ] + } + } + } + }, + "tiers_mode": { + "type": [ + "null", + "string" + ] + }, + "transform_quantity": { + "type": [ + "null", + "object" + ], + "properties": { + "divide_by": { + "type": [ + "null", + "integer" + ] + }, + "round": { + "type": [ + "null", + "string" + ] + } + } + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "unit_amount": { + "type": [ + "null", + "integer" + ] + }, + "unit_amount_decimal": { + "type": [ + "null", + "string" + ] + }, + "livemode": { + "type": [ + "null", + "boolean" + ] + }, + "metadata": { + "type": [ + "null", + "object" + ], + "properties": {} + } + } + }, "subscription": { "type": [ "null", diff --git a/tests/base.py b/tests/base.py index 0ea946d1..10345204 100644 --- a/tests/base.py +++ b/tests/base.py @@ -8,6 +8,7 @@ import decimal from datetime import datetime as dt from datetime import timezone as tz +from dateutil import parser from tap_tester import connections, menagerie, runner @@ -79,12 +80,7 @@ def expected_metadata(self): 'events': default, 'customers': default, 'plans': default, - 'invoices': { - self.AUTOMATIC_FIELDS: {"updated"}, - self.REPLICATION_KEYS: {"date"}, - self.PRIMARY_KEYS: {"id"}, - self.REPLICATION_METHOD: self.INCREMENTAL, - }, + 'invoices': default, 'invoice_items': { self.AUTOMATIC_FIELDS: {"updated"}, self.REPLICATION_KEYS: {"date"}, @@ -314,15 +310,31 @@ def split_records_into_created_and_updated(self, records): 'schema': batch['schema'], 'key_names' : batch.get('key_names'), 'table_version': batch.get('table_version')} - created[stream]['messages'] += [m for m in batch['messages'] - if m['data'].get("updated") == m['data'].get(bookmark_key)] + # Bookmark key changed for `invoices` from `date` to `created` due to latest API change + # but for `invoices` stream, the `created` field have integer type(epoch format) from starting so + # converting `updated` to epoch for comparison. + if stream == "invoices": + created[stream]['messages'] += [m for m in batch['messages'] + if self.dt_to_ts(m['data'].get("updated")) == m['data'].get(bookmark_key)] + else: + created[stream]['messages'] += [m for m in batch['messages'] + if m['data'].get("updated") == m['data'].get(bookmark_key)] + if stream not in updated: updated[stream] = {'messages': [], 'schema': batch['schema'], 'key_names' : batch.get('key_names'), 'table_version': batch.get('table_version')} - updated[stream]['messages'] += [m for m in batch['messages'] - if m['data'].get("updated") != m['data'].get(bookmark_key)] + + # Bookmark key changed for `invoices` from `date` to `created` due to latest API change + # but for `invoices` stream, the `created` field have integer type(epoch format) from starting so + # converting `updated` to epoch for comparison. + if stream == "invoices": + updated[stream]['messages'] += [m for m in batch['messages'] + if self.dt_to_ts(m['data'].get("updated")) != m['data'].get(bookmark_key)] + else: + updated[stream]['messages'] += [m for m in batch['messages'] + if m['data'].get("updated") != m['data'].get(bookmark_key)] return created, updated def select_all_streams_and_fields(self, conn_id, catalogs, select_all_fields: bool = True, exclude_streams=None): @@ -377,9 +389,10 @@ def records_data_type_conversions(self, records): int_or_float_to_decimal_keys = [ 'percent_off', 'percent_off_precise', 'height', 'length', 'weight', 'width' ] + object_keys = [ 'discount', 'plan', 'coupon', 'status_transitions', 'period', 'sources', 'source', - 'package_dimensions', + 'package_dimensions', 'price' # Convert epoch timestamp value of 'price.created' to standard datetime format. This field is available specific for invoice_line_items stream ] # timestamp to datetime @@ -523,3 +536,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.start_date = self.get_properties().get('start_date') self.maxDiff=None + + + def dt_to_ts(self, dtime): + return parser.parse(dtime).timestamp() diff --git a/tests/test_all_fields.py b/tests/test_all_fields.py index 4623b954..48b36a0b 100644 --- a/tests/test_all_fields.py +++ b/tests/test_all_fields.py @@ -18,56 +18,119 @@ # Fields that are consistently missing during replication # Original Ticket [https://stitchdata.atlassian.net/browse/SRCE-4736] KNOWN_MISSING_FIELDS = { - 'customers':{ - 'tax_ids', - }, + 'customers':set(), 'subscriptions':{ 'payment_settings', 'default_tax_rates', 'pending_update', 'automatic_tax', }, - 'products':{ - 'skus', - 'tax_code', - }, + 'products':set(), 'invoice_items':{ 'price', }, + 'payouts':set(), + 'charges': set(), + 'subscription_items': set(), + 'plans': set(), + 'invoice_line_items': set(), + 'invoices': set() +} + +FIELDS_TO_NOT_CHECK = { + 'customers': { + # Below fields are deprecated or renamed.(https://stripe.com/docs/upgrades#2019-10-17, https://stripe.com/docs/upgrades#2019-12-03) + 'account_balance', + 'tax_info', + 'tax_info_verification', + 'cards', + 'default_card' + }, + 'subscriptions':{ + # Below fields are deprecated or renamed.(https://stripe.com/docs/upgrades#2019-10-17, https://stripe.com/docs/upgrades#2019-12-03, https://stripe.com/docs/upgrades#2020-08-27) + 'billing', + 'start', + 'tax_percent', + 'invoice_customer_balance_settings' + }, + 'products':{ + # Below fields are available in the product record only if the value of the type field is `service`. + # But, currently, during crud operation in all_fields test case, it creates product records of type `good`. + 'statement_descriptor', + 'unit_label' + }, + 'coupons':{ + # Field is not available in stripe documentation and also not returned by API response.(https://stripe.com/docs/api/coupons/object) + 'percent_off_precise' + }, + 'invoice_items':set(), 'payouts':{ - 'application_fee', - 'reversals', - 'reversed', + + # Following fields are not mentioned in the documentation and also not returned by API (https://stripe.com/docs/api/payouts/object) + 'statement_description', + 'transfer_group', + 'source_transaction', + 'bank_account', + 'date', + 'amount_reversed', + 'recipient' + }, + 'charges': { + # Following both fields `card` and `statement_description` are deprecated. (https://stripe.com/docs/upgrades#2015-02-18, https://stripe.com/docs/upgrades#2014-12-17) + 'card', + 'statement_description' }, - 'charges': set(), 'subscription_items':{ - 'tax_rates', - 'price', - 'updated', + # Field is not available in stripe documentation and also not returned by API response. (https://stripe.com/docs/api/subscription_items/object) + 'current_period_end', + 'customer', + 'trial_start', + 'discount', + 'start', + 'tax_percent', + 'livemode', + 'application_fee_percent', + 'status', + 'trial_end', + 'ended_at', + 'current_period_start', + 'canceled_at', + 'cancel_at_period_end' }, 'invoices':{ - 'payment_settings', - 'on_behalf_of', - 'custom_fields', - 'automatic_tax', - 'quote', + # Below fields are deprecated or renamed.(https://stripe.com/docs/upgrades#2019-03-14, https://stripe.com/docs/upgrades#2019-10-17, https://stripe.com/docs/upgrades#2018-08-11 + # https://stripe.com/docs/upgrades#2020-08-27) + 'application_fee', + 'billing', + 'closed', + 'date', + # This field is deprcated in the version 2020-08-27 + 'finalized_at', + 'forgiven', + 'tax_percent', + 'statement_description', + 'payment' }, - 'plans': set(), - 'invoice_line_items': { - 'tax_rates', - 'unique_id', - 'updated', - 'price', + 'plans': { + # Below fields are deprecated or renamed. (https://stripe.com/docs/upgrades#2018-02-05, https://stripe.com/docs/api/plans/object) + 'statement_descriptor', + 'statement_description', + 'name', + 'tiers' # Field is not returned by API }, + 'invoice_line_items': { + # As per stripe documentation(https://stripe.com/docs/api/invoices/line_item#invoice_line_item_object-subscription_item), + # 'subscription_item' is field that generated invoice item. It does not replicate in response if the line item is not an explicit result of a subscription. + # So, due to uncertainty of this field, skipped it. + 'subscription_item' + } } KNOWN_FAILING_FIELDS = { 'coupons': { 'percent_off', # BUG_9720 | Decimal('67') != Decimal('66.6') (value is changing in duplicate records) }, - 'customers': { - 'discount', # BUG_12478 | missing subfields in coupon where coupon is subfield within discount - }, + 'customers': set(), 'subscriptions': { # BUG_12478 | missing subfields in coupon where coupon is subfield within discount # BUG_12478 | missing subfields on discount ['checkout_session', 'id', 'invoice', 'invoice_item', 'promotion_code'] @@ -88,14 +151,12 @@ 'plan', }, 'invoices': { - 'discount', # BUG_12478 | missing subfields 'plans', # BUG_12478 | missing subfields - 'finalized_at', # BUG_13711 | schema missing datetime format - 'created', # BUG_13711 | schema missing datetime format }, 'plans': { 'transform_usage' # BUG_13711 schema is wrong, should be an object not string }, + 'invoice_line_items': set() # 'invoice_line_items': { # TODO This is a test issue that prevents us from consistently passing # 'unique_line_item_id', # 'invoice_item', @@ -141,16 +202,19 @@ }, 'payouts': {'updated'}, 'charges': {'updated'}, - 'subscription_items': {'updated'}, + 'subscription_items': set(), # `updated` is not added by the tap for child streams. 'invoices': {'updated'}, 'plans': {'updated'}, 'invoice_line_items': { - 'updated', - 'invoice', - 'subscription_item' + 'invoice' }, } +# As for the `price` field added in the schema, the API doc doesn't mention any +# `trial_period_days` in the field, hence skipping the assertion error for the same. +KNOWN_NESTED_MISSING_FIELDS = { + 'subscription_items': {'price': 'recurring.trial_period_days'} +} class ALlFieldsTest(BaseTapTest): """Test tap sets a bookmark and respects it for the next sync of a stream""" @@ -248,11 +312,28 @@ def test_run(self): # run the test self.all_fields_test(streams_to_test) + def find_nested_key(self, nested_key, actual_field_value, field): + ''' + Find the nested key that is failing in the field and ignore the assertion error + gained from it, if any. + ''' + for field_name, each_keys in nested_key.items(): + # split the keys through `.`, for getting the nested keys + keys = each_keys.split('.') + temp_value = actual_field_value + if field == field_name: + for failing_key in keys: + # if the failing key is not present in the actual key or not + if not temp_value.get(failing_key, None): + return False + else: + temp_value = temp_value.get(failing_key) + if keys[-1] in temp_value: + return True def all_fields_test(self, streams_to_test): """ Verify that for each stream data is synced when all fields are selected. - Verify the synced data matches our expectations based off of the applied schema and results from the test client utils. """ @@ -296,30 +377,38 @@ def all_fields_test(self, streams_to_test): # collect actual values actual_records = synced_records.get(stream) - actual_records_data = [message['data'] for message in actual_records.get('messages')] + # Only 1st half records belong to actual stream, next half records belong to events of that stream + # So, skipping records of events + actual_record_message = actual_records.get('messages')[:len(actual_records.get('messages'))//2] + actual_records_data = [message['data'] for message in actual_record_message] actual_records_keys = set() - for message in actual_records['messages']: + for message in actual_record_message: if message['action'] == 'upsert': actual_records_keys.update(set(message['data'].keys())) schema_keys = set(self.expected_schema_keys(stream)) # read in from schema files + # To avoid warning, skipping fields of FIELDS_TO_NOT_CHECK + schema_keys = schema_keys - FIELDS_TO_NOT_CHECK.get(stream, set()) + actual_records_keys = actual_records_keys - FIELDS_TO_NOT_CHECK[stream] + + # Append fields which are added by tap to expectation + adjusted_expected_keys = expected_records_keys.union( + FIELDS_ADDED_BY_TAP.get(stream, set()) + ) # Log the fields that are included in the schema but not in the expectations. # These are fields we should strive to get data for in our test data set - if schema_keys.difference(expected_records_keys): + if schema_keys.difference(adjusted_expected_keys): print("WARNING Stream[{}] Fields missing from expectations: [{}]".format( - stream, schema_keys.difference(expected_records_keys) + stream, schema_keys.difference(adjusted_expected_keys) )) - # Verify schema covers all fields - adjusted_expected_keys = expected_records_keys.union( - FIELDS_ADDED_BY_TAP.get(stream, set()) - ) adjusted_actual_keys = actual_records_keys.union( # BUG_12478 KNOWN_MISSING_FIELDS.get(stream, set()) ) if stream == 'invoice_items': adjusted_actual_keys = adjusted_actual_keys.union({'subscription_item'}) # BUG_13666 + self.assertSetEqual(adjusted_expected_keys, adjusted_actual_keys) # verify the missing fields from KNOWN_MISSING_FIELDS are always missing (stability check) @@ -390,6 +479,11 @@ def all_fields_test(self, streams_to_test): expected_field_value = expected_record.get(field, "EXPECTED IS MISSING FIELD") actual_field_value = actual_record.get(field, "ACTUAL IS MISSING FIELD") + # to fix the failure warning of `created` for `invoices` stream + # BUG_13711 | the schema was missing datetime format and the tests were throwing a warning message. + # Hence, a workaround to remove that warning message. + if stream == 'invoices' and expected_field_value != "EXPECTED IS MISSING FIELD" and field == 'created': + expected_field_value = int(self.dt_to_ts(expected_field_value)) try: self.assertEqual(expected_field_value, actual_field_value) @@ -397,9 +491,13 @@ def all_fields_test(self, streams_to_test): except AssertionError as failure_1: print(f"WARNING {base_err_msg} failed exact comparison.\n" - f"AssertionError({failure_1})") + f"AssertionError({failure_1})") + + nested_key = KNOWN_NESTED_MISSING_FIELDS.get(stream, {}) + if self.find_nested_key(nested_key, expected_field_value, field): + continue - if field in KNOWN_FAILING_FIELDS[stream]: + if field in KNOWN_FAILING_FIELDS[stream] or field in FIELDS_TO_NOT_CHECK[stream]: continue # skip the following wokaround elif actual_field_value and field in FICKLE_FIELDS[stream]: @@ -409,4 +507,4 @@ def all_fields_test(self, streams_to_test): raise AssertionError(f"{base_err_msg} Unexpected field is being fickle.") else: - print(f"WARNING {base_err_msg} failed datatype comparison. Field is None.") + print(f"WARNING {base_err_msg} failed datatype comparison. Field is None.") \ No newline at end of file diff --git a/tests/unittests/test_dict_to_stripe_object.py b/tests/unittests/test_dict_to_stripe_object.py new file mode 100644 index 00000000..84df40c2 --- /dev/null +++ b/tests/unittests/test_dict_to_stripe_object.py @@ -0,0 +1,24 @@ +import stripe +import unittest +from tap_stripe import convert_dict_to_stripe_object + +class TestDictTOSTRIPEOBJECT(unittest.TestCase): + + def test_dict_to_stripe_object(self): + """ + Test that `convert_dict_to_stripe_object` function convert field datatype of dict to stripe.stripe_object.StripeObject + Example: + """ + mock_object = { + "id": "dummy_id", + "tiers": [ + { + "flat_amount": 10, + "unit_amount": 7241350 + } + ], + "tier_mode": "volume" + } + + # Verify that type of `tiers` field is stripe.stripe_object.StripeObject + self.assertTrue(isinstance(convert_dict_to_stripe_object(mock_object).get('tiers')[0], stripe.stripe_object.StripeObject)) \ No newline at end of file diff --git a/tests/unittests/test_get_and_write_bookmark.py b/tests/unittests/test_get_and_write_bookmark.py new file mode 100644 index 00000000..c9a545a2 --- /dev/null +++ b/tests/unittests/test_get_and_write_bookmark.py @@ -0,0 +1,86 @@ +import tap_stripe +import unittest +from unittest import mock + +class TestGetBookmarks(unittest.TestCase): + + @mock.patch("tap_stripe.singer.get_bookmark") + def test_get_bookmark_for_invoices(self, mocked_get_bookmark): + ''' + Verify that invoices use `date` field to get bookmark and not a replication key `created` for invoices + ''' + # Call get_bookmark_for_stream for invoices with `created` replication key + tap_stripe.get_bookmark_for_stream("invoices", "created") + + # Verify that get_bookmark is called with 'date' field + args, kwargs = mocked_get_bookmark.call_args + self.assertEquals(args[1], "invoices") + self.assertEquals(args[2], "date") + + @mock.patch("tap_stripe.singer.get_bookmark") + def test_get_bookmark_for_invoice_line_items(self, mocked_get_bookmark): + ''' + Verify that invoice_line_items use `date` field to get bookmark and not a replication key `created` for invoice_line_items + ''' + # Call get_bookmark_for_stream for invoice_line_items with `created` replication key + tap_stripe.get_bookmark_for_stream("invoice_line_items", "created") + + # Verify that get_bookmark is called with 'date' field + args, kwargs = mocked_get_bookmark.call_args + self.assertEquals(args[1], "invoice_line_items") + self.assertEquals(args[2], "date") + + @mock.patch("tap_stripe.singer.get_bookmark") + def test_get_bookmark_for_normal_streams(self, mocked_get_bookmark): + ''' + Verify that streams other than invoice and invoice_line_items use passed replication key to get bookmark + ''' + # Call get_bookmark_for_stream for other test stream with `test_replication_key` replication key + tap_stripe.get_bookmark_for_stream("test", "test_replication_key") + + # Verify that get_bookmark is called with 'test_replication_key' field which passed in get_bookmark_for_stream() + args, kwargs = mocked_get_bookmark.call_args + self.assertEquals(args[1], "test") + self.assertEquals(args[2], "test_replication_key") + + +class TestWriteBookmarks(unittest.TestCase): + + @mock.patch("tap_stripe.singer.write_bookmark") + def test_write_bookmark_for_invoices(self, mocked_write_bookmark): + ''' + Verify that invoices use `date` field to write bookmark and not a replication key `created` for invoices + ''' + # Call write_bookmark_for_stream for invoices with `created` replication key + tap_stripe.write_bookmark_for_stream("invoices", "created", "bookmark_value") + + # Verify that write_bookmark is called with 'date' field + args, kwargs = mocked_write_bookmark.call_args + self.assertEquals(args[1], "invoices") + self.assertEquals(args[2], "date") + + @mock.patch("tap_stripe.singer.write_bookmark") + def test_write_bookmark_for_invoice_line_items(self, mocked_write_bookmark): + ''' + Verify that invoice_line_items use `date` field to write bookmark and not a replication key `created` for invoice_line_items + ''' + # Call write_bookmark_for_stream for invoice_line_items with `created` replication key + tap_stripe.write_bookmark_for_stream("invoice_line_items", "created", "bookmark_value") + + # Verify that write_bookmark is called with 'date' field + args, kwargs = mocked_write_bookmark.call_args + self.assertEquals(args[1], "invoice_line_items") + self.assertEquals(args[2], "date") + + @mock.patch("tap_stripe.singer.write_bookmark") + def test_write_bookmark_for_normal_streams(self, mocked_write_bookmark): + ''' + Verify that streams other than invoice and invoice_line_items use passed replication key to write bookmark + ''' + # Call write_bookmark_for_stream for other test stream with `test_replication_key` replication key + tap_stripe.write_bookmark_for_stream("test", "test_replication_key", "bookmark_value") + + # Verify that write_bookmark is called with 'test_replication_key' field which passed in write_bookmark_for_stream() + args, kwargs = mocked_write_bookmark.call_args + self.assertEquals(args[1], "test") + self.assertEquals(args[2], "test_replication_key") \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py index 5937f302..c20820fa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,6 +16,7 @@ NOW = dt.utcnow() metadata_value = {"test_value": "senorita_alice_{}@stitchdata.com".format(NOW)} +stripe_client.api_version = '2020-08-27' stripe_client.api_key = BaseTapTest.get_credentials()["client_secret"] client = { 'balance_transactions': stripe_client.BalanceTransaction, @@ -218,16 +219,17 @@ def list_all_object(stream, max_limit: int = 100, get_invoice_lines: bool = Fals return objects elif stream == "customers": - stripe_obj = client[stream].list(limit=max_limit, created={"gte": midnight}) + stripe_obj = client[stream].list(limit=max_limit, created={"gte": midnight}, + expand=['data.sources', 'data.subscriptions', 'data.tax_ids']) # retrieve fields by passing expand paramater in SDK object dict_obj = stripe_obj_to_dict(stripe_obj) if dict_obj.get('data'): for obj in dict_obj['data']: - if obj['sources']: + if obj.get('sources'): sources = obj['sources']['data'] obj['sources'] = sources - if obj['subscriptions']: + if obj.get('subscriptions'): subscription_ids = [subscription['id'] for subscription in obj['subscriptions']['data']] obj['subscriptions'] = subscription_ids @@ -272,7 +274,7 @@ def standard_create(stream): address={'city': 'Philadelphia', 'country': 'US', 'line1': 'APT 2R.', 'line1': '666 Street Rd.', 'postal_code': '11111', 'state': 'PA'}, description="Description {}".format(NOW), - email="senor_bob_{}@stitchdata.com".format(NOW), + email="stitchdata.test@gmail.com", # In the latest API version, it is mandatory to provide a valid email address metadata=metadata_value, name="Roberto Alicia", # pyment_method=, see source explanation @@ -292,7 +294,7 @@ def standard_create(stream): ) elif stream == 'payouts': return client[stream].create( - amount=random.randint(0, 10000), + amount=random.randint(1, 10), currency="usd", description="Comfortable cotton t-shirt {}".format(NOW), statement_descriptor="desc", @@ -302,7 +304,7 @@ def standard_create(stream): elif stream == 'plans': return client[stream].create( active=True, - amount=random.randint(0, 10000), + amount=random.randint(1, 10000), currency="usd", interval="year", metadata=metadata_value, @@ -338,6 +340,7 @@ def standard_create(stream): package_dimensions={"height": 92670.16, "length": 9158.65, "weight": 582.73, "width": 96656496.18}, shippable=True, url='fakeurl.stitch', + type='good' # In the latest API version, it is mandatory to provide the value of the `type` field in the body. ) return None @@ -365,7 +368,7 @@ def create_object(stream): elif stream == 'customers': customer = standard_create(stream) customer_dict = stripe_obj_to_dict(customer) - if customer_dict['subscriptions']: + if customer_dict.get('subscriptions'): subscription_ids = [subscription['id'] for subscription in customer_dict['subscriptions']['data']] customer_dict['subscriptions'] = subscription_ids return customer_dict @@ -381,7 +384,7 @@ def create_object(stream): if stream == 'invoice_items': return client[stream].create( - amount=random.randint(0, 10000), + amount=random.randint(1, 10000), currency="usd", customer=cust['id'], description="Comfortable cotton t-shirt {}".format(NOW), @@ -393,7 +396,7 @@ def create_object(stream): elif stream == 'invoices': # Invoices requires the customer has an item associated with them item = client["{}_items".format(stream[:-1])].create( - amount=random.randint(0, 10000), + amount=random.randint(1, 10000), currency="usd", customer=cust['id'], description="Comfortable cotton t-shirt {}".format(NOW), @@ -519,7 +522,6 @@ def create_object(stream): def update_object(stream, oid): """ Update a specific record for a given object. - The update will always change the test_value under the metadata field which is found in all object streams. """ @@ -560,4 +562,4 @@ def delete_object(stream, oid): except: logging.info("DELETE FAILED of record {} in stream {}".format(oid, stream)) - return None + return None \ No newline at end of file