From 2a7c378644c238a6dedb83a8206b7f5178273be7 Mon Sep 17 00:00:00 2001 From: Dan Mosora Date: Fri, 6 Dec 2019 20:02:50 +0000 Subject: [PATCH 1/3] Add 5 min lookback for immutable streams to prevent missing records --- tap_stripe/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index 5a7eed95..53a80eb4 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -87,6 +87,9 @@ 'payouts': 'payout_transactions' } +# NB: These streams will only sync through once for creates, never updates. +IMMUTABLE_STREAMS = {'balance_transactions', 'events'} + LOGGER = singer.get_logger() DEFAULT_DATE_WINDOW_SIZE = 30 #days @@ -429,6 +432,16 @@ def sync_stream(stream_name): LOGGER.info('Using non-default date window size of %d', window_size) start_window = bookmark + # NB: Immutable streams are never synced for updates. We've + # observed a short lag period between when records are created and + # when they are available via the API, so these streams will need + # a short lookback window. + # TODO: This may be an issue for other streams' created_at + # entries, but to keep the surface small, doing this only for + # immutable streams at first to confirm the suspicion. + if stream_name in IMMUTABLE_STREAMS: + start_window -= 300 # 5 min in epoch time, Stripe accuracy is to the second + # NB: We observed records coming through newest->oldest and so # date-windowing was added and the tap only bookmarks after it has # gotten through a date window From 50ae48a1487eb9edc5f22f4bc1e2257beb1055f6 Mon Sep 17 00:00:00 2001 From: Dan Mosora Date: Fri, 6 Dec 2019 20:12:02 +0000 Subject: [PATCH 2/3] Pull magic number into a constant --- tap_stripe/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index 53a80eb4..e081dee3 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -89,6 +89,7 @@ # NB: These streams will only sync through once for creates, never updates. IMMUTABLE_STREAMS = {'balance_transactions', 'events'} +IMMUTABLE_STREAM_LOOKBACK = 300 # 5 min in epoch time, Stripe accuracy is to the second LOGGER = singer.get_logger() @@ -440,7 +441,7 @@ def sync_stream(stream_name): # entries, but to keep the surface small, doing this only for # immutable streams at first to confirm the suspicion. if stream_name in IMMUTABLE_STREAMS: - start_window -= 300 # 5 min in epoch time, Stripe accuracy is to the second + start_window -= IMMUTABLE_STREAM_LOOKBACK # NB: We observed records coming through newest->oldest and so # date-windowing was added and the tap only bookmarks after it has From e2ac1a73b1fc79dbd91fbeecaa4bf64d44a14485 Mon Sep 17 00:00:00 2001 From: Dan Mosora Date: Fri, 6 Dec 2019 20:27:59 +0000 Subject: [PATCH 3/3] Disable TODO warning in pylint --- tap_stripe/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index e081dee3..153aeb2b 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -437,10 +437,11 @@ def sync_stream(stream_name): # observed a short lag period between when records are created and # when they are available via the API, so these streams will need # a short lookback window. - # TODO: This may be an issue for other streams' created_at - # entries, but to keep the surface small, doing this only for - # immutable streams at first to confirm the suspicion. if stream_name in IMMUTABLE_STREAMS: + # pylint:disable=fixme + # TODO: This may be an issue for other streams' created_at + # entries, but to keep the surface small, doing this only for + # immutable streams at first to confirm the suspicion. start_window -= IMMUTABLE_STREAM_LOOKBACK # NB: We observed records coming through newest->oldest and so