From 16c6a6faa54f611b0a5bac3fd8eb639b81f1bcca Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 5 Dec 2022 22:20:26 -0600 Subject: [PATCH 01/36] start of account daily overview --- models/stripe__account_daily_overview.sql | 37 +++++++++++++++++++++++ models/stripe__balance_transactions.sql | 2 ++ models/stripe__customer_overview.sql | 2 +- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 models/stripe__account_daily_overview.sql diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql new file mode 100644 index 00000000..d53cb5db --- /dev/null +++ b/models/stripe__account_daily_overview.sql @@ -0,0 +1,37 @@ +with + +account as ( + + select * + from {{ var('account') }} + +), + +account_rolling_totals as ( + + select * + from + +), balance_transaction_joined as ( + + select * + from {{ ref('stripe__balance_transactions') }} + +), + +select + account.account_id, + account_rolling_totals.date_day, + {{ dbt_utils.surrogate_key(['account.account_id','account_rolling_totals.date_day']) }} as account_daily_id, + reporting_category, + sum(amount) + + + group by 1, 2, 3, 4 + + from account + left join account_rolling_totals + on account + left join balance_transaction_joined + on account.account_id = balance_transaction_id.connected_account_id + diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index 4073bdd3..f4bf05b5 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -72,6 +72,7 @@ select case when balance_transaction.type = 'charge' then charge.amount end as customer_facing_amount, case when balance_transaction.type = 'charge' then charge.currency end as customer_facing_currency, {{ dbt.dateadd('day', 1, 'balance_transaction.available_on') }} as effective_at, + balance_transaction.connected_account_id, coalesce(charge.customer_id, refund_charge.customer_id) as customer_id, charge.receipt_email, customer.description as customer_description, @@ -85,6 +86,7 @@ select charge.charge_id, charge.payment_intent_id, charge.created_at as charge_created_at, + cards.account_id, cards.brand as card_brand, cards.funding as card_funding, cards.country as card_country, diff --git a/models/stripe__customer_overview.sql b/models/stripe__customer_overview.sql index ab1461c9..63ff9e54 100644 --- a/models/stripe__customer_overview.sql +++ b/models/stripe__customer_overview.sql @@ -14,7 +14,7 @@ with balance_transaction_joined as ( from {{ var('customer') }} ), transactions_grouped as ( - + select customer_id, sum(case when type in ('charge', 'payment') From 142c01b9662512e1ed59c4300b5ebd71a8264853 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 5 Dec 2022 22:25:37 -0600 Subject: [PATCH 02/36] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af895c1..a8913310 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# dbt_stripe v0.9.0 + +FINISH +[#55](https://github.com/fivetran/dbt_stripe/pull/55) # dbt_stripe v0.8.0 ## 🚨 Breaking Changes 🚨: From 2e705ef3877f0704e19a19ac45a81e18b42c75d3 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 6 Dec 2022 11:27:09 -0600 Subject: [PATCH 03/36] date spine --- .../intermediate/int_stripe__date_spine.sql | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 models/intermediate/int_stripe__date_spine.sql diff --git a/models/intermediate/int_stripe__date_spine.sql b/models/intermediate/int_stripe__date_spine.sql new file mode 100644 index 00000000..a296a505 --- /dev/null +++ b/models/intermediate/int_stripe__date_spine.sql @@ -0,0 +1,91 @@ + +with spine as ( + + {% if execute %} + {% set first_date_query %} + select min( created_at ) as min_date from {{ ref('stripe__balance_transactions') }} + {% endset %} + {% set first_date = run_query(first_date_query).columns[0][0]|string %} + + {% if target.type == 'postgres' %} + {% set first_date_adjust = "cast('" ~ first_date[0:10] ~ "' as date)" %} + + {% else %} + {% set first_date_adjust = "'" ~ first_date[0:10] ~ "'" %} + + {% endif %} + + {% else %} {% set first_date_adjust = "'2009-01-01'" %} + {% endif %} + + {% if execute %} + {% set last_date_query %} + select max( created_at ) as max_date from {{ ref('stripe__balance_transactions') }} + {% endset %} + + {% set current_date_query %} + select current_date + {% endset %} + + {% if run_query(current_date_query).columns[0][0]|string < run_query(last_date_query).columns[0][0]|string %} + + {% set last_date = run_query(last_date_query).columns[0][0]|string %} + + {% else %} {% set last_date = run_query(current_date_query).columns[0][0]|string %} + {% endif %} + + {% if target.type == 'postgres' %} + {% set last_date_adjust = "cast('" ~ last_date[0:10] ~ "' as date)" %} + + {% else %} + {% set last_date_adjust = "'" ~ last_date[0:10] ~ "'" %} + + {% endif %} + {% endif %} + + {{ dbt_utils.date_spine( + datepart="day", + start_date=first_date_adjust, + end_date=dbt_utils.dateadd("day", 1, last_date_adjust) + ) + }} +), + +balance_transactions as ( + + select * + from {{ ref('stripe__balance_transactions') }} +), + +account as ( + + select * + from {{ var('account') }} +), + +date_spine as ( + + select + cast({{ dbt_utils.date_trunc("day", "date_day") }} as date) as date_day, + cast({{ dbt_utils.date_trunc("week", "date_day") }} as date) as date_week, + cast({{ dbt_utils.date_trunc("month", "date_day") }} as date) as date_month, + cast({{ dbt_utils.date_trunc("year", "date_day") }} as date) as date_year, + row_number() over (order by cast({{ dbt_utils.date_trunc("day", "date_day") }} as date)) as date_index + from spine +), + +final as ( + + select distinct + account.account_id, + date_spine.date_day, + date_spine.date_week, + date_spine.date_month, + date_spine.date_year, + date_spine.date_index + from account + cross join date_spine +) + +select * +from final \ No newline at end of file From e85db81c1c3eba20061ba055c0dacd552c9848a3 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 6 Dec 2022 12:03:39 -0600 Subject: [PATCH 04/36] update deps --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index 08e0f0e9..852a78c1 100644 --- a/packages.yml +++ b/packages.yml @@ -2,5 +2,5 @@ packages: # - package: fivetran/stripe_source # version: [">=0.8.0", "<0.9.0"] - git: https://github.com/fivetran/dbt_stripe_source.git - revision: MagicBot/dbt-utils-cross-db-migration + revision: stripe_updates warn-unpinned: false From 355b579309b8f19bbbd3d0ca33c7923461cd567a Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 6 Dec 2022 23:00:53 -0600 Subject: [PATCH 05/36] more account overview updates --- .../int_stripe__account_daily.sql | 24 +++++ .../int_stripe__account_totals.sql | 95 +++++++++++++++++++ models/stripe.yml | 2 + models/stripe__account_daily_overview.sql | 2 +- models/stripe__balance_transactions.sql | 5 +- 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 models/intermediate/int_stripe__account_daily.sql create mode 100644 models/intermediate/int_stripe__account_totals.sql diff --git a/models/intermediate/int_stripe__account_daily.sql b/models/intermediate/int_stripe__account_daily.sql new file mode 100644 index 00000000..d93bb53b --- /dev/null +++ b/models/intermediate/int_stripe__account_daily.sql @@ -0,0 +1,24 @@ +with balance_transaction as ( + + select * + from {{ ref('stripe__balance_transactions') }} +), + +final as ( + + select + account_id, + type, + reporting_category, + cast( {{dbt.date_trunc("day", "created_at") }} as date) as date_day, + count(distinct balance_transaction_id) as daily_transaction_count, + sum(amount) as daily_amount, + sum(fee) as daily_fee_amount, + sum(net) as dailt_net_amount, + from balance_transaction + {{ dbt_utils.group_by(4) }} + +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/int_stripe__account_totals.sql b/models/intermediate/int_stripe__account_totals.sql new file mode 100644 index 00000000..cbdeab20 --- /dev/null +++ b/models/intermediate/int_stripe__account_totals.sql @@ -0,0 +1,95 @@ +{% set fields = ['rolling_daily_amount','rolling_daily_charge_amount','rolling_daily_refund_amount','rolling_daily_payout_reversal_amount','rolling_daily_transfer_count','rolling_daily_transfer_reversal_amount','rolling_daily_other_amount'] + +with date_spine as ( + + select * + from {{ ref('int_stripe__date_spine') }} +), + +account_balances as ( + + select * + from {{ ref('int_stripe__account_daily') }} +), + +account_daily_totals_by_category as ( + + select + account_id, + date_day, + sum(amount) as daily_amount, + sum(case when lower(reporting_category) = 'charge' + then amount + else 0 + end) as daily_charge_amount, + sum(case when lower(reporting_category) = 'refund' + then amount + else 0 + end) as daily_refund_amount, + sum(case when lower(reporting_category) = 'payout_reversal' + then amount + else 0 + end) as daily_payout_reversal_amount, + sum(case when lower(reporting_category) = 'transfer' + then amount + else 0 + end) as daily_transfer_count, + sum(case when lower(reporting_category) = 'transfer_reversal' + then amount + else 0 + end) as daily_transfer_reversal_amount + sum(case when lower(reporting_category) not in ('charge','refund','payout_reversal','transfer','transfer_reversal') + then amount + else 0 + end) as daily_other_amount + from transactions_grouped + group by 1, 2 +), + +account_rolling_totals as ( + + select + *, + sum(daily_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_amount, + sum(daily_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_charge_amount, + sum(daily_refund_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_refund_amount, + sum(daily_payout_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_payout_reversal_amount, + sum(daily_transfer_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_count, + sum(daily_transfer_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_reversal_amount, + sum(daily_other_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_other_amount + from account_daily_totals_by_category +), + +final as ( + + select + coalesce(account_rolling_totals.account_id, balance_transaction_periods.account_id) as account_id, + coalesce(account_rolling_totals.date_day, balance_transaction_periods.date_day) as date_day, + account_rolling_totals.daily_amount, + account_rolling_totals.daily_charge_amount, + account_rolling_totals.daily_refund_amount, + account_rolling_totals.daily_payout_reversal_amount, + account_rolling_totals.daily_transfer_count, + account_rolling_totals.daily_transfer_reversal_amount, + account_rolling_totals.daily_other_amount, + + {% for f in fields %} + case when account_rolling_totals.{{ f }} is null and date_index = 1 + then 0 + else account_rolling_totals.{{ f }} + end as {{ f }}, + {% endfor %} + + balance_transaction_periods.date_index + from balance_transaction_periods + + left join account_rolling_totals + on account_rolling_totals.account_id = balance_transaction_periods.account_id + and account_rolling_totals.date_day = balance_transaction_periods.date_day + and account_rolling_totals.date_week = balance_transaction_periods.date_week + and account_rolling_totals.date_month = balance_transaction_periods.date_month + and account_rolling_totals.date_year = balance_transaction_periods.date_year +) + +select * +from final diff --git a/models/stripe.yml b/models/stripe.yml index b95c2bc2..1a490958 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -20,6 +20,8 @@ models: description: Fees (in cents) paid for this transaction. - name: net description: Net amount of the transaction, in cents. + - name: account_id + description: The ID of the account tied to the transaction. - name: type description: The type of transaction. Possible values are adjustment, advance, advance_funding, application_fee, application_fee_refund, charge, connect_collection_transfer, issuing_authorization_hold, issuing_authorization_release, issuing_dispute, issuing_transaction, payment, payment_failure_refund, payment_refund, payout, payout_cancel, payout_failure, refund, refund_failure, reserve_transaction, reserved_funds, stripe_fee, stripe_fx_fee, tax_fee, topup, topup_reversal, transfer, transfer_cancel, transfer_failure, or transfer_refund. - name: reporting_category diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index d53cb5db..53da2ab3 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -22,7 +22,7 @@ account_rolling_totals as ( select account.account_id, account_rolling_totals.date_day, - {{ dbt_utils.surrogate_key(['account.account_id','account_rolling_totals.date_day']) }} as account_daily_id, + {{ dbt_utils.generate_surrogate_key(['account.account_id','account_rolling_totals.date_day']) }} as account_daily_id, reporting_category, sum(amount) diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index f4bf05b5..7f3ccdf7 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -72,7 +72,7 @@ select case when balance_transaction.type = 'charge' then charge.amount end as customer_facing_amount, case when balance_transaction.type = 'charge' then charge.currency end as customer_facing_currency, {{ dbt.dateadd('day', 1, 'balance_transaction.available_on') }} as effective_at, - balance_transaction.connected_account_id, + coalesce(balance_transaction.connected_account_id, cards.account_id) as account_id, coalesce(charge.customer_id, refund_charge.customer_id) as customer_id, charge.receipt_email, customer.description as customer_description, @@ -86,7 +86,6 @@ select charge.charge_id, charge.payment_intent_id, charge.created_at as charge_created_at, - cards.account_id, cards.brand as card_brand, cards.funding as card_funding, cards.country as card_country, @@ -98,7 +97,7 @@ select refund.reason as refund_reason from balance_transaction -left join charge +left join charge on charge.balance_transaction_id = balance_transaction.balance_transaction_id left join customer on charge.customer_id = customer.customer_id From 1f3c7aa59d168ca46d3d36e19ebc85ac5e2323e1 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 7 Dec 2022 15:18:24 -0600 Subject: [PATCH 06/36] more updates for account overview --- .../int_stripe__account_daily.sql | 127 +++++++++++++++-- .../int_stripe__account_totals.sql | 95 ------------- .../int_stripe__incomplete_charges.sql | 1 + models/stripe.yml | 3 + models/stripe__account_daily_overview.sql | 132 +++++++++++++++--- models/stripe__balance_transactions.sql | 2 +- 6 files changed, 227 insertions(+), 133 deletions(-) delete mode 100644 models/intermediate/int_stripe__account_totals.sql diff --git a/models/intermediate/int_stripe__account_daily.sql b/models/intermediate/int_stripe__account_daily.sql index d93bb53b..5c2071ff 100644 --- a/models/intermediate/int_stripe__account_daily.sql +++ b/models/intermediate/int_stripe__account_daily.sql @@ -1,24 +1,121 @@ -with balance_transaction as ( +with date_spine as ( - select * + select * + from {{ ref('int_stripe__date_spine') }} + +), balance_transaction as ( + + select *, + case + when type = 'payout' + then {{ date_timezone('available_on') }} + else {{ date_timezone('created_at') }} + end as date from {{ ref('stripe__balance_transactions') }} -), -final as ( +), incomplete_charges as ( + + select * + from {{ ref('int_stripe__incomplete_charges') }} + +), daily_account_balance_transactions as ( select - account_id, - type, - reporting_category, - cast( {{dbt.date_trunc("day", "created_at") }} as date) as date_day, - count(distinct balance_transaction_id) as daily_transaction_count, - sum(amount) as daily_amount, - sum(fee) as daily_fee_amount, - sum(net) as dailt_net_amount, + date_spine.date_day, + balance_transaction.account_id, + sum(case when balance_transaction.type in ('charge', 'payment') + then balance_transaction.amount + else 0 end) as total_daily_sales_amount, + sum(case when balance_transaction.type in ('payment_refund', 'refund') + then balance_transaction.amount + else 0 end) as total_daily_refunds_amount, + sum(case when balance_transaction.type = 'adjustment' + then balance_transaction.amount + else 0 end) as total_daily_adjustments_amount, + sum(case when balance_transaction.type not in ('charge', 'payment', 'payment_refund', 'refund', 'adjustment', 'payout') and balance_transaction.type not like '%transfer%' + then balance_transaction.amount + else 0 end) as total_daily_other_transactions_amount, + sum(case when balance_transaction.type <> 'payout' and balance_transaction.type not like '%transfer%' + then balance_transaction.amount + else 0 end) as total_daily_gross_transaction_amount, + sum(case when balance_transaction.type <> 'payout' and balance_transaction.type not like '%transfer%' + then net + else 0 end) as total_daily_net_transactions_amount, + sum(case when balance_transaction.type = 'payout' or balance_transaction.type like '%transfer%' + then fee * -1.0 + else 0 end) as total_daily_payout_fee_amount, + sum(case when balance_transaction.type = 'payout' or balance_transaction.type like '%transfer%' + then balance_transaction.amount + else 0 end) as total_daily_gross_payout_amount, + sum(case when balance_transaction.type = 'payout' or balance_transaction.type like '%transfer%' + then fee * -1.0 + else net end) as daily_net_activity_amount, + sum(case when balance_transaction.type in ('payment', 'charge') + then 1 + else 0 end) as total_daily_sales_count, + sum(case when balance_transaction.type = 'payout' + then 1 + else 0 end) as total_daily_payouts_count, + count(distinct case when balance_transaction.type = 'adjustment' + then coalesce(source, payout_id) + else null end) as total_daily_adjustments_count from balance_transaction - {{ dbt_utils.group_by(4) }} + left join date_spine + on balance_transaction.account_id = date_spine.account_id + and balance_transaction.date = date_spine.date_day + group by 1, 2 +), daily_failed_charges as ( + + select + {{ date_timezone('created_at') }} as date, + connected_account_id, + count(*) as total_daily_failed_charge_count, + sum(amount) as total_daily_failed_charge_amount + from incomplete_charges + group by 1, 2 ) -select * -from final \ No newline at end of file +select + daily_account_balance_transactions.date_day, + daily_account_balance_transactions.total_daily_sales_amount/100.0 as total_daily_sales_amount, + daily_account_balance_transactions.total_daily_refunds_amount/100.0 as total_daily_refunds_amount, + daily_account_balance_transactions.total_daily_adjustments_amount/100.0 as total_daily_adjustments_amount, + daily_account_balance_transactions.total_daily_other_transactions_amount/100.0 as total_daily_other_transactions_amount, + daily_account_balance_transactions.total_daily_gross_transaction_amount/100.0 as total_daily_gross_transaction_amount, + daily_account_balance_transactions.total_daily_net_transactions_amount/100.0 as total_daily_net_transactions_amount, + daily_account_balance_transactions.total_daily_payout_fee_amount/100.0 as total_daily_payout_fee_amount, + daily_account_balance_transactions.total_daily_gross_payout_amount/100.0 as total_daily_gross_payout_amount, + daily_account_balance_transactions.daily_net_activity_amount/100.0 as daily_net_activity_amount, + (daily_account_balance_transactions.daily_net_activity_amount + daily_account_balance_transactions.total_daily_gross_payout_amount)/100.0 as daily_end_balance_amount, + daily_account_balance_transactions.total_daily_sales_count, + daily_account_balance_transactions.total_daily_payouts_count, + daily_account_balance_transactions.total_daily_adjustments_count, + coalesce(daily_failed_charges.total_daily_failed_charge_count, 0) as total_daily_failed_charge_count, + coalesce(daily_failed_charges.total_daily_failed_charge_amount/100, 0) as total_daily_failed_charge_amount +from daily_account_balance_transactions +left join daily_failed_charges + on daily_account_balance_transactions.date_day = daily_failed_charges.date + and daily_account_balance_transactions.account_id = daily_failed_charges.connected_account_id + + + + +-- final as ( + +-- select +-- account_id, +-- balance_transaction.type, +-- reporting_category, +-- cast( {{dbt.date_trunc("day", "created_at") }} as date) as date_day, +-- count(distinct balance_transaction_id) as daily_transaction_count, +-- sum(amount) as daily_amount, +-- sum(fee) as daily_fee_amount, +-- sum(net) as dailt_net_amount, +-- from balance_transaction +-- {{ dbt_utils.group_by(4) }} + +-- ) + +-- select * +-- from final \ No newline at end of file diff --git a/models/intermediate/int_stripe__account_totals.sql b/models/intermediate/int_stripe__account_totals.sql deleted file mode 100644 index cbdeab20..00000000 --- a/models/intermediate/int_stripe__account_totals.sql +++ /dev/null @@ -1,95 +0,0 @@ -{% set fields = ['rolling_daily_amount','rolling_daily_charge_amount','rolling_daily_refund_amount','rolling_daily_payout_reversal_amount','rolling_daily_transfer_count','rolling_daily_transfer_reversal_amount','rolling_daily_other_amount'] - -with date_spine as ( - - select * - from {{ ref('int_stripe__date_spine') }} -), - -account_balances as ( - - select * - from {{ ref('int_stripe__account_daily') }} -), - -account_daily_totals_by_category as ( - - select - account_id, - date_day, - sum(amount) as daily_amount, - sum(case when lower(reporting_category) = 'charge' - then amount - else 0 - end) as daily_charge_amount, - sum(case when lower(reporting_category) = 'refund' - then amount - else 0 - end) as daily_refund_amount, - sum(case when lower(reporting_category) = 'payout_reversal' - then amount - else 0 - end) as daily_payout_reversal_amount, - sum(case when lower(reporting_category) = 'transfer' - then amount - else 0 - end) as daily_transfer_count, - sum(case when lower(reporting_category) = 'transfer_reversal' - then amount - else 0 - end) as daily_transfer_reversal_amount - sum(case when lower(reporting_category) not in ('charge','refund','payout_reversal','transfer','transfer_reversal') - then amount - else 0 - end) as daily_other_amount - from transactions_grouped - group by 1, 2 -), - -account_rolling_totals as ( - - select - *, - sum(daily_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_amount, - sum(daily_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_charge_amount, - sum(daily_refund_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_refund_amount, - sum(daily_payout_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_payout_reversal_amount, - sum(daily_transfer_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_count, - sum(daily_transfer_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_reversal_amount, - sum(daily_other_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_other_amount - from account_daily_totals_by_category -), - -final as ( - - select - coalesce(account_rolling_totals.account_id, balance_transaction_periods.account_id) as account_id, - coalesce(account_rolling_totals.date_day, balance_transaction_periods.date_day) as date_day, - account_rolling_totals.daily_amount, - account_rolling_totals.daily_charge_amount, - account_rolling_totals.daily_refund_amount, - account_rolling_totals.daily_payout_reversal_amount, - account_rolling_totals.daily_transfer_count, - account_rolling_totals.daily_transfer_reversal_amount, - account_rolling_totals.daily_other_amount, - - {% for f in fields %} - case when account_rolling_totals.{{ f }} is null and date_index = 1 - then 0 - else account_rolling_totals.{{ f }} - end as {{ f }}, - {% endfor %} - - balance_transaction_periods.date_index - from balance_transaction_periods - - left join account_rolling_totals - on account_rolling_totals.account_id = balance_transaction_periods.account_id - and account_rolling_totals.date_day = balance_transaction_periods.date_day - and account_rolling_totals.date_week = balance_transaction_periods.date_week - and account_rolling_totals.date_month = balance_transaction_periods.date_month - and account_rolling_totals.date_year = balance_transaction_periods.date_year -) - -select * -from final diff --git a/models/intermediate/int_stripe__incomplete_charges.sql b/models/intermediate/int_stripe__incomplete_charges.sql index 7713e914..0a37c575 100644 --- a/models/intermediate/int_stripe__incomplete_charges.sql +++ b/models/intermediate/int_stripe__incomplete_charges.sql @@ -8,6 +8,7 @@ with charge as ( select created_at, customer_id, + connected_account_id, amount from charge where not is_captured diff --git a/models/stripe.yml b/models/stripe.yml index 1a490958..9164481a 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -500,3 +500,6 @@ models: description: The average sum of the line items for invoices generated for this subscription - name: avg_num_invoice_items description: The average number of line items for invoices generated for this subscription + +- name: stripe__account_daily_overview + description: Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index 53da2ab3..3dd8895e 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -1,37 +1,125 @@ -with +-- {% set fields = ['rolling_daily_amount','rolling_daily_charge_amount','rolling_daily_refund_amount','rolling_daily_payout_reversal_amount','rolling_daily_transfer_count','rolling_daily_transfer_reversal_amount','rolling_daily_other_amount'] %} -account as ( +{% set fields = ['rolling_total_daily_sales_amount', 'rolling_total_daily_refunds_amount', 'rolling_total_daily_adjustments_amount', 'rolling_total_daily_other_transactions_amount', 'rolling_total_daily_gross_transaction_amount', 'rolling_total_daily_net_transactions_amount', 'rolling_total_daily_payout_fee_amount', 'rolling_total_daily_gross_payout_amount', 'rolling_daily_net_activity_amount', 'rolling_daily_end_balance_amount', 'rolling_total_daily_sales_count', 'rolling_total_daily_payouts_count', 'rolling_total_daily_adjustments_count', 'rolling_total_daily_failed_charge_count', 'rolling_total_daily_failed_charge_amount'] %} - select * - from {{ var('account') }} +with date_spine as ( + select * + from {{ ref('int_stripe__date_spine') }} ), -account_rolling_totals as ( +account_daily_balances_by_type as ( - select * - from + select * + from {{ ref('int_stripe__account_daily')}} +), + -- select + -- account_id, + -- date_day, + -- sum(amount) as daily_amount, + -- sum(case when lower(reporting_category) = 'charge' + -- then amount + -- else 0 + -- end) as daily_charge_amount, + -- sum(case when lower(reporting_category) = 'refund' + -- then amount + -- else 0 + -- end) as daily_refund_amount, + -- sum(case when lower(reporting_category) = 'payout_reversal' + -- then amount + -- else 0 + -- end) as daily_payout_reversal_amount, + -- sum(case when lower(reporting_category) = 'transfer' + -- then amount + -- else 0 + -- end) as daily_transfer_count, + -- sum(case when lower(reporting_category) = 'transfer_reversal' + -- then amount + -- else 0 + -- end) as daily_transfer_reversal_amount + -- sum(case when lower(reporting_category) not in ('charge','refund','payout_reversal','transfer','transfer_reversal') + -- then amount + -- else 0 + -- end) as daily_other_amount + -- from transactions_grouped + -- group by 1, 2 +-- ), -), balance_transaction_joined as ( +account_rolling_totals as ( - select * - from {{ ref('stripe__balance_transactions') }} + select + *, + sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_amount, + sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_refunds_amount, + sum(total_daily_adjustments_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_amount, + sum(total_daily_other_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_other_transactions_amount, + sum(total_daily_gross_transaction_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_transaction_amount, + sum(total_daily_net_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_net_transactions_amount, + sum(total_daily_payout_fee_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payout_fee_amount, + sum(total_daily_gross_payout_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_payout_amount, + sum(daily_net_activity_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_net_activity_amount, + sum(daily_end_balance_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_end_balance_amount, + sum(total_daily_sales_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_count, + sum(total_daily_payouts_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payouts_count, + sum(total_daily_adjustments_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_count, + sum(total_daily_failed_charge_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_count, + sum(total_daily_failed_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_amount + + -- sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_amount, + -- sum(daily_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_charge_amount, + -- sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_refund_amount, + -- sum(daily_payout_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_payout_reversal_amount, + -- sum(daily_transfer_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_count, + -- sum(daily_transfer_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_reversal_amount, + -- sum(daily_other_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_other_amount + + from account_daily_balances_by_type ), -select - account.account_id, - account_rolling_totals.date_day, - {{ dbt_utils.generate_surrogate_key(['account.account_id','account_rolling_totals.date_day']) }} as account_daily_id, - reporting_category, - sum(amount) +final as ( + select + coalesce(account_rolling_totals.account_id, balance_transaction_periods.account_id) as account_id, + coalesce(account_rolling_totals.date_day, balance_transaction_periods.date_day) as date_day, + account_rolling_totals.total_daily_sales_amount, + account_rolling_totals.total_daily_refunds_amount, + account_rolling_totals.total_daily_adjustments_amount, + account_rolling_totals.total_daily_other_transactions_amount, + account_rolling_totals.total_daily_gross_transaction_amount, + account_rolling_totals.total_daily_net_transactions_amount, + account_rolling_totals.total_daily_payout_fee_amount, + account_rolling_totals.total_daily_gross_payout_amount, + account_rolling_totals.daily_net_activity_amount, + account_rolling_totals.daily_end_balance_amount, + account_rolling_totals.total_daily_sales_count, + account_rolling_totals.total_daily_payouts_count, + account_rolling_totals.total_daily_adjustments_count, + account_rolling_totals.total_daily_failed_charge_count, + account_rolling_totals.total_daily_failed_charge_amount, + + -- account_rolling_totals.daily_amount, + -- account_rolling_totals.daily_charge_amount, + -- account_rolling_totals.daily_refund_amount, + -- account_rolling_totals.daily_payout_reversal_amount, + -- account_rolling_totals.daily_transfer_count, + -- account_rolling_totals.daily_transfer_reversal_amount, + -- account_rolling_totals.daily_other_amount, + + {% for f in fields %} + case when account_rolling_totals.{{ f }} is null and date_index = 1 + then 0 + else account_rolling_totals.{{ f }} + end as {{ f }}, + {% endfor %} + + balance_transaction_periods.date_index + from balance_transaction_periods - group by 1, 2, 3, 4 - - from account left join account_rolling_totals - on account - left join balance_transaction_joined - on account.account_id = balance_transaction_id.connected_account_id + on account_rolling_totals.account_id = balance_transaction_periods.account_id + and account_rolling_totals.date_day = balance_transaction_periods.date_day +) +select * +from final diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index 7f3ccdf7..3b1011b3 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -72,7 +72,7 @@ select case when balance_transaction.type = 'charge' then charge.amount end as customer_facing_amount, case when balance_transaction.type = 'charge' then charge.currency end as customer_facing_currency, {{ dbt.dateadd('day', 1, 'balance_transaction.available_on') }} as effective_at, - coalesce(balance_transaction.connected_account_id, cards.account_id) as account_id, + coalesce(balance_transaction.connected_account_id, charge.connected_account_id) as account_id, coalesce(charge.customer_id, refund_charge.customer_id) as customer_id, charge.receipt_email, customer.description as customer_description, From bc9f55506326cf7744934cded2e26336d5b3852a Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 8 Dec 2022 11:08:24 -0600 Subject: [PATCH 07/36] yml fix --- models/stripe.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index 9164481a..6e1e65b2 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -501,5 +501,5 @@ models: - name: avg_num_invoice_items description: The average number of line items for invoices generated for this subscription -- name: stripe__account_daily_overview - description: Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). + - name: stripe__account_daily_overview + description: Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). From fb473e94a7b1d8a45ad3c3df9ba8463e998ad014 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 8 Dec 2022 17:04:11 -0500 Subject: [PATCH 08/36] invoice details model --- dbt_project.yml | 3 +- macros/does_table_exist.sql | 17 +++ .../int_stripe__account_daily.sql | 5 +- .../intermediate/int_stripe__date_spine.sql | 12 +- models/stripe__account_daily_overview.sql | 12 +- models/stripe__invoice_details.sql | 136 ++++++++++++++++++ models/stripe__invoice_line_items.sql | 96 ------------- models/stripe__subscription_line_items.sql | 2 +- 8 files changed, 169 insertions(+), 114 deletions(-) create mode 100644 macros/does_table_exist.sql create mode 100644 models/stripe__invoice_details.sql delete mode 100644 models/stripe__invoice_line_items.sql diff --git a/dbt_project.yml b/dbt_project.yml index 1ec64e8b..f0c9d3d2 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -11,6 +11,7 @@ models: +materialized: ephemeral vars: stripe: + account: "{{ ref('stg_stripe__account') }}" balance_transaction: "{{ ref('stg_stripe__balance_transaction') }}" card: "{{ ref('stg_stripe__card') }}" charge: "{{ ref('stg_stripe__charge') }}" @@ -22,7 +23,7 @@ vars: payment_method_card: "{{ ref('stg_stripe__payment_method_card') }}" payment_method: "{{ ref('stg_stripe__payment_method') }}" payout: "{{ ref('stg_stripe__payout') }}" - plan: "{{ ref('stg_stripe__plan') }}" + pricing: "{{ ref('stg_stripe__pricing') }}" refund: "{{ ref('stg_stripe__refund') }}" subscription: "{{ ref('stg_stripe__subscription') }}" using_invoices: true diff --git a/macros/does_table_exist.sql b/macros/does_table_exist.sql new file mode 100644 index 00000000..c219f688 --- /dev/null +++ b/macros/does_table_exist.sql @@ -0,0 +1,17 @@ +{%- macro does_table_exist(table_name) -%} + {%- if execute -%} -- returns true when dbt is in execute mode + {%- set ns = namespace(has_table=false) -%} -- declare boolean namespace and default value + {%- for node in graph.sources.values() -%} -- grab sources from the dictionary of nodes + -- call the database for the matching table + {%- set source_relation = adapter.get_relation( + database=node.database, + schema=node.schema, + identifier=node.identifier ) -%} + {%- if source_relation == None and node.name | lower == table_name | lower -%} + {{ return(False) }} -- return false if relation identified by the database.schema.identifier does not exist for the given table name + {%- elif source_relation != None and node.name | lower == table_name | lower -%} + {{ return(True) }} -- otherwise return True + {% endif %} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} \ No newline at end of file diff --git a/models/intermediate/int_stripe__account_daily.sql b/models/intermediate/int_stripe__account_daily.sql index 5c2071ff..a89cf83b 100644 --- a/models/intermediate/int_stripe__account_daily.sql +++ b/models/intermediate/int_stripe__account_daily.sql @@ -9,7 +9,7 @@ with date_spine as ( case when type = 'payout' then {{ date_timezone('available_on') }} - else {{ date_timezone('created_at') }} + else {{ date_timezone('created_at') }} end as date from {{ ref('stripe__balance_transactions') }} @@ -62,7 +62,7 @@ with date_spine as ( from balance_transaction left join date_spine on balance_transaction.account_id = date_spine.account_id - and balance_transaction.date = date_spine.date_day + and cast({{ dbt.date_trunc("day", "balance_transaction.date") }}) as date = date_spine.date_day group by 1, 2 ), daily_failed_charges as ( @@ -78,6 +78,7 @@ with date_spine as ( select daily_account_balance_transactions.date_day, + daily_account_balance_transactions.account_id, daily_account_balance_transactions.total_daily_sales_amount/100.0 as total_daily_sales_amount, daily_account_balance_transactions.total_daily_refunds_amount/100.0 as total_daily_refunds_amount, daily_account_balance_transactions.total_daily_adjustments_amount/100.0 as total_daily_adjustments_amount, diff --git a/models/intermediate/int_stripe__date_spine.sql b/models/intermediate/int_stripe__date_spine.sql index a296a505..2e13160b 100644 --- a/models/intermediate/int_stripe__date_spine.sql +++ b/models/intermediate/int_stripe__date_spine.sql @@ -46,7 +46,7 @@ with spine as ( {{ dbt_utils.date_spine( datepart="day", start_date=first_date_adjust, - end_date=dbt_utils.dateadd("day", 1, last_date_adjust) + end_date=dbt.dateadd("day", 1, last_date_adjust) ) }} ), @@ -66,11 +66,11 @@ account as ( date_spine as ( select - cast({{ dbt_utils.date_trunc("day", "date_day") }} as date) as date_day, - cast({{ dbt_utils.date_trunc("week", "date_day") }} as date) as date_week, - cast({{ dbt_utils.date_trunc("month", "date_day") }} as date) as date_month, - cast({{ dbt_utils.date_trunc("year", "date_day") }} as date) as date_year, - row_number() over (order by cast({{ dbt_utils.date_trunc("day", "date_day") }} as date)) as date_index + cast({{ dbt.date_trunc("day", "date_day") }} as date) as date_day, + cast({{ dbt.date_trunc("week", "date_day") }} as date) as date_week, + cast({{ dbt.date_trunc("month", "date_day") }} as date) as date_month, + cast({{ dbt.date_trunc("year", "date_day") }} as date) as date_year, + row_number() over (order by cast({{ dbt.date_trunc("day", "date_day") }} as date)) as date_index from spine ), diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index 3dd8895e..7bc6f120 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -1,18 +1,15 @@ --- {% set fields = ['rolling_daily_amount','rolling_daily_charge_amount','rolling_daily_refund_amount','rolling_daily_payout_reversal_amount','rolling_daily_transfer_count','rolling_daily_transfer_reversal_amount','rolling_daily_other_amount'] %} - {% set fields = ['rolling_total_daily_sales_amount', 'rolling_total_daily_refunds_amount', 'rolling_total_daily_adjustments_amount', 'rolling_total_daily_other_transactions_amount', 'rolling_total_daily_gross_transaction_amount', 'rolling_total_daily_net_transactions_amount', 'rolling_total_daily_payout_fee_amount', 'rolling_total_daily_gross_payout_amount', 'rolling_daily_net_activity_amount', 'rolling_daily_end_balance_amount', 'rolling_total_daily_sales_count', 'rolling_total_daily_payouts_count', 'rolling_total_daily_adjustments_count', 'rolling_total_daily_failed_charge_count', 'rolling_total_daily_failed_charge_amount'] %} with date_spine as ( select * from {{ ref('int_stripe__date_spine') }} -), -account_daily_balances_by_type as ( +), account_daily_balances_by_type as ( select * from {{ ref('int_stripe__account_daily')}} -), + -- select -- account_id, -- date_day, @@ -45,7 +42,7 @@ account_daily_balances_by_type as ( -- group by 1, 2 -- ), -account_rolling_totals as ( +), account_rolling_totals as ( select *, @@ -75,9 +72,8 @@ account_rolling_totals as ( -- sum(daily_other_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_other_amount from account_daily_balances_by_type -), -final as ( +), final as ( select coalesce(account_rolling_totals.account_id, balance_transaction_periods.account_id) as account_id, diff --git a/models/stripe__invoice_details.sql b/models/stripe__invoice_details.sql new file mode 100644 index 00000000..eea0326b --- /dev/null +++ b/models/stripe__invoice_details.sql @@ -0,0 +1,136 @@ +{{ config(enabled=var('stripe__using_invoices', True)) }} + +with invoice as ( + + select * + from {{ var('invoice') }} + +), charge as ( + + select * + from {{ var('charge') }} + +), invoice_line_item as ( + + select + invoice_id, + source_relation, + plan_id, + price_id, + count(distinct unique_id) as number_of_line_items, + sum(quantity) as total_quantity + + from {{ var('invoice_line_item') }} + group by 1,2,3,4 + +), customer as ( + + select * + from {{ var('customer') }} + +{% if var('stripe__using_subscriptions', True) %} + +), subscription as ( + + select * + from {{ var('subscription') }} + +), pricing as ( + + select * + from {{ var('pricing') }} + +{% endif %} +) + +select + invoice.invoice_id, + invoice.number, + invoice.created_at as invoice_created_at, + invoice.period_start, + invoice.period_end, + invoice.status, + invoice.due_date, + invoice.currency, + invoice.amount_due, + invoice.subtotal, + invoice.tax, + invoice.total, + invoice.amount_paid, + invoice.amount_remaining, + invoice.attempt_count, + invoice.description as invoice_memo, + invoice_line_item.number_of_line_items, + invoice_line_item.total_quantity, + charge.balance_transaction_id, + charge.amount as charge_amount, + charge.status as charge_status, + charge.connected_account_id, + charge.created_at as charge_created_at, + charge.is_refunded as charge_is_refunded, + customer.customer_id, + customer.description as customer_description, + customer.account_balance as customer_account_balance, + customer.currency as customer_currency, + customer.is_delinquent as customer_is_delinquent, + customer.email as customer_email, + + {% if var('stripe__using_subscriptions', True) %} + subscription.subscription_id, + subscription.billing as subscription_billing, + subscription.start_date_at as subscription_start_date, + subscription.ended_at as subscription_ended_at, + + {% if var('stripe__price', does_table_exist('price')) %} + pricing.price_id, + + {% else %} + pricing.plan_id, + + {% endif %} + pricing.is_active as pricing_is_active, + pricing.unit_amount as pricing_amount, + pricing.recurring_interval as pricing_interval, + pricing.recurring_interval_count as pricing_interval_count, + pricing.nickname as pricing_nickname, + pricing.product_id as pricing_product_id, + {% endif %} + + {% if var('stripe__invoice_metadata',[]) %} + {{ fivetran_utils.pivot_json_extract(string = 'metadata', list_of_properties = var('stripe__invoice_metadata')) }}, + {% endif %} + + invoice.source_relation + +from invoice + +left join invoice_line_item + on invoice.invoice_id = invoice_line_item.invoice_id + and invoice.source_relation = invoice_line_item.source_relation + +left join charge + on charge.charge_id = invoice.charge_id + and charge.invoice_id = invoice.invoice_id + and charge.source_relation = invoice.source_relation + +{% if var('stripe__using_subscriptions', True) %} +left join subscription + on invoice.subscription_id = subscription.subscription_id + and invoice.source_relation = subscription.source_relation + +left join pricing + {% if var('stripe__price', does_table_exist('price')) %} + on invoice_line_item.price_id = pricing.price_id + + {% else %} + on invoice_line_item.plan_id = pricing.plan_id + + {% endif %} + and invoice_line_item.source_relation = pricing.source_relation + +{% endif %} + + +left join customer + on invoice.customer_id = customer.customer_id + and invoice.source_relation = customer.source_relation \ No newline at end of file diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql deleted file mode 100644 index 2b189336..00000000 --- a/models/stripe__invoice_line_items.sql +++ /dev/null @@ -1,96 +0,0 @@ -{{ config(enabled=var('using_invoices', True)) }} - -with invoice as ( - - select * - from {{ var('invoice') }} - -), charge as ( - - select * - from {{ var('charge') }} - -), invoice_line_item as ( - - select * - from {{ var('invoice_line_item') }} - -), customer as ( - - select * - from {{ var('customer') }} - -{% if var('using_subscriptions', True) %} - -), subscription as ( - - select * - from {{ var('subscription') }} - -), plan as ( - - select * - from {{ var('plan') }} - -{% endif %} -) - -select - invoice.invoice_id, - invoice.number, - invoice.created_at as invoice_created_at, - invoice.status, - invoice.due_date, - invoice.currency, - invoice.amount_due, - invoice.subtotal, - invoice.tax, - invoice.total, - invoice.amount_paid, - invoice.amount_remaining, - invoice.attempt_count, - invoice.description as invoice_memo, - invoice_line_item.unique_id as invoice_line_item_id, - invoice_line_item.description as line_item_desc, - invoice_line_item.amount as line_item_amount, - invoice_line_item.quantity, - invoice_line_item.period_start, - invoice_line_item.period_end, - charge.balance_transaction_id, - charge.amount as charge_amount, - charge.status as charge_status, - charge.created_at as charge_created_at, - customer.description as customer_description, - customer.email as customer_email, - customer.customer_id - - {% if var('using_subscriptions', True) %} - ,subscription.subscription_id, - subscription.billing as subscription_billing, - subscription.start_date as subscription_start_date, - subscription.ended_at as subscription_ended_at, - plan.plan_id, - plan.is_active as plan_is_active, - plan.amount as plan_amount, - plan.plan_interval as plan_interval, - plan.interval_count as plan_interval_count, - plan.nickname as plan_nickname, - plan.product_id as plan_product_id - {% endif %} - -from invoice - -left join charge - on charge.charge_id = invoice.charge_id -left join invoice_line_item - on invoice.invoice_id = invoice_line_item.invoice_id - -{% if var('using_subscriptions', True) %} -left join subscription - on invoice_line_item.subscription_id = subscription.subscription_id -left join plan - on invoice_line_item.plan_id = plan.plan_id -{% endif %} - -left join customer - on invoice.customer_id = customer.customer_id diff --git a/models/stripe__subscription_line_items.sql b/models/stripe__subscription_line_items.sql index 997f3838..7b750558 100644 --- a/models/stripe__subscription_line_items.sql +++ b/models/stripe__subscription_line_items.sql @@ -4,7 +4,7 @@ with line_items as ( select * - from {{ ref('stripe__invoice_line_items') }} + from {{ ref('stripe__invoice_details') }} where subscription_id is not null ) From 922c448a94c64b045c7da4a4c6f3adcb588c7c1a Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sat, 10 Dec 2022 13:44:59 +0800 Subject: [PATCH 09/36] seed data type cast --- integration_tests/dbt_project.yml | 3 +++ .../int_stripe__account_daily.sql | 24 +------------------ models/stripe__subscription_details.sql | 2 +- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 263a2bd3..c7eff8c5 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -34,6 +34,9 @@ seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" +column_types: _fivetran_synced: timestamp + created: timestamp + voided_at: timestamp + canceled_at: timestamp balance_transaction_data: +column_types: created: timestamp diff --git a/models/intermediate/int_stripe__account_daily.sql b/models/intermediate/int_stripe__account_daily.sql index a89cf83b..56f7b939 100644 --- a/models/intermediate/int_stripe__account_daily.sql +++ b/models/intermediate/int_stripe__account_daily.sql @@ -97,26 +97,4 @@ select from daily_account_balance_transactions left join daily_failed_charges on daily_account_balance_transactions.date_day = daily_failed_charges.date - and daily_account_balance_transactions.account_id = daily_failed_charges.connected_account_id - - - - --- final as ( - --- select --- account_id, --- balance_transaction.type, --- reporting_category, --- cast( {{dbt.date_trunc("day", "created_at") }} as date) as date_day, --- count(distinct balance_transaction_id) as daily_transaction_count, --- sum(amount) as daily_amount, --- sum(fee) as daily_fee_amount, --- sum(net) as dailt_net_amount, --- from balance_transaction --- {{ dbt_utils.group_by(4) }} - --- ) - --- select * --- from final \ No newline at end of file + and daily_account_balance_transactions.account_id = daily_failed_charges.connected_account_id \ No newline at end of file diff --git a/models/stripe__subscription_details.sql b/models/stripe__subscription_details.sql index 215cb183..0cc7fee6 100644 --- a/models/stripe__subscription_details.sql +++ b/models/stripe__subscription_details.sql @@ -65,7 +65,7 @@ select customer.description as customer_description, customer.email as customer_email, subscription.status, - subscription.start_date, + subscription.start_date_at, subscription.ended_at, subscription.billing, subscription.billing_cycle_anchor, From 0b5932ddb4ecdf761d16f6a7ff89390f08a17e0b Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sat, 10 Dec 2022 14:04:10 +0800 Subject: [PATCH 10/36] forgot to add account and price to seed data --- integration_tests/seeds/account_data.csv | 2 ++ integration_tests/seeds/price_data.csv | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 integration_tests/seeds/account_data.csv create mode 100644 integration_tests/seeds/price_data.csv diff --git a/integration_tests/seeds/account_data.csv b/integration_tests/seeds/account_data.csv new file mode 100644 index 00000000..47917fcc --- /dev/null +++ b/integration_tests/seeds/account_data.csv @@ -0,0 +1,2 @@ +id,_fivetran_synced,business_profile_mcc,business_profile_name,business_profile_product_description,business_profile_support_address_city,business_profile_support_address_country,business_profile_support_address_line_1,business_profile_support_address_line_2,business_profile_support_address_postal_code,business_profile_support_address_state,business_profile_support_email,business_profile_support_phone,business_profile_support_url,business_profile_url,business_type,capabilities_afterpay_clearpay_payments,capabilities_au_becs_debit_payments,capabilities_bacs_debit_payments,capabilities_bancontact_payments,capabilities_card_issuing,capabilities_card_payments,capabilities_cartes_bancaires_payments,capabilities_eps_payments,capabilities_fpx_payments,capabilities_giropay_payments,capabilities_grabpay_payments,capabilities_ideal_payments,capabilities_jcb_payments,capabilities_legacy_payments,capabilities_oxxo_payments,capabilities_p_24_payments,capabilities_platform_payments,capabilities_sepa_debit_payments,capabilities_sofort_payments,capabilities_tax_reporting_us_1099_k,capabilities_tax_reporting_us_1099_misc,capabilities_transfers,charges_enabled,company_address_city,company_address_country,company_address_kana_city,company_address_kana_country,company_address_kana_line_1,company_address_kana_line_2,company_address_kana_postal_code,company_address_kana_state,company_address_kana_town,company_address_kanji_city,company_address_kanji_country,company_address_kanji_line_1,company_address_kanji_line_2,company_address_kanji_postal_code,company_address_kanji_state,company_address_kanji_town,company_address_line_1,company_address_line_2,company_address_postal_code,company_address_state,company_directors_provided,company_executives_provided,company_name,company_name_kana,company_name_kanji,company_owners_provided,company_phone,company_structure,company_tax_id_provided,company_tax_id_registrar,company_vat_id_provided,company_verification_document_back,company_verification_document_details,company_verification_document_details_code,company_verification_document_front,country,created,default_currency,details_submitted,email,individual_id,is_deleted,metadata,payouts_enabled,requirements_current_deadline,requirements_currently_due,requirements_disabled_reason,requirements_errors,requirements_eventually_due,requirements_past_due,requirements_pending_verification,settings_branding_icon,settings_branding_logo,settings_branding_primary_color,settings_card_payments_decline_on_avs_failure,settings_card_payments_decline_on_cvc_failure,settings_card_payments_statement_descriptor_prefix,settings_dashboard_display_name,settings_dashboard_timezone,settings_payments_statement_descriptor,settings_payments_statement_descriptor_kana,settings_payments_statement_descriptor_kanji,settings_payouts_debit_negative_balances,settings_payouts_schedule_delay_days,settings_payouts_schedule_interval,settings_payouts_schedule_monthly_anchor,settings_payouts_schedule_weekly_anchor,settings_payouts_statement_descriptor,tos_acceptance_date,tos_acceptance_ip,tos_acceptance_user_agent,type +1234,2022-12-06 14:19:07,737,Hogwarts School of Witchcraft and Wizardry,,Hogwarts,US,123 Street,Apt 123,12345,CA,severus@hogwarts.com,,https://www.wizardingworld.com/,https://www.wizardingworld.com/,,active,,,active,,active,,active,,active,,active,,,,active,active,active,active,,,,TRUE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,US,,usd,TRUE,dumbledore@hogwarts.com,,FALSE,,TRUE,,,,,,,,,file_asdfasgas,#0030ff,,,,Hogwarts Inc.,America/Los_Angeles,hogwarts.com,,,TRUE,7,daily,,,,,,,standard \ No newline at end of file diff --git a/integration_tests/seeds/price_data.csv b/integration_tests/seeds/price_data.csv new file mode 100644 index 00000000..2e61ef9f --- /dev/null +++ b/integration_tests/seeds/price_data.csv @@ -0,0 +1,2 @@ +id,_fivetran_synced,active,billing_scheme,created,currency,invoice_item_id,is_deleted,livemode,lookup_key,metadata,nickname,product_id,recurring_aggregate_usage,recurring_interval,recurring_interval_count,recurring_usage_type,tiers_mode,transform_quantity_divide_by,transform_quantity_round,type,unit_amount,unit_amount_decimal +price_1,2022-09-22 23:49:02,TRUE,per_unit,2020-05-02 19:21:44,usd,ii_FYFJSuzUa5YRk,FALSE,TRUE,,{},,prod_HCqb8lm6kfYk,,,,,,,,one_time,100000,100000 \ No newline at end of file From 2d76a8fee8584930a4b73dc2e6ee83a9e55bc5bf Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sat, 10 Dec 2022 14:11:30 +0800 Subject: [PATCH 11/36] update var names --- dbt_project.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index f0c9d3d2..49847cdd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe' -version: '0.8.0' +version: '0.9.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: stripe: @@ -26,6 +26,6 @@ vars: pricing: "{{ ref('stg_stripe__pricing') }}" refund: "{{ ref('stg_stripe__refund') }}" subscription: "{{ ref('stg_stripe__subscription') }}" - using_invoices: true - using_payment_method: true - using_subscriptions: true + stripe__using_invoices: true + stripe__using_payment_method: true + stripe__using_subscriptions: true From d3fcbac58666aee16d1cb201522d359344e82657 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sat, 10 Dec 2022 14:12:27 +0800 Subject: [PATCH 12/36] update var names --- .buildkite/scripts/run_models.sh | 2 +- README.md | 6 +++--- models/stripe__subscription_details.sql | 2 +- models/stripe__subscription_line_items.sql | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index e0182eba..56ac4fd8 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -19,5 +19,5 @@ dbt deps dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" -dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true}' --target "$db" +dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true, stripe__price: false}' --target "$db" dbt test --target "$db" \ No newline at end of file diff --git a/README.md b/README.md index 8dcdf695..d5b2d03e 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,9 @@ vars: This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, or `subscription` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable within your root `dbt_project.yml`: ```yml vars: - using_invoices: False #Disable if you are not using the invoice and invoice_line_item tables - using_payment_method: False #Disable if you are not using the payment_method and payment_method_card tables - using_subscriptions: False #Disable if you are not using the subscription and plan tables. + stripe__using_invoices: False #Disable if you are not using the invoice and invoice_line_item tables + stripe__using_payment_method: False #Disable if you are not using the payment_method and payment_method_card tables + stripe__using_subscriptions: False #Disable if you are not using the subscription and plan tables. ``` ## Step 5: Leveraging Subscription Vs Subscription History Sources For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription` source table. However, if you have a newer connector then you must leverage the `stripe__subscription_history` to have the package use the `subscription_history` source rather than the `subscription` table. diff --git a/models/stripe__subscription_details.sql b/models/stripe__subscription_details.sql index 0cc7fee6..fc1df9a1 100644 --- a/models/stripe__subscription_details.sql +++ b/models/stripe__subscription_details.sql @@ -1,4 +1,4 @@ -{{ config(enabled=fivetran_utils.enabled_vars(['using_invoices','using_subscriptions'])) }} +{{ config(enabled=fivetran_utils.enabled_vars(['stripe__using_invoices','stripe__using_subscriptions'])) }} with invoice as ( diff --git a/models/stripe__subscription_line_items.sql b/models/stripe__subscription_line_items.sql index 7b750558..4591efbf 100644 --- a/models/stripe__subscription_line_items.sql +++ b/models/stripe__subscription_line_items.sql @@ -1,4 +1,4 @@ -{{ config(enabled=fivetran_utils.enabled_vars(['using_invoices','using_subscriptions'])) }} +{{ config(enabled=fivetran_utils.enabled_vars(['stripe__using_invoices','stripe__using_subscriptions'])) }} with line_items as ( From abe4a87189756aca98359a1957533ed2f56052fb Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sat, 10 Dec 2022 14:21:22 +0800 Subject: [PATCH 13/36] fixes --- integration_tests/dbt_project.yml | 2 ++ integration_tests/seeds/plan_data.csv | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c7eff8c5..498f0f4b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -8,6 +8,7 @@ profile: 'integration_tests' vars: stripe_schema: stripe_integrations_tests stripe_source: + stripe_account_identifier: "account_data" stripe_group_identifier: "group_data" stripe_balance_transaction_identifier: "balance_transaction_data" stripe_card_identifier: "card_data" @@ -21,6 +22,7 @@ vars: stripe_payment_method_identifier: "payment_method_data" stripe_payout_identifier: "payout_data" stripe_plan_identifier: "plan_data" + stripe_price_identifier: "price_data" stripe_refund_identifier: "refund_data" stripe_subscription_history_identifier: "subscription_history_data" stripe_subscription_identifier: "subscription_data" diff --git a/integration_tests/seeds/plan_data.csv b/integration_tests/seeds/plan_data.csv index f79bb2f3..6b9f7e0d 100644 --- a/integration_tests/seeds/plan_data.csv +++ b/integration_tests/seeds/plan_data.csv @@ -5,7 +5,7 @@ ab2,2021-03-15 14:21:13,TRUE,,150000,per_unit,2017-05-05 16:56:44,usd,month,1,FA 3gf,2021-03-15 14:21:13,TRUE,,180000,per_unit,2017-06-15 16:01:06,usd,month,6,FALSE,TRUE,,prod_BUonomUNsdG3qx,,,,,licensed 214gfdg,2021-03-15 14:21:13,TRUE,,264030,per_unit,2017-08-04 16:29:13,usd,month,3,FALSE,TRUE,,prod_BUo1nHP2vIrVtF,,,,,licensed fgds754,2021-03-15 14:21:13,TRUE,,45000,per_unit,2017-06-21 16:23:32,usd,month,1,FALSE,TRUE,,prod_BUBtgJBLlTGcy9,,,,,licensed -dfgdw576,2021-03-15 14:21:13,TRUE,,20000,per_unit,2015-05-28 1:08:05,usd,month,1,FALSE,TRUE,,prod_BV5IFAOt2EwqSl,,,,,licensed +dfgdw576,2021-03-15 14:21:13,TRUE,,20000,per_unit,2015-05-28 01:08:05,usd,month,1,FALSE,TRUE,,prod_BV5IFAOt2EwqSl,,,,,licensed rhsdr456,2021-03-15 14:21:13,TRUE,,100,per_unit,2015-05-28 1:10:13,usd,month,1,FALSE,TRUE,,prod_BU23UCziUM7ezw,,,,,licensed 3gds,2021-03-15 14:21:13,TRUE,,50000,per_unit,2015-10-23 21:23:35,usd,month,1,FALSE,TRUE,,prod_BTXZakZKkKlf6g,,,,,licensed 234rtgh,2021-03-15 14:21:13,TRUE,,40000,per_unit,2015-12-01 21:06:02,usd,month,1,FALSE,TRUE,,prod_BUrKq3BcaxvjkU,,,,,licensed From 4082091b1db3847d6f31b72cbaf9b29f78fbdf31 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sun, 11 Dec 2022 01:38:14 +0800 Subject: [PATCH 14/36] fixes --- integration_tests/seeds/invoice_data.csv | 2 +- models/intermediate/int_stripe__account_daily.sql | 2 +- models/stripe__account_daily_overview.sql | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/seeds/invoice_data.csv b/integration_tests/seeds/invoice_data.csv index e0ff9239..3285f4b3 100644 --- a/integration_tests/seeds/invoice_data.csv +++ b/integration_tests/seeds/invoice_data.csv @@ -1,3 +1,3 @@ id,_fivetran_synced,amount,currency,customer_id,date,description,discountable,invoice_id,is_deleted,livemode,period_end,period_start,proration,quantity,subscription_id,subscription_item_id,unit_amount -sdfg54545,2021-02-18 14:16:37,45,usd,sdgfs84854,,description_here,TRUE,sdg545641,FALSE,TRUE,2021-02-19 14:16:37,2021-02-18 14:16:37,TRUE,45,sdkhg54645,kdshg455,5 +ab1111,2021-02-18 14:16:37,45,usd,sdgfs84854,,description_here,TRUE,sdg545641,FALSE,TRUE,2021-02-19 14:16:37,2021-02-18 14:16:37,TRUE,45,sdkhg54645,kdshg455,5 si8jo54,2021-02-18 14:16:37,22,usd,dfd54654,,description_here,TRUE,dgjksg6486,FALSE,FALSE,2021-02-19 14:16:37,2021-02-18 14:16:37,FALSE,55,dghisd165748,sgdfs5464,6 \ No newline at end of file diff --git a/models/intermediate/int_stripe__account_daily.sql b/models/intermediate/int_stripe__account_daily.sql index 56f7b939..f4f7f02a 100644 --- a/models/intermediate/int_stripe__account_daily.sql +++ b/models/intermediate/int_stripe__account_daily.sql @@ -62,7 +62,7 @@ with date_spine as ( from balance_transaction left join date_spine on balance_transaction.account_id = date_spine.account_id - and cast({{ dbt.date_trunc("day", "balance_transaction.date") }}) as date = date_spine.date_day + and cast({{ dbt.date_trunc('day', 'balance_transaction.date') }} as date) = date_spine.date_day group by 1, 2 ), daily_failed_charges as ( diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index 7bc6f120..7672baf9 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -76,8 +76,8 @@ with date_spine as ( ), final as ( select - coalesce(account_rolling_totals.account_id, balance_transaction_periods.account_id) as account_id, - coalesce(account_rolling_totals.date_day, balance_transaction_periods.date_day) as date_day, + coalesce(account_rolling_totals.account_id, date_spine.account_id) as account_id, + coalesce(account_rolling_totals.date_day, date_spine.date_day) as date_day, account_rolling_totals.total_daily_sales_amount, account_rolling_totals.total_daily_refunds_amount, account_rolling_totals.total_daily_adjustments_amount, @@ -109,12 +109,12 @@ with date_spine as ( end as {{ f }}, {% endfor %} - balance_transaction_periods.date_index - from balance_transaction_periods + date_spine.date_index + from date_spine left join account_rolling_totals - on account_rolling_totals.account_id = balance_transaction_periods.account_id - and account_rolling_totals.date_day = balance_transaction_periods.date_day + on account_rolling_totals.account_id = date_spine.account_id + and account_rolling_totals.date_day = date_spine.date_day ) select * From 5459ef0a7c65089fa3ba5ab90e423944f60f05c0 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sun, 11 Dec 2022 01:39:59 +0800 Subject: [PATCH 15/36] add data casts --- integration_tests/dbt_project.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 498f0f4b..5e976b24 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -39,13 +39,14 @@ seeds: created: timestamp voided_at: timestamp canceled_at: timestamp + invoice_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" balance_transaction_data: +column_types: created: timestamp available_on: timestamp charge_data: +column_types: - balance_transaction_id: "{{ 'varchar(250)' if target.type in ('redshift','postgres') else 'string'}}" + balance_transaction_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" created: timestamp customer_data: +column_types: From 04fb1affd99a13383b08f1297d1699bce61263a3 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Sun, 11 Dec 2022 01:46:03 +0800 Subject: [PATCH 16/36] rm invoice line item from sub line item model --- models/stripe.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index 6e1e65b2..47d4f645 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -371,7 +371,9 @@ models: description: Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. columns: - name: invoice_id - description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. + description: The unique Identifier of the invoice object. + tests: + - unique - name: number description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. - name: invoice_created_at @@ -396,10 +398,6 @@ models: description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. - name: invoice_memo description: An arbitrary string attached to the object. Often useful for displaying to users. - - name: invoice_line_item_id - description: The unique Identifier of the invoice line object. - tests: - - unique - name: line_item_desc description: The description of the invoice line item - name: line_item_amount From 99e81aa9581686afb45a3c771b34db2f634f5688 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 12 Dec 2022 13:34:02 +0800 Subject: [PATCH 17/36] update var names --- models/stripe__balance_transactions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index 3b1011b3..5da367a3 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -29,7 +29,7 @@ with balance_transaction as ( select * from {{ var('customer')}} -{% if var('using_payment_method', True) %} +{% if var('stripe__using_payment_method', True) %} ), payment_method as ( @@ -77,7 +77,7 @@ select charge.receipt_email, customer.description as customer_description, - {% if var('using_payment_method', True) %} + {% if var('stripe__using_payment_method', True) %} payment_method.type as payment_method_type, payment_method_card.brand as payment_method_brand, payment_method_card.funding as payment_method_funding, @@ -104,7 +104,7 @@ left join customer left join payment_intent on charge.payment_intent_id = payment_intent.payment_intent_id -{% if var('using_payment_method', True) %} +{% if var('stripe__using_payment_method', True) %} left join payment_method on payment_intent.payment_method_id = payment_method.payment_method_id left join payment_method_card From 802fb5f0ce3f81ddf183b60457f6c6e445c13ad5 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 12 Dec 2022 13:35:52 +0800 Subject: [PATCH 18/36] seed timestamp fix and update yml --- integration_tests/seeds/plan_data.csv | 2 +- models/stripe.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/seeds/plan_data.csv b/integration_tests/seeds/plan_data.csv index 6b9f7e0d..d8bb0adc 100644 --- a/integration_tests/seeds/plan_data.csv +++ b/integration_tests/seeds/plan_data.csv @@ -6,7 +6,7 @@ ab2,2021-03-15 14:21:13,TRUE,,150000,per_unit,2017-05-05 16:56:44,usd,month,1,FA 214gfdg,2021-03-15 14:21:13,TRUE,,264030,per_unit,2017-08-04 16:29:13,usd,month,3,FALSE,TRUE,,prod_BUo1nHP2vIrVtF,,,,,licensed fgds754,2021-03-15 14:21:13,TRUE,,45000,per_unit,2017-06-21 16:23:32,usd,month,1,FALSE,TRUE,,prod_BUBtgJBLlTGcy9,,,,,licensed dfgdw576,2021-03-15 14:21:13,TRUE,,20000,per_unit,2015-05-28 01:08:05,usd,month,1,FALSE,TRUE,,prod_BV5IFAOt2EwqSl,,,,,licensed -rhsdr456,2021-03-15 14:21:13,TRUE,,100,per_unit,2015-05-28 1:10:13,usd,month,1,FALSE,TRUE,,prod_BU23UCziUM7ezw,,,,,licensed +rhsdr456,2021-03-15 14:21:13,TRUE,,100,per_unit,2015-05-28 01:10:13,usd,month,1,FALSE,TRUE,,prod_BU23UCziUM7ezw,,,,,licensed 3gds,2021-03-15 14:21:13,TRUE,,50000,per_unit,2015-10-23 21:23:35,usd,month,1,FALSE,TRUE,,prod_BTXZakZKkKlf6g,,,,,licensed 234rtgh,2021-03-15 14:21:13,TRUE,,40000,per_unit,2015-12-01 21:06:02,usd,month,1,FALSE,TRUE,,prod_BUrKq3BcaxvjkU,,,,,licensed 658jfgh,2021-03-15 14:21:13,TRUE,,15000,per_unit,2015-11-16 22:42:15,usd,month,1,FALSE,TRUE,,prod_BUaE6AwT15sLSX,,,,,licensed diff --git a/models/stripe.yml b/models/stripe.yml index 47d4f645..6716d089 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -284,7 +284,8 @@ models: description: Total amount from incomplete charges during the reporting period. - - name: stripe__invoice_line_items + - name: stripe__invoice_details + # UPDATE BELOW description: Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and plan. columns: - name: invoice_id From b8c378c157a52e0f2e97457c5200e3a3dd059fb0 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 12 Dec 2022 13:40:46 +0800 Subject: [PATCH 19/36] yml --- models/stripe.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index 6716d089..e8bd6c8f 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -290,6 +290,8 @@ models: columns: - name: invoice_id description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. + tests: + - unique - name: number description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. - name: invoice_created_at @@ -316,10 +318,6 @@ models: description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. - name: invoice_memo description: An arbitrary string attached to the object. Often useful for displaying to users. - - name: invoice_line_item_id - description: The unique Identifier of the invoice line object. - tests: - - unique - name: line_item_desc description: The description of the invoice line item - name: line_item_amount From 9819c911aab3fa3100d1e9998b83766a6cd01344 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 12 Dec 2022 16:23:38 +0800 Subject: [PATCH 20/36] yml --- models/stripe.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index e8bd6c8f..c0ab3e30 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -214,7 +214,7 @@ models: description: Number of incomplete charges during the reporting period. - name: total_failed_charge_amount description: Total amount from incomplete charges during the reporting period. - + - name: stripe__monthly_overview description: Each record represents a single month, enriched with metrics about balances, payments, refunds, payouts, and other transactions. columns: @@ -285,10 +285,9 @@ models: - name: stripe__invoice_details - # UPDATE BELOW - description: Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and plan. + description: Each record represents an invoice, enriched with details about the associated charge, customer, subscription, and pricing. columns: - - name: invoice_id + - name: invoice_id description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. tests: - unique @@ -351,19 +350,21 @@ models: - name: customer_id description: The associated customer reference. - name: plan_id - description: The ID of the plan of the subscription, if the line item is a subscription or a proration. - - name: plan_is_active - description: Boolean indicating if the plan is active (true) or in-active (false). - - name: plan_amount + description: The ID of the plan of the subscription, if you are using the Plan API and the invoice is a subscription or a proration. + - name: price_id + description: The ID of the price of the subscription, if you are using the Price API. + - name: pricing_is_active + description: Boolean indicating if the plan or price is active (true) or in-active (false). + - name: pricing_amount description: The unit amount in cents to be charged, represented as a whole integer if possible. - - name: plan_interval + - name: pricing_interval description: The frequency at which a subscription is billed. One of day, week, month or year. - - name: plan_interval_count + - name: pricing_interval_count description: The number of intervals between subscription billings. For example, interval_count=3 bills every 3 months. - - name: plan_nickname - description: A brief description of the plan, hidden from customers. - - name: plan_product_id - description: Reference to the product whose pricing this plan determines. + - name: pricing_nickname + description: A brief description of the plan or price, hidden from customers. + - name: pricing_product_id + description: Reference to the product whose pricing this price or plan determines. - name: stripe__subscription_line_items From 501af98346774d04ad1291d55b710f248e5131af Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 12 Dec 2022 23:25:55 +0800 Subject: [PATCH 21/36] rm pricing bc it's at the invoice line item level --- models/stripe__invoice_details.sql | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/models/stripe__invoice_details.sql b/models/stripe__invoice_details.sql index eea0326b..78120300 100644 --- a/models/stripe__invoice_details.sql +++ b/models/stripe__invoice_details.sql @@ -53,10 +53,10 @@ select invoice.due_date, invoice.currency, invoice.amount_due, + invoice.amount_paid, invoice.subtotal, invoice.tax, invoice.total, - invoice.amount_paid, invoice.amount_remaining, invoice.attempt_count, invoice.description as invoice_memo, @@ -81,19 +81,6 @@ select subscription.start_date_at as subscription_start_date, subscription.ended_at as subscription_ended_at, - {% if var('stripe__price', does_table_exist('price')) %} - pricing.price_id, - - {% else %} - pricing.plan_id, - - {% endif %} - pricing.is_active as pricing_is_active, - pricing.unit_amount as pricing_amount, - pricing.recurring_interval as pricing_interval, - pricing.recurring_interval_count as pricing_interval_count, - pricing.nickname as pricing_nickname, - pricing.product_id as pricing_product_id, {% endif %} {% if var('stripe__invoice_metadata',[]) %} @@ -118,16 +105,6 @@ left join subscription on invoice.subscription_id = subscription.subscription_id and invoice.source_relation = subscription.source_relation -left join pricing - {% if var('stripe__price', does_table_exist('price')) %} - on invoice_line_item.price_id = pricing.price_id - - {% else %} - on invoice_line_item.plan_id = pricing.plan_id - - {% endif %} - and invoice_line_item.source_relation = pricing.source_relation - {% endif %} From a152f1f9f702c8bcff81cabfb70e128b958b4ae1 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Mon, 12 Dec 2022 23:26:11 +0800 Subject: [PATCH 22/36] add updated invoice line item model with pricing --- models/stripe__invoice_line_items.sql | 91 +++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 models/stripe__invoice_line_items.sql diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql new file mode 100644 index 00000000..01811a2f --- /dev/null +++ b/models/stripe__invoice_line_items.sql @@ -0,0 +1,91 @@ +{{ config(enabled=var('using_invoices', True)) }} + +with invoice_line_item as ( + + select * + from {{ var('invoice_line_item') }} + +), invoice_details as ( + + select * + from {{ ref('stripe__invoice_details') }} + +), subscription as ( + + select * + from {{ var('subscription') }} + +{% if var('using_subscriptions', True) %} +), pricing as ( + + select * + from {{ var('pricing') }} + +{% endif %} +) + +select + invoice_line_item.invoice_line_item_id, + invoice_line_item.invoice_id, + invoice_line_item.invoice_item_id, + invoice_line_item.amount as invoice_line_item_amount, + invoice_line_item.currency, + invoice_line_item.description as invoice_line_item_memo, + invoice_line_item.is_discountable, + invoice_line_item.plan_id, + invoice_line_item.price_id, + invoice_line_item.proration, + invoice_line_item.quantity, + invoice_line_item.subscription_id, + invoice_line_item.subscription_item_id, + invoice_line_item.type, + invoice_line_item.unique_id, + invoice_line_item.period_start, + invoice_line_item.period_end, + invoice_details.invoice_created_at, + invoice_details.status as invoice_status, + invoice_details.due_date as invoice_due_date, + invoice_details.amount_due as invoice_amount_due, + invoice_details.amount_paid as invoice_amount_paid, + invoice_details.subtotal as invoice_subtotal, + invoice_details.tax as invoice_tax, + invoice_details.total as invoice_total, + invoice_details.connected_account_id as connected_account_id, + invoice_details.customer_id as customer_id + + {% if var('stripe__using_subscriptions', True) %} + , + subscription.billing as subscription_billing, + subscription.start_date_at as subscription_start_date, + subscription.ended_at as subscription_ended_at, + pricing.is_active as pricing_is_active, + pricing.unit_amount as pricing_amount, + pricing.recurring_interval as pricing_interval, + pricing.recurring_interval_count as pricing_interval_count, + pricing.nickname as pricing_nickname, + pricing.product_id as pricing_product_id + {% endif %} + +from invoice_line_item + +left join invoice_details + on invoice_line_item.invoice_id = invoice_details.invoice_id + and invoice_line_item.source_relation = invoice_details.source_relation + +{% if var('stripe__using_subscriptions', True) %} + +left join subscription + on invoice_line_item.subscription_id = subscription.subscription_id + and invoice_line_item.source_relation = subscription.source_relation + +left join pricing + {% if var('stripe__price', does_table_exist('price')) %} + on invoice_line_item.price_id = pricing.price_id + + {% else %} + on invoice_line_item.plan_id = pricing.plan_id + + {% endif %} + and invoice_line_item.source_relation = pricing.source_relation + +{% endif %} \ No newline at end of file From dbdeb4417d197c5c03da439a359fa2517316b761 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 00:56:36 +0800 Subject: [PATCH 23/36] invoice details and line item yml --- models/stripe.yml | 103 ++++++++++++++++++++++++----- models/stripe__invoice_details.sql | 2 +- 2 files changed, 86 insertions(+), 19 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index c0ab3e30..c133aff2 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -285,7 +285,7 @@ models: - name: stripe__invoice_details - description: Each record represents an invoice, enriched with details about the associated charge, customer, subscription, and pricing. + description: Each record represents an invoice, enriched with details about the associated charge, customer, and subscription data. columns: - name: invoice_id description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. @@ -295,34 +295,38 @@ models: description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. - name: invoice_created_at description: Timestamp of when the invoice was created. + - name: period_start + description: Timestamp of the start of the period. + - name: period_end + description: Timestamp of the end of the period. - name: status description: Current status of the invoice. - name: due_date description: Date when payment for the invoice is due. - name: currency - description: The currency that the invoice is in. Three-letter ISO currency code, in lowercase. + description: The currency that the invoice is in. Three-letter ISO currency code, in lowercase. - name: amount_due - description: The amount that the invoice is for. Because this query includes line item details, this amount may be repeated. + description: The amount, if any, that the customer has paid on the invoice + - name: amount_paid + description: The amount, if any, that the customer has paid on the invoice + - name: amount_due + description: Final amount due at this time for this invoice. If the invoice’s total is smaller than the minimum charge amount, for example, or if there is account credit that can be applied to the invoice, the amount_due may be 0. If there is a positive starting_balance for the invoice (the customer owes money), the amount_due will also take that into account. The charge that gets generated for the invoice will be for the amount specified in amount_due. - name: subtotal description: The amount of the invoice before discounts and taxes. - name: tax description: The amount of tax being charged in the invoice. - name: total description: The Total after discounts and taxes. - - name: amount_paid - description: The amount, if any, that the customer has paid on the invoice - name: amount_remaining description: The amount of the invoice remaining to be paid. - name: attempt_count description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. - name: invoice_memo description: An arbitrary string attached to the object. Often useful for displaying to users. - - name: line_item_desc - description: The description of the invoice line item - - name: line_item_amount - description: The amount for the line item. - - name: quantity - description: The quantity of the line item. + - name: number_of_line_items + description: Number of line item records for this invoice. + - name: total_quantity + description: The total quantity of items for this invoice. - name: balance_transaction_id description: The ID of the balance transaction object representing payment - name: charge_amount @@ -331,8 +335,20 @@ models: description: The status of the charge for the invoice. - name: charge_created_at description: When the charge for the invoice was created. + - name: charge_is_refunded + description: Whether the charge has been fully refunded. If the charge is only partially refunded, this attribute will still be false. + - name: connected_account_id + description: The ID of the account connected to the charge. + - name: customer_id + description: The associated customer reference. - name: customer_description description: Description of the customer. + - name: customer_account_balance + description: Current balance, if any, being stored on the customer. If negative, the customer has credit to apply to their next invoice. If positive, the customer has an amount owed that will be added to their next invoice. + - name: customer_currency + description: Three-letter ISO code for the currency the customer can be charged in for recurring billing purposes. + - name: customer_is_delinquent + description: When the customer’s latest invoice is billed by charging automatically, delinquent is true if the invoice’s latest charge is failed. When the customer’s latest invoice is billed by sending an invoice, delinquent is true if the invoice is not paid by its due date. - name: customer_email description: Email of the customer. - name: subscription_id @@ -343,16 +359,68 @@ models: description: The start date of the subscription - name: subscription_ended_at description: The end date of the subscription + + - name: stripe__invoice_line_items + description: Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and pricing data. + columns: + - name: invoice_line_item_id + description: The unique Identifier of the invoice line object. + tests: + - unique + - name: invoice_id + description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. + - name: invoice_item_id + description: The ID of the invoice item this item is a part of + - name: invoice_line_item_amount + description: Amount for this line item. + - name: currency + description: Currency of this line item. + - name: invoice_line_item_memo + description: Description for this line item. + - name: is_discountable + description: Whether this line item is discountable. + - name: proration + description: Whether this is a proration. + - name: quantity + description: The quantity of the subscription, if the line item is a subscription or a proration. + - name: subscription_id + description: The ID of the subscription that the invoice item pertains to, if any. + - name: subscription_item_id + description: The subscription item that generated this invoice item. Left empty if the line item is not an explicit result of a subscription. + - name: type + description: A string identifying the type of the source of this line item, either an invoice item or a subscription. + - name: unique_id + description: A unique id generated for old invoice ID's from a past version of the API - name: period_start - description: Timestamp of the start of the period. + description: Start of the usage period during which invoice items were added to this invoice. - name: period_end - description: Timestamp of the end of the period. + description: End of the usage period during which invoice items were added to this invoice. + - name: invoice_created_at + description: Timestamp of when the invoice was created. + - name: invoice_status + description: Current status of the invoice. + - name: invoice_due_date + description: Date when payment for the invoice is due. + - name: invoice_amount_due + description: Final amount due at this time for this invoice. If the invoice’s total is smaller than the minimum charge amount, for example, or if there is account credit that can be applied to the invoice, the amount_due may be 0. If there is a positive starting_balance for the invoice (the customer owes money), the amount_due will also take that into account. The charge that gets generated for the invoice will be for the amount specified in amount_due. + - name: invoice_amount_paid + description: The amount, if any, that the customer has paid on the invoice + - name: invoice_subtotal + description: The amount of the invoice before discounts and taxes. + - name: invoice_tax + description: The amount of tax being charged in the invoice. + - name: invoice_total + description: The total of the invoice after discounts and taxes. + - name: connected_account_id + description: The ID of the account connected to the charge. - name: customer_id description: The associated customer reference. - - name: plan_id - description: The ID of the plan of the subscription, if you are using the Plan API and the invoice is a subscription or a proration. - - name: price_id - description: The ID of the price of the subscription, if you are using the Price API. + - name: subscription_billing + description: How the subscription is billed + - name: subscription_start_date + description: The start date of the subscription + - name: subscription_ended_at + description: The end date of the subscription - name: pricing_is_active description: Boolean indicating if the plan or price is active (true) or in-active (false). - name: pricing_amount @@ -366,7 +434,6 @@ models: - name: pricing_product_id description: Reference to the product whose pricing this price or plan determines. - - name: stripe__subscription_line_items description: Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. columns: diff --git a/models/stripe__invoice_details.sql b/models/stripe__invoice_details.sql index 78120300..f47b0fd6 100644 --- a/models/stripe__invoice_details.sql +++ b/models/stripe__invoice_details.sql @@ -45,7 +45,7 @@ with invoice as ( select invoice.invoice_id, - invoice.number, + invoice.number as invoice_number, invoice.created_at as invoice_created_at, invoice.period_start, invoice.period_end, From 3d3c810882b8d2db1c651d7ffd8715cd61740382 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 02:18:07 +0800 Subject: [PATCH 24/36] thinking of just deleting this model --- models/stripe__subscription_line_items.sql | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 models/stripe__subscription_line_items.sql diff --git a/models/stripe__subscription_line_items.sql b/models/stripe__subscription_line_items.sql deleted file mode 100644 index 4591efbf..00000000 --- a/models/stripe__subscription_line_items.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config(enabled=fivetran_utils.enabled_vars(['stripe__using_invoices','stripe__using_subscriptions'])) }} - - -with line_items as ( - - select * - from {{ ref('stripe__invoice_details') }} - where subscription_id is not null - -) - -select * -from line_items From 365e70c9158daec86081e4e030501409c6222c25 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 02:29:32 +0800 Subject: [PATCH 25/36] fix vars --- models/stripe.yml | 5 ++++- models/stripe__invoice_line_items.sql | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index c133aff2..307f1404 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -36,6 +36,8 @@ models: description: Three-letter ISO currency code that the customer sees. - name: effective_at description: calendar day after available_at + - name: account_id + description: Account ID associated with this transaction - name: customer_id description: The customer associated with the balance transaction (based on charge or refund details) - name: receipt_email @@ -527,7 +529,7 @@ models: description: Email of the customer who owns the subscription. - name: status description: Possible values are incomplete, incomplete_expired, trialing, active, past_due, canceled, or unpaid. - - name: start_date + - name: start_date_at description: Date when the subscription was first created. The date might differ from the created date due to backdating. - name: ended_at description: If the subscription has ended, the date the subscription ended. @@ -568,3 +570,4 @@ models: - name: stripe__account_daily_overview description: Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). + # holding off on columns until finalized \ No newline at end of file diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 01811a2f..42eb66f1 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -1,4 +1,4 @@ -{{ config(enabled=var('using_invoices', True)) }} +{{ config(enabled=var('stripe__using_invoices', True)) }} with invoice_line_item as ( @@ -15,7 +15,7 @@ with invoice_line_item as ( select * from {{ var('subscription') }} -{% if var('using_subscriptions', True) %} +{% if var('stripe__using_subscriptions', True) %} ), pricing as ( select * From d50715935f6afe3a0e58000949e5b1bd6c60deb9 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 02:34:15 +0800 Subject: [PATCH 26/36] yml updates --- models/stripe.yml | 154 +++++++++++++++++++++++----------------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/models/stripe.yml b/models/stripe.yml index 307f1404..8166eccf 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -436,83 +436,83 @@ models: - name: pricing_product_id description: Reference to the product whose pricing this price or plan determines. - - name: stripe__subscription_line_items - description: Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. - columns: - - name: invoice_id - description: The unique Identifier of the invoice object. - tests: - - unique - - name: number - description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. - - name: invoice_created_at - description: Timestamp of when the invoice was created. - - name: status - description: Current status of the invoice. - - name: due_date - description: Date when payment for the invoice is due. - - name: amount_due - description: The amount that the invoice is for. Because this query includes line item details, this amount may be repeated. - - name: subtotal - description: The amount of the invoice before discounts and taxes. - - name: tax - description: The amount of tax being charged in the invoice. - - name: total - description: The Total after discounts and taxes. - - name: amount_paid - description: The amount, if any, that the customer has paid on the invoice - - name: amount_remaining - description: The amount of the invoice remaining to be paid. - - name: attempt_count - description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. - - name: invoice_memo - description: An arbitrary string attached to the object. Often useful for displaying to users. - - name: line_item_desc - description: The description of the invoice line item - - name: line_item_amount - description: The amount for the line item. - - name: quantity - description: The quantity of the line item. - - name: balance_transaction_id - description: The ID of the balance transaction object representing payment - - name: charge_amount - description: The amount charged to the customer. - - name: charge_status - description: The status of the charge for the invoice. - - name: charge_created_at - description: When the charge for the invoice was created. - - name: customer_description - description: Description of the customer. - - name: customer_email - description: Email of the customer. - - name: subscription_id - description: ID of the subscription this invoice relates to. - - name: subscription_billing - description: How the subscription is billed - - name: subscription_start_date - description: The start date of the subscription - - name: subscription_ended_at - description: The end date of the subscription - - name: period_start - description: Timestamp of the start of the period. - - name: period_end - description: Timestamp of the end of the period. - - name: customer_id - description: The associated customer reference. - - name: plan_id - description: The ID of the plan of the subscription, if the line item is a subscription or a proration. - - name: plan_is_active - description: Boolean indicating if the plan is active (true) or in-active (false). - - name: plan_amount - description: The unit amount in cents to be charged, represented as a whole integer if possible. - - name: plan_interval - description: The frequency at which a subscription is billed. One of day, week, month or year. - - name: plan_interval_count - description: The number of intervals between subscription billings. For example, interval_count=3 bills every 3 months. - - name: plan_nickname - description: A brief description of the plan, hidden from customers. - - name: plan_product_id - description: Reference to the product whose pricing this plan determines. + # - name: stripe__subscription_line_items + # description: Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. + # columns: + # - name: invoice_id + # description: The unique Identifier of the invoice object. + # tests: + # - unique + # - name: number + # description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. + # - name: invoice_created_at + # description: Timestamp of when the invoice was created. + # - name: status + # description: Current status of the invoice. + # - name: due_date + # description: Date when payment for the invoice is due. + # - name: amount_due + # description: The amount that the invoice is for. Because this query includes line item details, this amount may be repeated. + # - name: subtotal + # description: The amount of the invoice before discounts and taxes. + # - name: tax + # description: The amount of tax being charged in the invoice. + # - name: total + # description: The Total after discounts and taxes. + # - name: amount_paid + # description: The amount, if any, that the customer has paid on the invoice + # - name: amount_remaining + # description: The amount of the invoice remaining to be paid. + # - name: attempt_count + # description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. + # - name: invoice_memo + # description: An arbitrary string attached to the object. Often useful for displaying to users. + # - name: line_item_desc + # description: The description of the invoice line item + # - name: line_item_amount + # description: The amount for the line item. + # - name: quantity + # description: The quantity of the line item. + # - name: balance_transaction_id + # description: The ID of the balance transaction object representing payment + # - name: charge_amount + # description: The amount charged to the customer. + # - name: charge_status + # description: The status of the charge for the invoice. + # - name: charge_created_at + # description: When the charge for the invoice was created. + # - name: customer_description + # description: Description of the customer. + # - name: customer_email + # description: Email of the customer. + # - name: subscription_id + # description: ID of the subscription this invoice relates to. + # - name: subscription_billing + # description: How the subscription is billed + # - name: subscription_start_date + # description: The start date of the subscription + # - name: subscription_ended_at + # description: The end date of the subscription + # - name: period_start + # description: Timestamp of the start of the period. + # - name: period_end + # description: Timestamp of the end of the period. + # - name: customer_id + # description: The associated customer reference. + # - name: plan_id + # description: The ID of the plan of the subscription, if the line item is a subscription or a proration. + # - name: plan_is_active + # description: Boolean indicating if the plan is active (true) or in-active (false). + # - name: plan_amount + # description: The unit amount in cents to be charged, represented as a whole integer if possible. + # - name: plan_interval + # description: The frequency at which a subscription is billed. One of day, week, month or year. + # - name: plan_interval_count + # description: The number of intervals between subscription billings. For example, interval_count=3 bills every 3 months. + # - name: plan_nickname + # description: A brief description of the plan, hidden from customers. + # - name: plan_product_id + # description: Reference to the product whose pricing this plan determines. - name: stripe__subscription_details description: Each record represents a subscription, enriched with customer details and payment aggregations. From 9e22111c75b11e57f89b33e4b6b946fe5ef9386c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 03:16:12 +0800 Subject: [PATCH 27/36] try new schema for integration tests --- integration_tests/ci/sample.profiles.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 4f697621..c5e40f69 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: stripe_integrations_tests + schema: stripe_integrations_tests_1 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: stripe_integrations_tests + schema: stripe_integrations_tests_1 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: stripe_integrations_tests + schema: stripe_integrations_tests_1 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: stripe_integrations_tests + schema: stripe_integrations_tests_1 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: stripe_integrations_tests + schema: stripe_integrations_tests_1 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file From 1e441347a4b6c862f60c51affdc358bba33e228c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 03:16:35 +0800 Subject: [PATCH 28/36] try new schema for integration tests --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 5e976b24..b52db49b 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,7 +6,7 @@ version: '0.8.0' profile: 'integration_tests' vars: - stripe_schema: stripe_integrations_tests + stripe_schema: stripe_integrations_tests_1 stripe_source: stripe_account_identifier: "account_data" stripe_group_identifier: "group_data" From cfaaa9868b3c2411128dd4f706eded3cea0bd405 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Tue, 13 Dec 2022 03:30:54 +0800 Subject: [PATCH 29/36] readme updates --- README.md | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d5b2d03e..af408a64 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,10 @@ The following table provides a detailed list of all models materialized within t | **model** | **description** | |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [stripe__balance_transactions](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__balance_transactions) | Each record represents a change to your account balance, enriched with data about the transaction. | -| [stripe__invoice_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items) | Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and plan. | +| [stripe__invoice_details](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_details) | Each record represents an invoice, enriched with details about the associated charge, customer, and subscription data. +| [stripe__invoice_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items) | Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and pricing data. +| [stripe__account_daily_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items) | Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). | | [stripe__subscription_details](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__subscription_details) | Each record represents a subscription, enriched with customer details and payment aggregations. | -| [stripe__subscription_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__subscription_line_items) | Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. | | [stripe__customer_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__customer_overview) | Each record represents a customer, enriched with metrics about their associated transactions. Transactions with no associated customer will have a customer description of "No associated customer". | | [stripe__daily_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__daily_overview) | Each record represents a single day, enriched with metrics about balances, payments, refunds, payouts, and other transactions. | | [stripe__weekly_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__weekly_overview) | Each record represents a single week, enriched with metrics about balances, payments, refunds, payouts, and other transactions. | @@ -55,7 +56,7 @@ Include the following stripe package version in your `packages.yml` file: ```yaml packages: - package: fivetran/stripe - version: [">=0.8.0", "<0.9.0"] + version: [">=0.9.0", "<0.10.0"] ``` ## Step 3: Define database and schema variables @@ -68,7 +69,7 @@ vars: ``` ## Step 4: Disable models for non-existent sources -This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, or `subscription` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable within your root `dbt_project.yml`: +This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, or `subscription` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable within your root `dbt_project.yml`: ```yml vars: stripe__using_invoices: False #Disable if you are not using the invoice and invoice_line_item tables @@ -76,13 +77,41 @@ vars: stripe__using_subscriptions: False #Disable if you are not using the subscription and plan tables. ``` ## Step 5: Leveraging Subscription Vs Subscription History Sources -For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription` source table. However, if you have a newer connector then you must leverage the `stripe__subscription_history` to have the package use the `subscription_history` source rather than the `subscription` table. +For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription_history` source table. However, if you have a older connector then you must leverage the `stripe__subscription` to have the package use the `subscription` source rather than the `subscription_history` table. > **Please note that if you have `stripe__subscription_history` enabled then the package will filter for only active records.** ```yml vars: - stripe__subscription_history: True # False by default. Set to True if your connector syncs the `subscription_history` table. + stripe__subscription_history: False # True by default. Set to False if your connector syncs the `subscription_history` table instead. ``` -## (Optional) Step 6: Additional configurations +## Step 6: Leveraging Plan vs Price Sources + +Customers using Fivetran with the newest Stripe pricing model will have a `price` table in place of the older `plan` table. Therefore to accommodate two different source tables we added additional logic in the `stg_stripe__pricing` model, which replaces the `stg_stripe__plan` model. This model checks if there exists a `price` table using a new `does_table_exist()` macro. If not, it will look for a `plan` table. While the default is to use the `price` table if it exists, you may add the following to your `dbt_project.yml` to override using the macro. + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + stripe: + stripe__price: false # If true, will look `price ` table. If false, will look for the `plan` table. +``` +## Step 7: Unioning Multiple Stripe Connectors +If you have multiple Stripe connectors you would like to use this package on simultaneously, we have added the ability to do so. Data from disparate connectors will be unioned together and be passed downstream to the end models. The `source_relation` column will specify where each record comes from. To use this functionality, you will need to either set the `stripe_union_schemas` or `stripe_union_databases` variables. Please also make sure the single-source `stripe_database` and `stripe_schema` variables are removed. + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + stripe_union_schemas: ['stripe_us','stripe_mx'] # use this if the data is in different schemas/datasets of the same database/project + stripe_union_databases: ['stripe_db_1','stripe_db_2'] # use this if the data is in different databases/projects but uses the same schema name +``` + +## (Optional) Step 8: Additional configurations
Expand for configurations ### Setting your timezone From 4153d98ca8821875fe64d4e01c9b48133f9c4edb Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 14 Dec 2022 15:38:57 +0800 Subject: [PATCH 30/36] update docs and run script and var names --- .buildkite/scripts/run_models.sh | 2 +- CHANGELOG.md | 2 ++ README.md | 11 +++++++---- models/stripe__invoice_line_items.sql | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 56ac4fd8..9daee3de 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -19,5 +19,5 @@ dbt deps dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" -dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true, stripe__price: false}' --target "$db" +dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__using_subscription_history: false, stripe__price: false}' --target "$db" dbt test --target "$db" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a8913310..3133d31e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ FINISH [#55](https://github.com/fivetran/dbt_stripe/pull/55) + +stripe__using_subscription_history # dbt_stripe v0.8.0 ## 🚨 Breaking Changes 🚨: diff --git a/README.md b/README.md index af408a64..27968e64 100644 --- a/README.md +++ b/README.md @@ -69,19 +69,22 @@ vars: ``` ## Step 4: Disable models for non-existent sources -This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, or `subscription` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable within your root `dbt_project.yml`: +This package takes into consideration that not every Stripe account utilizes the **invoices**, **payment method**, and **subscription** features. Therefore we allow you to configure the following variables below, which will then disable the corresponding related tables: `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, or `subscription`. The `plan` and `price` tables are toggled automatically (see *Step 6: Leveraging Plan vs Price Sources*) + +By default, all variables' values are assumed to be `true`. Only add variables within your root `dbt_project.yml` for only the tables you would want to disable: + ```yml vars: stripe__using_invoices: False #Disable if you are not using the invoice and invoice_line_item tables stripe__using_payment_method: False #Disable if you are not using the payment_method and payment_method_card tables - stripe__using_subscriptions: False #Disable if you are not using the subscription and plan tables. + stripe__using_subscriptions: False #Disable if you are not using the subscription and plan/pricing tables. ``` ## Step 5: Leveraging Subscription Vs Subscription History Sources -For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription_history` source table. However, if you have a older connector then you must leverage the `stripe__subscription` to have the package use the `subscription` source rather than the `subscription_history` table. +For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription_history` source table. However, if you have a older connector then you must leverage the `stripe__using_subscription_history` toggle to have the package use the `subscription` source rather than the `subscription_history` table. > **Please note that if you have `stripe__subscription_history` enabled then the package will filter for only active records.** ```yml vars: - stripe__subscription_history: False # True by default. Set to False if your connector syncs the `subscription_history` table instead. + stripe__usingsubscription_history: False # True by default. Set to False if your connector syncs the `subscription_history` table instead. ``` ## Step 6: Leveraging Plan vs Price Sources diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 42eb66f1..b6ef2939 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -79,7 +79,7 @@ left join subscription and invoice_line_item.source_relation = subscription.source_relation left join pricing - {% if var('stripe__price', does_table_exist('price')) %} + {% if var('stripe__using_price', does_table_exist('price')) %} on invoice_line_item.price_id = pricing.price_id {% else %} From b9ca8504fb2387b768222a1d04520a3e40982237 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Wed, 14 Dec 2022 15:39:03 +0800 Subject: [PATCH 31/36] try using more for loops --- models/stripe__account_daily_overview.sql | 132 ++++++++-------------- 1 file changed, 48 insertions(+), 84 deletions(-) diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index 7672baf9..72419631 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -1,4 +1,5 @@ -{% set fields = ['rolling_total_daily_sales_amount', 'rolling_total_daily_refunds_amount', 'rolling_total_daily_adjustments_amount', 'rolling_total_daily_other_transactions_amount', 'rolling_total_daily_gross_transaction_amount', 'rolling_total_daily_net_transactions_amount', 'rolling_total_daily_payout_fee_amount', 'rolling_total_daily_gross_payout_amount', 'rolling_daily_net_activity_amount', 'rolling_daily_end_balance_amount', 'rolling_total_daily_sales_count', 'rolling_total_daily_payouts_count', 'rolling_total_daily_adjustments_count', 'rolling_total_daily_failed_charge_count', 'rolling_total_daily_failed_charge_amount'] %} +{% set total_fields = ['total_daily_sales_amount', 'total_daily_refunds_amount', 'total_daily_adjustments_amount', 'total_daily_other_transactions_amount', 'total_daily_gross_transaction_amount', 'total_daily_net_transactions_amount', 'total_daily_payout_fee_amount', 'total_daily_gross_payout_amount', 'daily_net_activity_amount', 'daily_end_balance_amount', 'total_daily_sales_count', 'total_daily_payouts_count', 'total_daily_adjustments_count', 'total_daily_failed_charge_count', 'total_daily_failed_charge_amount'] %} +{% set rolling_fields = ['rolling_total_daily_sales_amount', 'rolling_total_daily_refunds_amount', 'rolling_total_daily_adjustments_amount', 'rolling_total_daily_other_transactions_amount', 'rolling_total_daily_gross_transaction_amount', 'rolling_total_daily_net_transactions_amount', 'rolling_total_daily_payout_fee_amount', 'rolling_total_daily_gross_payout_amount', 'rolling_daily_net_activity_amount', 'rolling_daily_end_balance_amount', 'rolling_total_daily_sales_count', 'rolling_total_daily_payouts_count', 'rolling_total_daily_adjustments_count', 'rolling_total_daily_failed_charge_count', 'rolling_total_daily_failed_charge_amount'] %} with date_spine as ( @@ -10,67 +11,33 @@ with date_spine as ( select * from {{ ref('int_stripe__account_daily')}} - -- select - -- account_id, - -- date_day, - -- sum(amount) as daily_amount, - -- sum(case when lower(reporting_category) = 'charge' - -- then amount - -- else 0 - -- end) as daily_charge_amount, - -- sum(case when lower(reporting_category) = 'refund' - -- then amount - -- else 0 - -- end) as daily_refund_amount, - -- sum(case when lower(reporting_category) = 'payout_reversal' - -- then amount - -- else 0 - -- end) as daily_payout_reversal_amount, - -- sum(case when lower(reporting_category) = 'transfer' - -- then amount - -- else 0 - -- end) as daily_transfer_count, - -- sum(case when lower(reporting_category) = 'transfer_reversal' - -- then amount - -- else 0 - -- end) as daily_transfer_reversal_amount - -- sum(case when lower(reporting_category) not in ('charge','refund','payout_reversal','transfer','transfer_reversal') - -- then amount - -- else 0 - -- end) as daily_other_amount - -- from transactions_grouped - -- group by 1, 2 --- ), - ), account_rolling_totals as ( select *, - sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_amount, - sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_refunds_amount, - sum(total_daily_adjustments_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_amount, - sum(total_daily_other_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_other_transactions_amount, - sum(total_daily_gross_transaction_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_transaction_amount, - sum(total_daily_net_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_net_transactions_amount, - sum(total_daily_payout_fee_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payout_fee_amount, - sum(total_daily_gross_payout_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_payout_amount, - sum(daily_net_activity_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_net_activity_amount, - sum(daily_end_balance_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_end_balance_amount, - sum(total_daily_sales_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_count, - sum(total_daily_payouts_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payouts_count, - sum(total_daily_adjustments_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_count, - sum(total_daily_failed_charge_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_count, - sum(total_daily_failed_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_amount - - - -- sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_amount, - -- sum(daily_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_charge_amount, - -- sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_refund_amount, - -- sum(daily_payout_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_payout_reversal_amount, - -- sum(daily_transfer_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_count, - -- sum(daily_transfer_reversal_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_transfer_reversal_amount, - -- sum(daily_other_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_other_amount - + + {% for t in total_fields %} + sum{{ t }} over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_{{ t }}, + {% endfor %} + + source_relation + + -- sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_amount, + -- sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_refunds_amount, + -- sum(total_daily_adjustments_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_amount, + -- sum(total_daily_other_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_other_transactions_amount, + -- sum(total_daily_gross_transaction_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_transaction_amount, + -- sum(total_daily_net_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_net_transactions_amount, + -- sum(total_daily_payout_fee_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payout_fee_amount, + -- sum(total_daily_gross_payout_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_payout_amount, + -- sum(daily_net_activity_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_net_activity_amount, + -- sum(daily_end_balance_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_end_balance_amount, + -- sum(total_daily_sales_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_count, + -- sum(total_daily_payouts_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payouts_count, + -- sum(total_daily_adjustments_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_count, + -- sum(total_daily_failed_charge_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_count, + -- sum(total_daily_failed_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_amount + from account_daily_balances_by_type ), final as ( @@ -78,38 +45,35 @@ with date_spine as ( select coalesce(account_rolling_totals.account_id, date_spine.account_id) as account_id, coalesce(account_rolling_totals.date_day, date_spine.date_day) as date_day, - account_rolling_totals.total_daily_sales_amount, - account_rolling_totals.total_daily_refunds_amount, - account_rolling_totals.total_daily_adjustments_amount, - account_rolling_totals.total_daily_other_transactions_amount, - account_rolling_totals.total_daily_gross_transaction_amount, - account_rolling_totals.total_daily_net_transactions_amount, - account_rolling_totals.total_daily_payout_fee_amount, - account_rolling_totals.total_daily_gross_payout_amount, - account_rolling_totals.daily_net_activity_amount, - account_rolling_totals.daily_end_balance_amount, - account_rolling_totals.total_daily_sales_count, - account_rolling_totals.total_daily_payouts_count, - account_rolling_totals.total_daily_adjustments_count, - account_rolling_totals.total_daily_failed_charge_count, - account_rolling_totals.total_daily_failed_charge_amount, - - -- account_rolling_totals.daily_amount, - -- account_rolling_totals.daily_charge_amount, - -- account_rolling_totals.daily_refund_amount, - -- account_rolling_totals.daily_payout_reversal_amount, - -- account_rolling_totals.daily_transfer_count, - -- account_rolling_totals.daily_transfer_reversal_amount, - -- account_rolling_totals.daily_other_amount, - - {% for f in fields %} + -- account_rolling_totals.total_daily_sales_amount, + -- account_rolling_totals.total_daily_refunds_amount, + -- account_rolling_totals.total_daily_adjustments_amount, + -- account_rolling_totals.total_daily_other_transactions_amount, + -- account_rolling_totals.total_daily_gross_transaction_amount, + -- account_rolling_totals.total_daily_net_transactions_amount, + -- account_rolling_totals.total_daily_payout_fee_amount, + -- account_rolling_totals.total_daily_gross_payout_amount, + -- account_rolling_totals.daily_net_activity_amount, + -- account_rolling_totals.daily_end_balance_amount, + -- account_rolling_totals.total_daily_sales_count, + -- account_rolling_totals.total_daily_payouts_count, + -- account_rolling_totals.total_daily_adjustments_count, + -- account_rolling_totals.total_daily_failed_charge_count, + -- account_rolling_totals.total_daily_failed_charge_amount, + + {% for t in total_fields %} + account_rolling_totals.{{ t }}, + {% endfor %} + + {% for f in rolling_fields %} case when account_rolling_totals.{{ f }} is null and date_index = 1 then 0 else account_rolling_totals.{{ f }} end as {{ f }}, {% endfor %} - date_spine.date_index + date_spine.date_index, + source_relation from date_spine left join account_rolling_totals From ac62aa18b1229bab5b7a485d100686752b59401c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 15 Dec 2022 02:21:15 +0800 Subject: [PATCH 32/36] updates --- README.md | 2 +- models/stripe__account_daily_overview.sql | 6 +++--- models/stripe__invoice_line_items.sql | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 27968e64..71f17a8d 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ config-version: 2 vars: stripe: - stripe__price: false # If true, will look `price ` table. If false, will look for the `plan` table. + stripe__using_price: false # True by default. If true, will look `price ` table. If false, will look for the `plan` table. ``` ## Step 7: Unioning Multiple Stripe Connectors If you have multiple Stripe connectors you would like to use this package on simultaneously, we have added the ability to do so. Data from disparate connectors will be unioned together and be passed downstream to the end models. The `source_relation` column will specify where each record comes from. To use this functionality, you will need to either set the `stripe_union_schemas` or `stripe_union_databases` variables. Please also make sure the single-source `stripe_database` and `stripe_schema` variables are removed. diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index 72419631..b56fadc8 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -17,10 +17,10 @@ with date_spine as ( *, {% for t in total_fields %} - sum{{ t }} over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_{{ t }}, + sum({{ t }}) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_{{ t }}, {% endfor %} - source_relation + -- source_relation # add this in upon union_feature merge -- sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_amount, -- sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_refunds_amount, @@ -73,7 +73,7 @@ with date_spine as ( {% endfor %} date_spine.date_index, - source_relation + -- source_relation # add later! from date_spine left join account_rolling_totals diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index b6ef2939..5f83f4de 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -10,12 +10,12 @@ with invoice_line_item as ( select * from {{ ref('stripe__invoice_details') }} +{% if var('stripe__using_subscriptions', True) %} ), subscription as ( select * from {{ var('subscription') }} -{% if var('stripe__using_subscriptions', True) %} ), pricing as ( select * From 0b5c5d2fc598adf6f95728a9e053875479d61b67 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 15 Dec 2022 02:28:10 +0800 Subject: [PATCH 33/36] fixes and new schema integration testing --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 2 +- models/stripe__account_daily_overview.sql | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index c5e40f69..1986ed6f 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: stripe_integrations_tests_1 + schema: stripe_integrations_tests_2 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: stripe_integrations_tests_1 + schema: stripe_integrations_tests_2 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: stripe_integrations_tests_1 + schema: stripe_integrations_tests_2 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: stripe_integrations_tests_1 + schema: stripe_integrations_tests_2 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: stripe_integrations_tests_1 + schema: stripe_integrations_tests_2 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index b52db49b..1a82ec69 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,7 +6,7 @@ version: '0.8.0' profile: 'integration_tests' vars: - stripe_schema: stripe_integrations_tests_1 + stripe_schema: stripe_integrations_tests_2 stripe_source: stripe_account_identifier: "account_data" stripe_group_identifier: "group_data" diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index b56fadc8..9dc27ddb 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -14,13 +14,13 @@ with date_spine as ( ), account_rolling_totals as ( select - *, + * {% for t in total_fields %} - sum({{ t }}) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_{{ t }}, + , sum({{ t }}) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_{{ t }} {% endfor %} - -- source_relation # add this in upon union_feature merge + -- , source_relation # add this in upon union_feature merge -- sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_amount, -- sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_refunds_amount, @@ -72,7 +72,8 @@ with date_spine as ( end as {{ f }}, {% endfor %} - date_spine.date_index, + date_spine.date_index + -- , -- source_relation # add later! from date_spine From db948625b35e63229dde1a756bd29baaeb89cb0c Mon Sep 17 00:00:00 2001 From: Renee Li <91097070+fivetran-reneeli@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:56:08 +0800 Subject: [PATCH 34/36] Update .buildkite/scripts/run_models.sh Co-authored-by: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> --- .buildkite/scripts/run_models.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 9daee3de..0b8ea6c8 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -19,5 +19,5 @@ dbt deps dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" -dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__using_subscription_history: false, stripe__price: false}' --target "$db" +dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__using_subscription_history: false, stripe__using_price: false}' --target "$db" dbt test --target "$db" \ No newline at end of file From ea6625e03e4cfa8e2b68723a2623653478d5f96c Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Thu, 15 Dec 2022 17:46:53 +0800 Subject: [PATCH 35/36] updates and finish changelog --- CHANGELOG.md | 10 ++++++-- models/stripe__account_daily_overview.sql | 31 ----------------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3133d31e..563b951c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,15 @@ # dbt_stripe v0.9.0 -FINISH +## 🎉 Feature Updates +- New models `stripe__account_daily_overview` and `stripe__invoice_details` have been added. +- `subscription_item_id` has been added to the `stripe__invoice_line_items` model. + + +## 🚨 Breaking Changes 🚨: +- `stripe__subscription_line_items` has been removed. To recreate it, simply filter `stripe__invoice_line_items` for where `subscription_id` is not null. + [#55](https://github.com/fivetran/dbt_stripe/pull/55) -stripe__using_subscription_history # dbt_stripe v0.8.0 ## 🚨 Breaking Changes 🚨: diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql index 9dc27ddb..4f8a64ff 100644 --- a/models/stripe__account_daily_overview.sql +++ b/models/stripe__account_daily_overview.sql @@ -22,22 +22,6 @@ with date_spine as ( -- , source_relation # add this in upon union_feature merge - -- sum(total_daily_sales_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_amount, - -- sum(total_daily_refunds_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_refunds_amount, - -- sum(total_daily_adjustments_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_amount, - -- sum(total_daily_other_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_other_transactions_amount, - -- sum(total_daily_gross_transaction_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_transaction_amount, - -- sum(total_daily_net_transactions_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_net_transactions_amount, - -- sum(total_daily_payout_fee_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payout_fee_amount, - -- sum(total_daily_gross_payout_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_gross_payout_amount, - -- sum(daily_net_activity_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_net_activity_amount, - -- sum(daily_end_balance_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_daily_end_balance_amount, - -- sum(total_daily_sales_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_sales_count, - -- sum(total_daily_payouts_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_payouts_count, - -- sum(total_daily_adjustments_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_adjustments_count, - -- sum(total_daily_failed_charge_count) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_count, - -- sum(total_daily_failed_charge_amount) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_total_daily_failed_charge_amount - from account_daily_balances_by_type ), final as ( @@ -45,21 +29,6 @@ with date_spine as ( select coalesce(account_rolling_totals.account_id, date_spine.account_id) as account_id, coalesce(account_rolling_totals.date_day, date_spine.date_day) as date_day, - -- account_rolling_totals.total_daily_sales_amount, - -- account_rolling_totals.total_daily_refunds_amount, - -- account_rolling_totals.total_daily_adjustments_amount, - -- account_rolling_totals.total_daily_other_transactions_amount, - -- account_rolling_totals.total_daily_gross_transaction_amount, - -- account_rolling_totals.total_daily_net_transactions_amount, - -- account_rolling_totals.total_daily_payout_fee_amount, - -- account_rolling_totals.total_daily_gross_payout_amount, - -- account_rolling_totals.daily_net_activity_amount, - -- account_rolling_totals.daily_end_balance_amount, - -- account_rolling_totals.total_daily_sales_count, - -- account_rolling_totals.total_daily_payouts_count, - -- account_rolling_totals.total_daily_adjustments_count, - -- account_rolling_totals.total_daily_failed_charge_count, - -- account_rolling_totals.total_daily_failed_charge_amount, {% for t in total_fields %} account_rolling_totals.{{ t }}, From 7e5e3a208f380083095114eed2301c7a14087f96 Mon Sep 17 00:00:00 2001 From: fivetran-reneeli Date: Fri, 16 Dec 2022 02:07:08 +0800 Subject: [PATCH 36/36] updates --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 563b951c..aed9a7cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,14 @@ # dbt_stripe v0.9.0 + +[PR #55](https://github.com/fivetran/dbt_stripe/pull/55): ## 🎉 Feature Updates - New models `stripe__account_daily_overview` and `stripe__invoice_details` have been added. - `subscription_item_id` has been added to the `stripe__invoice_line_items` model. - ## 🚨 Breaking Changes 🚨: - `stripe__subscription_line_items` has been removed. To recreate it, simply filter `stripe__invoice_line_items` for where `subscription_id` is not null. -[#55](https://github.com/fivetran/dbt_stripe/pull/55) # dbt_stripe v0.8.0