diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index e0182eba..0b8ea6c8 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -19,5 +19,5 @@ dbt deps dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" -dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true}' --target "$db" +dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__using_subscription_history: false, stripe__using_price: false}' --target "$db" dbt test --target "$db" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af895c1..aed9a7cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# dbt_stripe v0.9.0 + + +[PR #55](https://github.com/fivetran/dbt_stripe/pull/55): +## 🎉 Feature Updates +- New models `stripe__account_daily_overview` and `stripe__invoice_details` have been added. +- `subscription_item_id` has been added to the `stripe__invoice_line_items` model. + +## 🚨 Breaking Changes 🚨: +- `stripe__subscription_line_items` has been removed. To recreate it, simply filter `stripe__invoice_line_items` for where `subscription_id` is not null. + + # dbt_stripe v0.8.0 ## 🚨 Breaking Changes 🚨: diff --git a/README.md b/README.md index 8dcdf695..71f17a8d 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,10 @@ The following table provides a detailed list of all models materialized within t | **model** | **description** | |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [stripe__balance_transactions](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__balance_transactions) | Each record represents a change to your account balance, enriched with data about the transaction. | -| [stripe__invoice_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items) | Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and plan. | +| [stripe__invoice_details](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_details) | Each record represents an invoice, enriched with details about the associated charge, customer, and subscription data. +| [stripe__invoice_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items) | Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and pricing data. +| [stripe__account_daily_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items) | Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). | | [stripe__subscription_details](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__subscription_details) | Each record represents a subscription, enriched with customer details and payment aggregations. | -| [stripe__subscription_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__subscription_line_items) | Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. | | [stripe__customer_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__customer_overview) | Each record represents a customer, enriched with metrics about their associated transactions. Transactions with no associated customer will have a customer description of "No associated customer". | | [stripe__daily_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__daily_overview) | Each record represents a single day, enriched with metrics about balances, payments, refunds, payouts, and other transactions. | | [stripe__weekly_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__weekly_overview) | Each record represents a single week, enriched with metrics about balances, payments, refunds, payouts, and other transactions. | @@ -55,7 +56,7 @@ Include the following stripe package version in your `packages.yml` file: ```yaml packages: - package: fivetran/stripe - version: [">=0.8.0", "<0.9.0"] + version: [">=0.9.0", "<0.10.0"] ``` ## Step 3: Define database and schema variables @@ -68,21 +69,52 @@ vars: ``` ## Step 4: Disable models for non-existent sources -This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, or `subscription` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable within your root `dbt_project.yml`: +This package takes into consideration that not every Stripe account utilizes the **invoices**, **payment method**, and **subscription** features. Therefore we allow you to configure the following variables below, which will then disable the corresponding related tables: `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, or `subscription`. The `plan` and `price` tables are toggled automatically (see *Step 6: Leveraging Plan vs Price Sources*) + +By default, all variables' values are assumed to be `true`. Only add variables within your root `dbt_project.yml` for only the tables you would want to disable: + ```yml vars: - using_invoices: False #Disable if you are not using the invoice and invoice_line_item tables - using_payment_method: False #Disable if you are not using the payment_method and payment_method_card tables - using_subscriptions: False #Disable if you are not using the subscription and plan tables. + stripe__using_invoices: False #Disable if you are not using the invoice and invoice_line_item tables + stripe__using_payment_method: False #Disable if you are not using the payment_method and payment_method_card tables + stripe__using_subscriptions: False #Disable if you are not using the subscription and plan/pricing tables. ``` ## Step 5: Leveraging Subscription Vs Subscription History Sources -For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription` source table. However, if you have a newer connector then you must leverage the `stripe__subscription_history` to have the package use the `subscription_history` source rather than the `subscription` table. +For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription_history` source table. However, if you have a older connector then you must leverage the `stripe__using_subscription_history` toggle to have the package use the `subscription` source rather than the `subscription_history` table. > **Please note that if you have `stripe__subscription_history` enabled then the package will filter for only active records.** ```yml vars: - stripe__subscription_history: True # False by default. Set to True if your connector syncs the `subscription_history` table. + stripe__usingsubscription_history: False # True by default. Set to False if your connector syncs the `subscription_history` table instead. +``` +## Step 6: Leveraging Plan vs Price Sources + +Customers using Fivetran with the newest Stripe pricing model will have a `price` table in place of the older `plan` table. Therefore to accommodate two different source tables we added additional logic in the `stg_stripe__pricing` model, which replaces the `stg_stripe__plan` model. This model checks if there exists a `price` table using a new `does_table_exist()` macro. If not, it will look for a `plan` table. While the default is to use the `price` table if it exists, you may add the following to your `dbt_project.yml` to override using the macro. + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + stripe: + stripe__using_price: false # True by default. If true, will look `price ` table. If false, will look for the `plan` table. +``` +## Step 7: Unioning Multiple Stripe Connectors +If you have multiple Stripe connectors you would like to use this package on simultaneously, we have added the ability to do so. Data from disparate connectors will be unioned together and be passed downstream to the end models. The `source_relation` column will specify where each record comes from. To use this functionality, you will need to either set the `stripe_union_schemas` or `stripe_union_databases` variables. Please also make sure the single-source `stripe_database` and `stripe_schema` variables are removed. + +```yml +# dbt_project.yml + +... +config-version: 2 + +vars: + stripe_union_schemas: ['stripe_us','stripe_mx'] # use this if the data is in different schemas/datasets of the same database/project + stripe_union_databases: ['stripe_db_1','stripe_db_2'] # use this if the data is in different databases/projects but uses the same schema name ``` -## (Optional) Step 6: Additional configurations + +## (Optional) Step 8: Additional configurations
Expand for configurations ### Setting your timezone diff --git a/dbt_project.yml b/dbt_project.yml index 1ec64e8b..49847cdd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'stripe' -version: '0.8.0' +version: '0.9.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: stripe: @@ -11,6 +11,7 @@ models: +materialized: ephemeral vars: stripe: + account: "{{ ref('stg_stripe__account') }}" balance_transaction: "{{ ref('stg_stripe__balance_transaction') }}" card: "{{ ref('stg_stripe__card') }}" charge: "{{ ref('stg_stripe__charge') }}" @@ -22,9 +23,9 @@ vars: payment_method_card: "{{ ref('stg_stripe__payment_method_card') }}" payment_method: "{{ ref('stg_stripe__payment_method') }}" payout: "{{ ref('stg_stripe__payout') }}" - plan: "{{ ref('stg_stripe__plan') }}" + pricing: "{{ ref('stg_stripe__pricing') }}" refund: "{{ ref('stg_stripe__refund') }}" subscription: "{{ ref('stg_stripe__subscription') }}" - using_invoices: true - using_payment_method: true - using_subscriptions: true + stripe__using_invoices: true + stripe__using_payment_method: true + stripe__using_subscriptions: true diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 4f697621..1986ed6f 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: stripe_integrations_tests + schema: stripe_integrations_tests_2 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: stripe_integrations_tests + schema: stripe_integrations_tests_2 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: stripe_integrations_tests + schema: stripe_integrations_tests_2 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: stripe_integrations_tests + schema: stripe_integrations_tests_2 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: stripe_integrations_tests + schema: stripe_integrations_tests_2 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 263a2bd3..1a82ec69 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,8 +6,9 @@ version: '0.8.0' profile: 'integration_tests' vars: - stripe_schema: stripe_integrations_tests + stripe_schema: stripe_integrations_tests_2 stripe_source: + stripe_account_identifier: "account_data" stripe_group_identifier: "group_data" stripe_balance_transaction_identifier: "balance_transaction_data" stripe_card_identifier: "card_data" @@ -21,6 +22,7 @@ vars: stripe_payment_method_identifier: "payment_method_data" stripe_payout_identifier: "payout_data" stripe_plan_identifier: "plan_data" + stripe_price_identifier: "price_data" stripe_refund_identifier: "refund_data" stripe_subscription_history_identifier: "subscription_history_data" stripe_subscription_identifier: "subscription_data" @@ -34,13 +36,17 @@ seeds: +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}" +column_types: _fivetran_synced: timestamp + created: timestamp + voided_at: timestamp + canceled_at: timestamp + invoice_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" balance_transaction_data: +column_types: created: timestamp available_on: timestamp charge_data: +column_types: - balance_transaction_id: "{{ 'varchar(250)' if target.type in ('redshift','postgres') else 'string'}}" + balance_transaction_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}" created: timestamp customer_data: +column_types: diff --git a/integration_tests/seeds/account_data.csv b/integration_tests/seeds/account_data.csv new file mode 100644 index 00000000..47917fcc --- /dev/null +++ b/integration_tests/seeds/account_data.csv @@ -0,0 +1,2 @@ +id,_fivetran_synced,business_profile_mcc,business_profile_name,business_profile_product_description,business_profile_support_address_city,business_profile_support_address_country,business_profile_support_address_line_1,business_profile_support_address_line_2,business_profile_support_address_postal_code,business_profile_support_address_state,business_profile_support_email,business_profile_support_phone,business_profile_support_url,business_profile_url,business_type,capabilities_afterpay_clearpay_payments,capabilities_au_becs_debit_payments,capabilities_bacs_debit_payments,capabilities_bancontact_payments,capabilities_card_issuing,capabilities_card_payments,capabilities_cartes_bancaires_payments,capabilities_eps_payments,capabilities_fpx_payments,capabilities_giropay_payments,capabilities_grabpay_payments,capabilities_ideal_payments,capabilities_jcb_payments,capabilities_legacy_payments,capabilities_oxxo_payments,capabilities_p_24_payments,capabilities_platform_payments,capabilities_sepa_debit_payments,capabilities_sofort_payments,capabilities_tax_reporting_us_1099_k,capabilities_tax_reporting_us_1099_misc,capabilities_transfers,charges_enabled,company_address_city,company_address_country,company_address_kana_city,company_address_kana_country,company_address_kana_line_1,company_address_kana_line_2,company_address_kana_postal_code,company_address_kana_state,company_address_kana_town,company_address_kanji_city,company_address_kanji_country,company_address_kanji_line_1,company_address_kanji_line_2,company_address_kanji_postal_code,company_address_kanji_state,company_address_kanji_town,company_address_line_1,company_address_line_2,company_address_postal_code,company_address_state,company_directors_provided,company_executives_provided,company_name,company_name_kana,company_name_kanji,company_owners_provided,company_phone,company_structure,company_tax_id_provided,company_tax_id_registrar,company_vat_id_provided,company_verification_document_back,company_verification_document_details,company_verification_document_details_code,company_verification_document_front,country,created,default_currency,details_submitted,email,individual_id,is_deleted,metadata,payouts_enabled,requirements_current_deadline,requirements_currently_due,requirements_disabled_reason,requirements_errors,requirements_eventually_due,requirements_past_due,requirements_pending_verification,settings_branding_icon,settings_branding_logo,settings_branding_primary_color,settings_card_payments_decline_on_avs_failure,settings_card_payments_decline_on_cvc_failure,settings_card_payments_statement_descriptor_prefix,settings_dashboard_display_name,settings_dashboard_timezone,settings_payments_statement_descriptor,settings_payments_statement_descriptor_kana,settings_payments_statement_descriptor_kanji,settings_payouts_debit_negative_balances,settings_payouts_schedule_delay_days,settings_payouts_schedule_interval,settings_payouts_schedule_monthly_anchor,settings_payouts_schedule_weekly_anchor,settings_payouts_statement_descriptor,tos_acceptance_date,tos_acceptance_ip,tos_acceptance_user_agent,type +1234,2022-12-06 14:19:07,737,Hogwarts School of Witchcraft and Wizardry,,Hogwarts,US,123 Street,Apt 123,12345,CA,severus@hogwarts.com,,https://www.wizardingworld.com/,https://www.wizardingworld.com/,,active,,,active,,active,,active,,active,,active,,,,active,active,active,active,,,,TRUE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,US,,usd,TRUE,dumbledore@hogwarts.com,,FALSE,,TRUE,,,,,,,,,file_asdfasgas,#0030ff,,,,Hogwarts Inc.,America/Los_Angeles,hogwarts.com,,,TRUE,7,daily,,,,,,,standard \ No newline at end of file diff --git a/integration_tests/seeds/invoice_data.csv b/integration_tests/seeds/invoice_data.csv index e0ff9239..3285f4b3 100644 --- a/integration_tests/seeds/invoice_data.csv +++ b/integration_tests/seeds/invoice_data.csv @@ -1,3 +1,3 @@ id,_fivetran_synced,amount,currency,customer_id,date,description,discountable,invoice_id,is_deleted,livemode,period_end,period_start,proration,quantity,subscription_id,subscription_item_id,unit_amount -sdfg54545,2021-02-18 14:16:37,45,usd,sdgfs84854,,description_here,TRUE,sdg545641,FALSE,TRUE,2021-02-19 14:16:37,2021-02-18 14:16:37,TRUE,45,sdkhg54645,kdshg455,5 +ab1111,2021-02-18 14:16:37,45,usd,sdgfs84854,,description_here,TRUE,sdg545641,FALSE,TRUE,2021-02-19 14:16:37,2021-02-18 14:16:37,TRUE,45,sdkhg54645,kdshg455,5 si8jo54,2021-02-18 14:16:37,22,usd,dfd54654,,description_here,TRUE,dgjksg6486,FALSE,FALSE,2021-02-19 14:16:37,2021-02-18 14:16:37,FALSE,55,dghisd165748,sgdfs5464,6 \ No newline at end of file diff --git a/integration_tests/seeds/plan_data.csv b/integration_tests/seeds/plan_data.csv index f79bb2f3..d8bb0adc 100644 --- a/integration_tests/seeds/plan_data.csv +++ b/integration_tests/seeds/plan_data.csv @@ -5,8 +5,8 @@ ab2,2021-03-15 14:21:13,TRUE,,150000,per_unit,2017-05-05 16:56:44,usd,month,1,FA 3gf,2021-03-15 14:21:13,TRUE,,180000,per_unit,2017-06-15 16:01:06,usd,month,6,FALSE,TRUE,,prod_BUonomUNsdG3qx,,,,,licensed 214gfdg,2021-03-15 14:21:13,TRUE,,264030,per_unit,2017-08-04 16:29:13,usd,month,3,FALSE,TRUE,,prod_BUo1nHP2vIrVtF,,,,,licensed fgds754,2021-03-15 14:21:13,TRUE,,45000,per_unit,2017-06-21 16:23:32,usd,month,1,FALSE,TRUE,,prod_BUBtgJBLlTGcy9,,,,,licensed -dfgdw576,2021-03-15 14:21:13,TRUE,,20000,per_unit,2015-05-28 1:08:05,usd,month,1,FALSE,TRUE,,prod_BV5IFAOt2EwqSl,,,,,licensed -rhsdr456,2021-03-15 14:21:13,TRUE,,100,per_unit,2015-05-28 1:10:13,usd,month,1,FALSE,TRUE,,prod_BU23UCziUM7ezw,,,,,licensed +dfgdw576,2021-03-15 14:21:13,TRUE,,20000,per_unit,2015-05-28 01:08:05,usd,month,1,FALSE,TRUE,,prod_BV5IFAOt2EwqSl,,,,,licensed +rhsdr456,2021-03-15 14:21:13,TRUE,,100,per_unit,2015-05-28 01:10:13,usd,month,1,FALSE,TRUE,,prod_BU23UCziUM7ezw,,,,,licensed 3gds,2021-03-15 14:21:13,TRUE,,50000,per_unit,2015-10-23 21:23:35,usd,month,1,FALSE,TRUE,,prod_BTXZakZKkKlf6g,,,,,licensed 234rtgh,2021-03-15 14:21:13,TRUE,,40000,per_unit,2015-12-01 21:06:02,usd,month,1,FALSE,TRUE,,prod_BUrKq3BcaxvjkU,,,,,licensed 658jfgh,2021-03-15 14:21:13,TRUE,,15000,per_unit,2015-11-16 22:42:15,usd,month,1,FALSE,TRUE,,prod_BUaE6AwT15sLSX,,,,,licensed diff --git a/integration_tests/seeds/price_data.csv b/integration_tests/seeds/price_data.csv new file mode 100644 index 00000000..2e61ef9f --- /dev/null +++ b/integration_tests/seeds/price_data.csv @@ -0,0 +1,2 @@ +id,_fivetran_synced,active,billing_scheme,created,currency,invoice_item_id,is_deleted,livemode,lookup_key,metadata,nickname,product_id,recurring_aggregate_usage,recurring_interval,recurring_interval_count,recurring_usage_type,tiers_mode,transform_quantity_divide_by,transform_quantity_round,type,unit_amount,unit_amount_decimal +price_1,2022-09-22 23:49:02,TRUE,per_unit,2020-05-02 19:21:44,usd,ii_FYFJSuzUa5YRk,FALSE,TRUE,,{},,prod_HCqb8lm6kfYk,,,,,,,,one_time,100000,100000 \ No newline at end of file diff --git a/macros/does_table_exist.sql b/macros/does_table_exist.sql new file mode 100644 index 00000000..c219f688 --- /dev/null +++ b/macros/does_table_exist.sql @@ -0,0 +1,17 @@ +{%- macro does_table_exist(table_name) -%} + {%- if execute -%} -- returns true when dbt is in execute mode + {%- set ns = namespace(has_table=false) -%} -- declare boolean namespace and default value + {%- for node in graph.sources.values() -%} -- grab sources from the dictionary of nodes + -- call the database for the matching table + {%- set source_relation = adapter.get_relation( + database=node.database, + schema=node.schema, + identifier=node.identifier ) -%} + {%- if source_relation == None and node.name | lower == table_name | lower -%} + {{ return(False) }} -- return false if relation identified by the database.schema.identifier does not exist for the given table name + {%- elif source_relation != None and node.name | lower == table_name | lower -%} + {{ return(True) }} -- otherwise return True + {% endif %} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} \ No newline at end of file diff --git a/models/intermediate/int_stripe__account_daily.sql b/models/intermediate/int_stripe__account_daily.sql new file mode 100644 index 00000000..f4f7f02a --- /dev/null +++ b/models/intermediate/int_stripe__account_daily.sql @@ -0,0 +1,100 @@ +with date_spine as ( + + select * + from {{ ref('int_stripe__date_spine') }} + +), balance_transaction as ( + + select *, + case + when type = 'payout' + then {{ date_timezone('available_on') }} + else {{ date_timezone('created_at') }} + end as date + from {{ ref('stripe__balance_transactions') }} + +), incomplete_charges as ( + + select * + from {{ ref('int_stripe__incomplete_charges') }} + +), daily_account_balance_transactions as ( + + select + date_spine.date_day, + balance_transaction.account_id, + sum(case when balance_transaction.type in ('charge', 'payment') + then balance_transaction.amount + else 0 end) as total_daily_sales_amount, + sum(case when balance_transaction.type in ('payment_refund', 'refund') + then balance_transaction.amount + else 0 end) as total_daily_refunds_amount, + sum(case when balance_transaction.type = 'adjustment' + then balance_transaction.amount + else 0 end) as total_daily_adjustments_amount, + sum(case when balance_transaction.type not in ('charge', 'payment', 'payment_refund', 'refund', 'adjustment', 'payout') and balance_transaction.type not like '%transfer%' + then balance_transaction.amount + else 0 end) as total_daily_other_transactions_amount, + sum(case when balance_transaction.type <> 'payout' and balance_transaction.type not like '%transfer%' + then balance_transaction.amount + else 0 end) as total_daily_gross_transaction_amount, + sum(case when balance_transaction.type <> 'payout' and balance_transaction.type not like '%transfer%' + then net + else 0 end) as total_daily_net_transactions_amount, + sum(case when balance_transaction.type = 'payout' or balance_transaction.type like '%transfer%' + then fee * -1.0 + else 0 end) as total_daily_payout_fee_amount, + sum(case when balance_transaction.type = 'payout' or balance_transaction.type like '%transfer%' + then balance_transaction.amount + else 0 end) as total_daily_gross_payout_amount, + sum(case when balance_transaction.type = 'payout' or balance_transaction.type like '%transfer%' + then fee * -1.0 + else net end) as daily_net_activity_amount, + sum(case when balance_transaction.type in ('payment', 'charge') + then 1 + else 0 end) as total_daily_sales_count, + sum(case when balance_transaction.type = 'payout' + then 1 + else 0 end) as total_daily_payouts_count, + count(distinct case when balance_transaction.type = 'adjustment' + then coalesce(source, payout_id) + else null end) as total_daily_adjustments_count + from balance_transaction + left join date_spine + on balance_transaction.account_id = date_spine.account_id + and cast({{ dbt.date_trunc('day', 'balance_transaction.date') }} as date) = date_spine.date_day + group by 1, 2 + +), daily_failed_charges as ( + + select + {{ date_timezone('created_at') }} as date, + connected_account_id, + count(*) as total_daily_failed_charge_count, + sum(amount) as total_daily_failed_charge_amount + from incomplete_charges + group by 1, 2 +) + +select + daily_account_balance_transactions.date_day, + daily_account_balance_transactions.account_id, + daily_account_balance_transactions.total_daily_sales_amount/100.0 as total_daily_sales_amount, + daily_account_balance_transactions.total_daily_refunds_amount/100.0 as total_daily_refunds_amount, + daily_account_balance_transactions.total_daily_adjustments_amount/100.0 as total_daily_adjustments_amount, + daily_account_balance_transactions.total_daily_other_transactions_amount/100.0 as total_daily_other_transactions_amount, + daily_account_balance_transactions.total_daily_gross_transaction_amount/100.0 as total_daily_gross_transaction_amount, + daily_account_balance_transactions.total_daily_net_transactions_amount/100.0 as total_daily_net_transactions_amount, + daily_account_balance_transactions.total_daily_payout_fee_amount/100.0 as total_daily_payout_fee_amount, + daily_account_balance_transactions.total_daily_gross_payout_amount/100.0 as total_daily_gross_payout_amount, + daily_account_balance_transactions.daily_net_activity_amount/100.0 as daily_net_activity_amount, + (daily_account_balance_transactions.daily_net_activity_amount + daily_account_balance_transactions.total_daily_gross_payout_amount)/100.0 as daily_end_balance_amount, + daily_account_balance_transactions.total_daily_sales_count, + daily_account_balance_transactions.total_daily_payouts_count, + daily_account_balance_transactions.total_daily_adjustments_count, + coalesce(daily_failed_charges.total_daily_failed_charge_count, 0) as total_daily_failed_charge_count, + coalesce(daily_failed_charges.total_daily_failed_charge_amount/100, 0) as total_daily_failed_charge_amount +from daily_account_balance_transactions +left join daily_failed_charges + on daily_account_balance_transactions.date_day = daily_failed_charges.date + and daily_account_balance_transactions.account_id = daily_failed_charges.connected_account_id \ No newline at end of file diff --git a/models/intermediate/int_stripe__date_spine.sql b/models/intermediate/int_stripe__date_spine.sql new file mode 100644 index 00000000..2e13160b --- /dev/null +++ b/models/intermediate/int_stripe__date_spine.sql @@ -0,0 +1,91 @@ + +with spine as ( + + {% if execute %} + {% set first_date_query %} + select min( created_at ) as min_date from {{ ref('stripe__balance_transactions') }} + {% endset %} + {% set first_date = run_query(first_date_query).columns[0][0]|string %} + + {% if target.type == 'postgres' %} + {% set first_date_adjust = "cast('" ~ first_date[0:10] ~ "' as date)" %} + + {% else %} + {% set first_date_adjust = "'" ~ first_date[0:10] ~ "'" %} + + {% endif %} + + {% else %} {% set first_date_adjust = "'2009-01-01'" %} + {% endif %} + + {% if execute %} + {% set last_date_query %} + select max( created_at ) as max_date from {{ ref('stripe__balance_transactions') }} + {% endset %} + + {% set current_date_query %} + select current_date + {% endset %} + + {% if run_query(current_date_query).columns[0][0]|string < run_query(last_date_query).columns[0][0]|string %} + + {% set last_date = run_query(last_date_query).columns[0][0]|string %} + + {% else %} {% set last_date = run_query(current_date_query).columns[0][0]|string %} + {% endif %} + + {% if target.type == 'postgres' %} + {% set last_date_adjust = "cast('" ~ last_date[0:10] ~ "' as date)" %} + + {% else %} + {% set last_date_adjust = "'" ~ last_date[0:10] ~ "'" %} + + {% endif %} + {% endif %} + + {{ dbt_utils.date_spine( + datepart="day", + start_date=first_date_adjust, + end_date=dbt.dateadd("day", 1, last_date_adjust) + ) + }} +), + +balance_transactions as ( + + select * + from {{ ref('stripe__balance_transactions') }} +), + +account as ( + + select * + from {{ var('account') }} +), + +date_spine as ( + + select + cast({{ dbt.date_trunc("day", "date_day") }} as date) as date_day, + cast({{ dbt.date_trunc("week", "date_day") }} as date) as date_week, + cast({{ dbt.date_trunc("month", "date_day") }} as date) as date_month, + cast({{ dbt.date_trunc("year", "date_day") }} as date) as date_year, + row_number() over (order by cast({{ dbt.date_trunc("day", "date_day") }} as date)) as date_index + from spine +), + +final as ( + + select distinct + account.account_id, + date_spine.date_day, + date_spine.date_week, + date_spine.date_month, + date_spine.date_year, + date_spine.date_index + from account + cross join date_spine +) + +select * +from final \ No newline at end of file diff --git a/models/intermediate/int_stripe__incomplete_charges.sql b/models/intermediate/int_stripe__incomplete_charges.sql index 7713e914..0a37c575 100644 --- a/models/intermediate/int_stripe__incomplete_charges.sql +++ b/models/intermediate/int_stripe__incomplete_charges.sql @@ -8,6 +8,7 @@ with charge as ( select created_at, customer_id, + connected_account_id, amount from charge where not is_captured diff --git a/models/stripe.yml b/models/stripe.yml index b95c2bc2..8166eccf 100644 --- a/models/stripe.yml +++ b/models/stripe.yml @@ -20,6 +20,8 @@ models: description: Fees (in cents) paid for this transaction. - name: net description: Net amount of the transaction, in cents. + - name: account_id + description: The ID of the account tied to the transaction. - name: type description: The type of transaction. Possible values are adjustment, advance, advance_funding, application_fee, application_fee_refund, charge, connect_collection_transfer, issuing_authorization_hold, issuing_authorization_release, issuing_dispute, issuing_transaction, payment, payment_failure_refund, payment_refund, payout, payout_cancel, payout_failure, refund, refund_failure, reserve_transaction, reserved_funds, stripe_fee, stripe_fx_fee, tax_fee, topup, topup_reversal, transfer, transfer_cancel, transfer_failure, or transfer_refund. - name: reporting_category @@ -34,6 +36,8 @@ models: description: Three-letter ISO currency code that the customer sees. - name: effective_at description: calendar day after available_at + - name: account_id + description: Account ID associated with this transaction - name: customer_id description: The customer associated with the balance transaction (based on charge or refund details) - name: receipt_email @@ -212,7 +216,7 @@ models: description: Number of incomplete charges during the reporting period. - name: total_failed_charge_amount description: Total amount from incomplete charges during the reporting period. - + - name: stripe__monthly_overview description: Each record represents a single month, enriched with metrics about balances, payments, refunds, payouts, and other transactions. columns: @@ -282,47 +286,49 @@ models: description: Total amount from incomplete charges during the reporting period. - - name: stripe__invoice_line_items - description: Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and plan. + - name: stripe__invoice_details + description: Each record represents an invoice, enriched with details about the associated charge, customer, and subscription data. columns: - - name: invoice_id + - name: invoice_id description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. + tests: + - unique - name: number description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. - name: invoice_created_at description: Timestamp of when the invoice was created. + - name: period_start + description: Timestamp of the start of the period. + - name: period_end + description: Timestamp of the end of the period. - name: status description: Current status of the invoice. - name: due_date description: Date when payment for the invoice is due. - name: currency - description: The currency that the invoice is in. Three-letter ISO currency code, in lowercase. + description: The currency that the invoice is in. Three-letter ISO currency code, in lowercase. + - name: amount_due + description: The amount, if any, that the customer has paid on the invoice + - name: amount_paid + description: The amount, if any, that the customer has paid on the invoice - name: amount_due - description: The amount that the invoice is for. Because this query includes line item details, this amount may be repeated. + description: Final amount due at this time for this invoice. If the invoice’s total is smaller than the minimum charge amount, for example, or if there is account credit that can be applied to the invoice, the amount_due may be 0. If there is a positive starting_balance for the invoice (the customer owes money), the amount_due will also take that into account. The charge that gets generated for the invoice will be for the amount specified in amount_due. - name: subtotal description: The amount of the invoice before discounts and taxes. - name: tax description: The amount of tax being charged in the invoice. - name: total description: The Total after discounts and taxes. - - name: amount_paid - description: The amount, if any, that the customer has paid on the invoice - name: amount_remaining description: The amount of the invoice remaining to be paid. - name: attempt_count description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. - name: invoice_memo description: An arbitrary string attached to the object. Often useful for displaying to users. - - name: invoice_line_item_id - description: The unique Identifier of the invoice line object. - tests: - - unique - - name: line_item_desc - description: The description of the invoice line item - - name: line_item_amount - description: The amount for the line item. - - name: quantity - description: The quantity of the line item. + - name: number_of_line_items + description: Number of line item records for this invoice. + - name: total_quantity + description: The total quantity of items for this invoice. - name: balance_transaction_id description: The ID of the balance transaction object representing payment - name: charge_amount @@ -331,8 +337,20 @@ models: description: The status of the charge for the invoice. - name: charge_created_at description: When the charge for the invoice was created. + - name: charge_is_refunded + description: Whether the charge has been fully refunded. If the charge is only partially refunded, this attribute will still be false. + - name: connected_account_id + description: The ID of the account connected to the charge. + - name: customer_id + description: The associated customer reference. - name: customer_description description: Description of the customer. + - name: customer_account_balance + description: Current balance, if any, being stored on the customer. If negative, the customer has credit to apply to their next invoice. If positive, the customer has an amount owed that will be added to their next invoice. + - name: customer_currency + description: Three-letter ISO code for the currency the customer can be charged in for recurring billing purposes. + - name: customer_is_delinquent + description: When the customer’s latest invoice is billed by charging automatically, delinquent is true if the invoice’s latest charge is failed. When the customer’s latest invoice is billed by sending an invoice, delinquent is true if the invoice is not paid by its due date. - name: customer_email description: Email of the customer. - name: subscription_id @@ -343,107 +361,158 @@ models: description: The start date of the subscription - name: subscription_ended_at description: The end date of the subscription - - name: period_start - description: Timestamp of the start of the period. - - name: period_end - description: Timestamp of the end of the period. - - name: customer_id - description: The associated customer reference. - - name: plan_id - description: The ID of the plan of the subscription, if the line item is a subscription or a proration. - - name: plan_is_active - description: Boolean indicating if the plan is active (true) or in-active (false). - - name: plan_amount - description: The unit amount in cents to be charged, represented as a whole integer if possible. - - name: plan_interval - description: The frequency at which a subscription is billed. One of day, week, month or year. - - name: plan_interval_count - description: The number of intervals between subscription billings. For example, interval_count=3 bills every 3 months. - - name: plan_nickname - description: A brief description of the plan, hidden from customers. - - name: plan_product_id - description: Reference to the product whose pricing this plan determines. - - - name: stripe__subscription_line_items - description: Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. + - name: stripe__invoice_line_items + description: Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and pricing data. columns: + - name: invoice_line_item_id + description: The unique Identifier of the invoice line object. + tests: + - unique - name: invoice_id description: The unique Identifier of the invoice object. Note invoices can have many line items, so this value can appear multiple times. - - name: number - description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. + - name: invoice_item_id + description: The ID of the invoice item this item is a part of + - name: invoice_line_item_amount + description: Amount for this line item. + - name: currency + description: Currency of this line item. + - name: invoice_line_item_memo + description: Description for this line item. + - name: is_discountable + description: Whether this line item is discountable. + - name: proration + description: Whether this is a proration. + - name: quantity + description: The quantity of the subscription, if the line item is a subscription or a proration. + - name: subscription_id + description: The ID of the subscription that the invoice item pertains to, if any. + - name: subscription_item_id + description: The subscription item that generated this invoice item. Left empty if the line item is not an explicit result of a subscription. + - name: type + description: A string identifying the type of the source of this line item, either an invoice item or a subscription. + - name: unique_id + description: A unique id generated for old invoice ID's from a past version of the API + - name: period_start + description: Start of the usage period during which invoice items were added to this invoice. + - name: period_end + description: End of the usage period during which invoice items were added to this invoice. - name: invoice_created_at description: Timestamp of when the invoice was created. - - name: status + - name: invoice_status description: Current status of the invoice. - - name: due_date + - name: invoice_due_date description: Date when payment for the invoice is due. - - name: amount_due - description: The amount that the invoice is for. Because this query includes line item details, this amount may be repeated. - - name: subtotal + - name: invoice_amount_due + description: Final amount due at this time for this invoice. If the invoice’s total is smaller than the minimum charge amount, for example, or if there is account credit that can be applied to the invoice, the amount_due may be 0. If there is a positive starting_balance for the invoice (the customer owes money), the amount_due will also take that into account. The charge that gets generated for the invoice will be for the amount specified in amount_due. + - name: invoice_amount_paid + description: The amount, if any, that the customer has paid on the invoice + - name: invoice_subtotal description: The amount of the invoice before discounts and taxes. - - name: tax + - name: invoice_tax description: The amount of tax being charged in the invoice. - - name: total - description: The Total after discounts and taxes. - - name: amount_paid - description: The amount, if any, that the customer has paid on the invoice - - name: amount_remaining - description: The amount of the invoice remaining to be paid. - - name: attempt_count - description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. - - name: invoice_memo - description: An arbitrary string attached to the object. Often useful for displaying to users. - - name: invoice_line_item_id - description: The unique Identifier of the invoice line object. - tests: - - unique - - name: line_item_desc - description: The description of the invoice line item - - name: line_item_amount - description: The amount for the line item. - - name: quantity - description: The quantity of the line item. - - name: balance_transaction_id - description: The ID of the balance transaction object representing payment - - name: charge_amount - description: The amount charged to the customer. - - name: charge_status - description: The status of the charge for the invoice. - - name: charge_created_at - description: When the charge for the invoice was created. - - name: customer_description - description: Description of the customer. - - name: customer_email - description: Email of the customer. - - name: subscription_id - description: ID of the subscription this invoice relates to. + - name: invoice_total + description: The total of the invoice after discounts and taxes. + - name: connected_account_id + description: The ID of the account connected to the charge. + - name: customer_id + description: The associated customer reference. - name: subscription_billing description: How the subscription is billed - name: subscription_start_date description: The start date of the subscription - name: subscription_ended_at description: The end date of the subscription - - name: period_start - description: Timestamp of the start of the period. - - name: period_end - description: Timestamp of the end of the period. - - name: customer_id - description: The associated customer reference. - - name: plan_id - description: The ID of the plan of the subscription, if the line item is a subscription or a proration. - - name: plan_is_active - description: Boolean indicating if the plan is active (true) or in-active (false). - - name: plan_amount + - name: pricing_is_active + description: Boolean indicating if the plan or price is active (true) or in-active (false). + - name: pricing_amount description: The unit amount in cents to be charged, represented as a whole integer if possible. - - name: plan_interval + - name: pricing_interval description: The frequency at which a subscription is billed. One of day, week, month or year. - - name: plan_interval_count + - name: pricing_interval_count description: The number of intervals between subscription billings. For example, interval_count=3 bills every 3 months. - - name: plan_nickname - description: A brief description of the plan, hidden from customers. - - name: plan_product_id - description: Reference to the product whose pricing this plan determines. + - name: pricing_nickname + description: A brief description of the plan or price, hidden from customers. + - name: pricing_product_id + description: Reference to the product whose pricing this price or plan determines. + + # - name: stripe__subscription_line_items + # description: Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. + # columns: + # - name: invoice_id + # description: The unique Identifier of the invoice object. + # tests: + # - unique + # - name: number + # description: A unique, identifying string that appears on emails sent to the customer for this invoice. This starts with the customer’s unique invoice_prefix if it is specified. + # - name: invoice_created_at + # description: Timestamp of when the invoice was created. + # - name: status + # description: Current status of the invoice. + # - name: due_date + # description: Date when payment for the invoice is due. + # - name: amount_due + # description: The amount that the invoice is for. Because this query includes line item details, this amount may be repeated. + # - name: subtotal + # description: The amount of the invoice before discounts and taxes. + # - name: tax + # description: The amount of tax being charged in the invoice. + # - name: total + # description: The Total after discounts and taxes. + # - name: amount_paid + # description: The amount, if any, that the customer has paid on the invoice + # - name: amount_remaining + # description: The amount of the invoice remaining to be paid. + # - name: attempt_count + # description: Number of payment attempts made for this invoice, from the perspective of the payment retry schedule. + # - name: invoice_memo + # description: An arbitrary string attached to the object. Often useful for displaying to users. + # - name: line_item_desc + # description: The description of the invoice line item + # - name: line_item_amount + # description: The amount for the line item. + # - name: quantity + # description: The quantity of the line item. + # - name: balance_transaction_id + # description: The ID of the balance transaction object representing payment + # - name: charge_amount + # description: The amount charged to the customer. + # - name: charge_status + # description: The status of the charge for the invoice. + # - name: charge_created_at + # description: When the charge for the invoice was created. + # - name: customer_description + # description: Description of the customer. + # - name: customer_email + # description: Email of the customer. + # - name: subscription_id + # description: ID of the subscription this invoice relates to. + # - name: subscription_billing + # description: How the subscription is billed + # - name: subscription_start_date + # description: The start date of the subscription + # - name: subscription_ended_at + # description: The end date of the subscription + # - name: period_start + # description: Timestamp of the start of the period. + # - name: period_end + # description: Timestamp of the end of the period. + # - name: customer_id + # description: The associated customer reference. + # - name: plan_id + # description: The ID of the plan of the subscription, if the line item is a subscription or a proration. + # - name: plan_is_active + # description: Boolean indicating if the plan is active (true) or in-active (false). + # - name: plan_amount + # description: The unit amount in cents to be charged, represented as a whole integer if possible. + # - name: plan_interval + # description: The frequency at which a subscription is billed. One of day, week, month or year. + # - name: plan_interval_count + # description: The number of intervals between subscription billings. For example, interval_count=3 bills every 3 months. + # - name: plan_nickname + # description: A brief description of the plan, hidden from customers. + # - name: plan_product_id + # description: Reference to the product whose pricing this plan determines. - name: stripe__subscription_details description: Each record represents a subscription, enriched with customer details and payment aggregations. @@ -460,7 +529,7 @@ models: description: Email of the customer who owns the subscription. - name: status description: Possible values are incomplete, incomplete_expired, trialing, active, past_due, canceled, or unpaid. - - name: start_date + - name: start_date_at description: Date when the subscription was first created. The date might differ from the created date due to backdating. - name: ended_at description: If the subscription has ended, the date the subscription ended. @@ -498,3 +567,7 @@ models: description: The average sum of the line items for invoices generated for this subscription - name: avg_num_invoice_items description: The average number of line items for invoices generated for this subscription + + - name: stripe__account_daily_overview + description: Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions). + # holding off on columns until finalized \ No newline at end of file diff --git a/models/stripe__account_daily_overview.sql b/models/stripe__account_daily_overview.sql new file mode 100644 index 00000000..4f8a64ff --- /dev/null +++ b/models/stripe__account_daily_overview.sql @@ -0,0 +1,55 @@ +{% set total_fields = ['total_daily_sales_amount', 'total_daily_refunds_amount', 'total_daily_adjustments_amount', 'total_daily_other_transactions_amount', 'total_daily_gross_transaction_amount', 'total_daily_net_transactions_amount', 'total_daily_payout_fee_amount', 'total_daily_gross_payout_amount', 'daily_net_activity_amount', 'daily_end_balance_amount', 'total_daily_sales_count', 'total_daily_payouts_count', 'total_daily_adjustments_count', 'total_daily_failed_charge_count', 'total_daily_failed_charge_amount'] %} +{% set rolling_fields = ['rolling_total_daily_sales_amount', 'rolling_total_daily_refunds_amount', 'rolling_total_daily_adjustments_amount', 'rolling_total_daily_other_transactions_amount', 'rolling_total_daily_gross_transaction_amount', 'rolling_total_daily_net_transactions_amount', 'rolling_total_daily_payout_fee_amount', 'rolling_total_daily_gross_payout_amount', 'rolling_daily_net_activity_amount', 'rolling_daily_end_balance_amount', 'rolling_total_daily_sales_count', 'rolling_total_daily_payouts_count', 'rolling_total_daily_adjustments_count', 'rolling_total_daily_failed_charge_count', 'rolling_total_daily_failed_charge_amount'] %} + +with date_spine as ( + + select * + from {{ ref('int_stripe__date_spine') }} + +), account_daily_balances_by_type as ( + + select * + from {{ ref('int_stripe__account_daily')}} + +), account_rolling_totals as ( + + select + * + + {% for t in total_fields %} + , sum({{ t }}) over (partition by account_id order by account_id, date_day rows unbounded preceding) as rolling_{{ t }} + {% endfor %} + + -- , source_relation # add this in upon union_feature merge + + from account_daily_balances_by_type + +), final as ( + + select + coalesce(account_rolling_totals.account_id, date_spine.account_id) as account_id, + coalesce(account_rolling_totals.date_day, date_spine.date_day) as date_day, + + {% for t in total_fields %} + account_rolling_totals.{{ t }}, + {% endfor %} + + {% for f in rolling_fields %} + case when account_rolling_totals.{{ f }} is null and date_index = 1 + then 0 + else account_rolling_totals.{{ f }} + end as {{ f }}, + {% endfor %} + + date_spine.date_index + -- , + -- source_relation # add later! + + from date_spine + left join account_rolling_totals + on account_rolling_totals.account_id = date_spine.account_id + and account_rolling_totals.date_day = date_spine.date_day +) + +select * +from final diff --git a/models/stripe__balance_transactions.sql b/models/stripe__balance_transactions.sql index 4073bdd3..5da367a3 100644 --- a/models/stripe__balance_transactions.sql +++ b/models/stripe__balance_transactions.sql @@ -29,7 +29,7 @@ with balance_transaction as ( select * from {{ var('customer')}} -{% if var('using_payment_method', True) %} +{% if var('stripe__using_payment_method', True) %} ), payment_method as ( @@ -72,11 +72,12 @@ select case when balance_transaction.type = 'charge' then charge.amount end as customer_facing_amount, case when balance_transaction.type = 'charge' then charge.currency end as customer_facing_currency, {{ dbt.dateadd('day', 1, 'balance_transaction.available_on') }} as effective_at, + coalesce(balance_transaction.connected_account_id, charge.connected_account_id) as account_id, coalesce(charge.customer_id, refund_charge.customer_id) as customer_id, charge.receipt_email, customer.description as customer_description, - {% if var('using_payment_method', True) %} + {% if var('stripe__using_payment_method', True) %} payment_method.type as payment_method_type, payment_method_card.brand as payment_method_brand, payment_method_card.funding as payment_method_funding, @@ -96,14 +97,14 @@ select refund.reason as refund_reason from balance_transaction -left join charge +left join charge on charge.balance_transaction_id = balance_transaction.balance_transaction_id left join customer on charge.customer_id = customer.customer_id left join payment_intent on charge.payment_intent_id = payment_intent.payment_intent_id -{% if var('using_payment_method', True) %} +{% if var('stripe__using_payment_method', True) %} left join payment_method on payment_intent.payment_method_id = payment_method.payment_method_id left join payment_method_card diff --git a/models/stripe__customer_overview.sql b/models/stripe__customer_overview.sql index ab1461c9..63ff9e54 100644 --- a/models/stripe__customer_overview.sql +++ b/models/stripe__customer_overview.sql @@ -14,7 +14,7 @@ with balance_transaction_joined as ( from {{ var('customer') }} ), transactions_grouped as ( - + select customer_id, sum(case when type in ('charge', 'payment') diff --git a/models/stripe__invoice_details.sql b/models/stripe__invoice_details.sql new file mode 100644 index 00000000..f47b0fd6 --- /dev/null +++ b/models/stripe__invoice_details.sql @@ -0,0 +1,113 @@ +{{ config(enabled=var('stripe__using_invoices', True)) }} + +with invoice as ( + + select * + from {{ var('invoice') }} + +), charge as ( + + select * + from {{ var('charge') }} + +), invoice_line_item as ( + + select + invoice_id, + source_relation, + plan_id, + price_id, + count(distinct unique_id) as number_of_line_items, + sum(quantity) as total_quantity + + from {{ var('invoice_line_item') }} + group by 1,2,3,4 + +), customer as ( + + select * + from {{ var('customer') }} + +{% if var('stripe__using_subscriptions', True) %} + +), subscription as ( + + select * + from {{ var('subscription') }} + +), pricing as ( + + select * + from {{ var('pricing') }} + +{% endif %} +) + +select + invoice.invoice_id, + invoice.number as invoice_number, + invoice.created_at as invoice_created_at, + invoice.period_start, + invoice.period_end, + invoice.status, + invoice.due_date, + invoice.currency, + invoice.amount_due, + invoice.amount_paid, + invoice.subtotal, + invoice.tax, + invoice.total, + invoice.amount_remaining, + invoice.attempt_count, + invoice.description as invoice_memo, + invoice_line_item.number_of_line_items, + invoice_line_item.total_quantity, + charge.balance_transaction_id, + charge.amount as charge_amount, + charge.status as charge_status, + charge.connected_account_id, + charge.created_at as charge_created_at, + charge.is_refunded as charge_is_refunded, + customer.customer_id, + customer.description as customer_description, + customer.account_balance as customer_account_balance, + customer.currency as customer_currency, + customer.is_delinquent as customer_is_delinquent, + customer.email as customer_email, + + {% if var('stripe__using_subscriptions', True) %} + subscription.subscription_id, + subscription.billing as subscription_billing, + subscription.start_date_at as subscription_start_date, + subscription.ended_at as subscription_ended_at, + + {% endif %} + + {% if var('stripe__invoice_metadata',[]) %} + {{ fivetran_utils.pivot_json_extract(string = 'metadata', list_of_properties = var('stripe__invoice_metadata')) }}, + {% endif %} + + invoice.source_relation + +from invoice + +left join invoice_line_item + on invoice.invoice_id = invoice_line_item.invoice_id + and invoice.source_relation = invoice_line_item.source_relation + +left join charge + on charge.charge_id = invoice.charge_id + and charge.invoice_id = invoice.invoice_id + and charge.source_relation = invoice.source_relation + +{% if var('stripe__using_subscriptions', True) %} +left join subscription + on invoice.subscription_id = subscription.subscription_id + and invoice.source_relation = subscription.source_relation + +{% endif %} + + +left join customer + on invoice.customer_id = customer.customer_id + and invoice.source_relation = customer.source_relation \ No newline at end of file diff --git a/models/stripe__invoice_line_items.sql b/models/stripe__invoice_line_items.sql index 2b189336..5f83f4de 100644 --- a/models/stripe__invoice_line_items.sql +++ b/models/stripe__invoice_line_items.sql @@ -1,96 +1,91 @@ -{{ config(enabled=var('using_invoices', True)) }} +{{ config(enabled=var('stripe__using_invoices', True)) }} -with invoice as ( +with invoice_line_item as ( select * - from {{ var('invoice') }} + from {{ var('invoice_line_item') }} -), charge as ( +), invoice_details as ( select * - from {{ var('charge') }} - -), invoice_line_item as ( - - select * - from {{ var('invoice_line_item') }} - -), customer as ( - - select * - from {{ var('customer') }} - -{% if var('using_subscriptions', True) %} + from {{ ref('stripe__invoice_details') }} +{% if var('stripe__using_subscriptions', True) %} ), subscription as ( select * from {{ var('subscription') }} -), plan as ( +), pricing as ( select * - from {{ var('plan') }} + from {{ var('pricing') }} {% endif %} ) select - invoice.invoice_id, - invoice.number, - invoice.created_at as invoice_created_at, - invoice.status, - invoice.due_date, - invoice.currency, - invoice.amount_due, - invoice.subtotal, - invoice.tax, - invoice.total, - invoice.amount_paid, - invoice.amount_remaining, - invoice.attempt_count, - invoice.description as invoice_memo, - invoice_line_item.unique_id as invoice_line_item_id, - invoice_line_item.description as line_item_desc, - invoice_line_item.amount as line_item_amount, + invoice_line_item.invoice_line_item_id, + invoice_line_item.invoice_id, + invoice_line_item.invoice_item_id, + invoice_line_item.amount as invoice_line_item_amount, + invoice_line_item.currency, + invoice_line_item.description as invoice_line_item_memo, + invoice_line_item.is_discountable, + invoice_line_item.plan_id, + invoice_line_item.price_id, + invoice_line_item.proration, invoice_line_item.quantity, + invoice_line_item.subscription_id, + invoice_line_item.subscription_item_id, + invoice_line_item.type, + invoice_line_item.unique_id, invoice_line_item.period_start, invoice_line_item.period_end, - charge.balance_transaction_id, - charge.amount as charge_amount, - charge.status as charge_status, - charge.created_at as charge_created_at, - customer.description as customer_description, - customer.email as customer_email, - customer.customer_id - - {% if var('using_subscriptions', True) %} - ,subscription.subscription_id, + invoice_details.invoice_created_at, + invoice_details.status as invoice_status, + invoice_details.due_date as invoice_due_date, + invoice_details.amount_due as invoice_amount_due, + invoice_details.amount_paid as invoice_amount_paid, + invoice_details.subtotal as invoice_subtotal, + invoice_details.tax as invoice_tax, + invoice_details.total as invoice_total, + invoice_details.connected_account_id as connected_account_id, + invoice_details.customer_id as customer_id + + {% if var('stripe__using_subscriptions', True) %} + , subscription.billing as subscription_billing, - subscription.start_date as subscription_start_date, + subscription.start_date_at as subscription_start_date, subscription.ended_at as subscription_ended_at, - plan.plan_id, - plan.is_active as plan_is_active, - plan.amount as plan_amount, - plan.plan_interval as plan_interval, - plan.interval_count as plan_interval_count, - plan.nickname as plan_nickname, - plan.product_id as plan_product_id + pricing.is_active as pricing_is_active, + pricing.unit_amount as pricing_amount, + pricing.recurring_interval as pricing_interval, + pricing.recurring_interval_count as pricing_interval_count, + pricing.nickname as pricing_nickname, + pricing.product_id as pricing_product_id {% endif %} -from invoice +from invoice_line_item -left join charge - on charge.charge_id = invoice.charge_id -left join invoice_line_item - on invoice.invoice_id = invoice_line_item.invoice_id +left join invoice_details + on invoice_line_item.invoice_id = invoice_details.invoice_id + and invoice_line_item.source_relation = invoice_details.source_relation -{% if var('using_subscriptions', True) %} -left join subscription +{% if var('stripe__using_subscriptions', True) %} + +left join subscription on invoice_line_item.subscription_id = subscription.subscription_id -left join plan - on invoice_line_item.plan_id = plan.plan_id -{% endif %} + and invoice_line_item.source_relation = subscription.source_relation + +left join pricing + {% if var('stripe__using_price', does_table_exist('price')) %} + on invoice_line_item.price_id = pricing.price_id + + {% else %} + on invoice_line_item.plan_id = pricing.plan_id + + {% endif %} + and invoice_line_item.source_relation = pricing.source_relation -left join customer - on invoice.customer_id = customer.customer_id +{% endif %} \ No newline at end of file diff --git a/models/stripe__subscription_details.sql b/models/stripe__subscription_details.sql index 215cb183..fc1df9a1 100644 --- a/models/stripe__subscription_details.sql +++ b/models/stripe__subscription_details.sql @@ -1,4 +1,4 @@ -{{ config(enabled=fivetran_utils.enabled_vars(['using_invoices','using_subscriptions'])) }} +{{ config(enabled=fivetran_utils.enabled_vars(['stripe__using_invoices','stripe__using_subscriptions'])) }} with invoice as ( @@ -65,7 +65,7 @@ select customer.description as customer_description, customer.email as customer_email, subscription.status, - subscription.start_date, + subscription.start_date_at, subscription.ended_at, subscription.billing, subscription.billing_cycle_anchor, diff --git a/models/stripe__subscription_line_items.sql b/models/stripe__subscription_line_items.sql deleted file mode 100644 index 997f3838..00000000 --- a/models/stripe__subscription_line_items.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config(enabled=fivetran_utils.enabled_vars(['using_invoices','using_subscriptions'])) }} - - -with line_items as ( - - select * - from {{ ref('stripe__invoice_line_items') }} - where subscription_id is not null - -) - -select * -from line_items diff --git a/packages.yml b/packages.yml index 08e0f0e9..852a78c1 100644 --- a/packages.yml +++ b/packages.yml @@ -2,5 +2,5 @@ packages: # - package: fivetran/stripe_source # version: [">=0.8.0", "<0.9.0"] - git: https://github.com/fivetran/dbt_stripe_source.git - revision: MagicBot/dbt-utils-cross-db-migration + revision: stripe_updates warn-unpinned: false