Merge pull request #55 from fivetran/new_models

add new models

.buildkite/scripts/run_models.sh

@@ -19,5 +19,5 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{using_invoices: false, using_payment_method: false, using_subscriptions: false, stripe_timezone: "America/New_York", stripe__subscription_history: true}' --target "$db"
+dbt run --vars '{stripe__using_invoices: false, stripe__using_payment_method: false, stripe__using_subscriptions: false, stripe_timezone: "America/New_York", stripe__using_subscription_history: false, stripe__using_price: false}' --target "$db"
 dbt test --target "$db"

CHANGELOG.md

@@ -1,3 +1,15 @@
+# dbt_stripe v0.9.0
+
+
+[PR #55](https://github.com/fivetran/dbt_stripe/pull/55): 
+## 🎉 Feature Updates
+- New models `stripe__account_daily_overview` and `stripe__invoice_details` have been added.
+- `subscription_item_id` has been added to the `stripe__invoice_line_items` model.
+
+## 🚨 Breaking Changes 🚨:
+- `stripe__subscription_line_items` has been removed. To recreate it, simply filter `stripe__invoice_line_items` for where `subscription_id` is not null.
+
+
 # dbt_stripe v0.8.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -24,9 +24,10 @@ The following table provides a detailed list of all models materialized within t
 | **model**                          | **description**                                                                                                                                                                                                                              |
 |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [stripe__balance_transactions](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__balance_transactions)    | Each record represents a change to your account balance, enriched with data about the transaction.                                                                                                                                       |
-| [stripe__invoice_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items)      | Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and plan.                                                                                                        |
+| [stripe__invoice_details](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_details)      | Each record represents an invoice, enriched with details about the associated charge, customer, and subscription data.                                                                                         
+| [stripe__invoice_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items)      | Each record represents an invoice line item, enriched with details about the associated charge, customer, subscription, and pricing data.                                                                                 
+| [stripe__account_daily_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__invoice_line_items)      | Each record represents, per account per day, a summary of daily totals and rolling totals by transaction type (balances, payments, refunds, payouts, and other transactions).                                                  |
 | [stripe__subscription_details](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__subscription_details)    | Each record represents a subscription, enriched with customer details and payment aggregations.                                                                                                                                          |
-| [stripe__subscription_line_items](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__subscription_line_items) | Each record represents a subscription invoice line item, enriched with details about the associated charge, customer, subscription, and plan. Use this table as the starting point for your company-specific churn and MRR calculations. |
 | [stripe__customer_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__customer_overview)       | Each record represents a customer, enriched with metrics about their associated transactions.  Transactions with no associated customer will have a customer description of "No associated customer".                                                                                                                                          |
 | [stripe__daily_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__daily_overview)          | Each record represents a single day, enriched with metrics about balances, payments, refunds, payouts, and other transactions.                                                                                                           |
 | [stripe__weekly_overview](https://fivetran.github.io/dbt_stripe/#!/model/model.stripe.stripe__weekly_overview)         | Each record represents a single week, enriched with metrics about balances, payments, refunds, payouts, and other transactions.                                                                                                          |
@@ -55,7 +56,7 @@ Include the following stripe package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/stripe
-    version: [">=0.8.0", "<0.9.0"]
+    version: [">=0.9.0", "<0.10.0"]
 
 ```
 ## Step 3: Define database and schema variables
@@ -68,21 +69,52 @@ vars:
 ```
 
 ## Step 4: Disable models for non-existent sources
-This package takes into consideration that not every Stripe account utilizes the `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, or `subscription` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable within your root `dbt_project.yml`:
+This package takes into consideration that not every Stripe account utilizes the **invoices**, **payment method**, and **subscription** features. Therefore we allow you to configure the following variables below, which will then disable the corresponding related tables: `invoice`, `invoice_line_item`, `payment_method`, `payment_method_card`, `plan`, `price`, or `subscription`. The `plan` and `price` tables are toggled automatically (see *Step 6: Leveraging Plan vs Price Sources*)
+
+By default, all variables' values are assumed to be `true`. Only add variables within your root `dbt_project.yml` for only the tables you would want to disable:
+
 ```yml
 vars:
-    using_invoices:        False  #Disable if you are not using the invoice and invoice_line_item tables
-    using_payment_method:  False  #Disable if you are not using the payment_method and payment_method_card tables
-    using_subscriptions:   False  #Disable if you are not using the subscription and plan tables.
+    stripe__using_invoices:        False  #Disable if you are not using the invoice and invoice_line_item tables
+    stripe__using_payment_method:  False  #Disable if you are not using the payment_method and payment_method_card tables
+    stripe__using_subscriptions:   False  #Disable if you are not using the subscription and plan/pricing tables.
 ```
 ## Step 5: Leveraging Subscription Vs Subscription History Sources
-For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription` source table. However, if you have a newer connector then you must leverage the `stripe__subscription_history` to have the package use the `subscription_history` source rather than the `subscription` table.
+For Stripe connectors set up after February 09, 2022 the `subscription` table has been replaced with the new `subscription_history` table. By default this package will look for your subscription data within the `subscription_history` source table. However, if you have a older connector then you must leverage the `stripe__using_subscription_history` toggle to have the package use the `subscription` source rather than the `subscription_history` table.
 > **Please note that if you have `stripe__subscription_history` enabled then the package will filter for only active records.**
 ```yml
 vars:
-    stripe__subscription_history: True  # False by default. Set to True if your connector syncs the `subscription_history` table. 
+    stripe__usingsubscription_history: False  # True by default. Set to False if your connector syncs the `subscription_history` table instead. 
+```
+## Step 6: Leveraging Plan vs Price Sources
+
+Customers using Fivetran with the newest Stripe pricing model will have a `price` table in place of the older `plan` table. Therefore to accommodate two different source tables we added additional logic in the `stg_stripe__pricing` model, which replaces the `stg_stripe__plan` model. This model checks if there exists a `price` table using a new `does_table_exist()` macro. If not, it will look for a `plan` table. While the default is to use the `price` table if it exists, you may add the following to your `dbt_project.yml` to override using the macro. 
+
+```yml
+# dbt_project.yml
+
+...
+config-version: 2
+
+vars:
+  stripe:
+    stripe__using_price: false #  True by default. If true, will look `price ` table. If false, will look for the `plan` table. 
+```
+## Step 7: Unioning Multiple Stripe Connectors
+If you have multiple Stripe connectors you would like to use this package on simultaneously, we have added the ability to do so. Data from disparate connectors will be unioned together and be passed downstream to the end models. The `source_relation` column will specify where each record comes from. To use this functionality, you will need to either set the `stripe_union_schemas` or `stripe_union_databases` variables. Please also make sure the single-source `stripe_database` and `stripe_schema` variables are removed.
+
+```yml
+# dbt_project.yml
+
+...
+config-version: 2
+
+vars:
+    stripe_union_schemas: ['stripe_us','stripe_mx'] # use this if the data is in different schemas/datasets of the same database/project
+    stripe_union_databases: ['stripe_db_1','stripe_db_2'] # use this if the data is in different databases/projects but uses the same schema name
 ```
-## (Optional) Step 6: Additional configurations
+
+## (Optional) Step 8: Additional configurations
 <details><summary>Expand for configurations</summary>
 
 ### Setting your timezone

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 name: 'stripe'
 
-version: '0.8.0'
+version: '0.9.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   stripe:
@@ -11,6 +11,7 @@ models:
       +materialized: ephemeral
 vars:
   stripe:
+    account: "{{ ref('stg_stripe__account') }}"
     balance_transaction: "{{ ref('stg_stripe__balance_transaction') }}"
     card: "{{ ref('stg_stripe__card') }}"
     charge: "{{ ref('stg_stripe__charge') }}"
@@ -22,9 +23,9 @@ vars:
     payment_method_card: "{{ ref('stg_stripe__payment_method_card') }}"
     payment_method: "{{ ref('stg_stripe__payment_method') }}"
     payout: "{{ ref('stg_stripe__payout') }}"
-    plan: "{{ ref('stg_stripe__plan') }}"
+    pricing: "{{ ref('stg_stripe__pricing') }}"
     refund: "{{ ref('stg_stripe__refund') }}"
     subscription: "{{ ref('stg_stripe__subscription') }}"
-    using_invoices: true
-    using_payment_method: true
-    using_subscriptions: true
+    stripe__using_invoices: true
+    stripe__using_payment_method: true
+    stripe__using_subscriptions: true

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: stripe_integrations_tests
+      schema: stripe_integrations_tests_2
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: stripe_integrations_tests
+      schema: stripe_integrations_tests_2
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: stripe_integrations_tests
+      schema: stripe_integrations_tests_2
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: stripe_integrations_tests
+      schema: stripe_integrations_tests_2
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: stripe_integrations_tests
+      schema: stripe_integrations_tests_2
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -6,8 +6,9 @@ version: '0.8.0'
 profile: 'integration_tests'
 
 vars:
-  stripe_schema: stripe_integrations_tests
+  stripe_schema: stripe_integrations_tests_2
   stripe_source:
+    stripe_account_identifier:                "account_data" 
     stripe_group_identifier:                  "group_data"
     stripe_balance_transaction_identifier:    "balance_transaction_data"
     stripe_card_identifier:                   "card_data"
@@ -21,6 +22,7 @@ vars:
     stripe_payment_method_identifier:         "payment_method_data"
     stripe_payout_identifier:                 "payout_data"
     stripe_plan_identifier:                   "plan_data"
+    stripe_price_identifier:                  "price_data"
     stripe_refund_identifier:                 "refund_data"
     stripe_subscription_history_identifier:   "subscription_history_data"
     stripe_subscription_identifier:           "subscription_data"
@@ -34,13 +36,17 @@ seeds:
     +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}"
     +column_types:
       _fivetran_synced: timestamp
+      created: timestamp
+      voided_at: timestamp
+      canceled_at: timestamp
+      invoice_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
     balance_transaction_data:
       +column_types:
         created: timestamp
         available_on: timestamp
     charge_data:
       +column_types:
-        balance_transaction_id: "{{ 'varchar(250)' if target.type in ('redshift','postgres') else 'string'}}"
+        balance_transaction_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
         created: timestamp
     customer_data:
       +column_types:

integration_tests/seeds/account_data.csv

@@ -0,0 +1,2 @@
+id,_fivetran_synced,business_profile_mcc,business_profile_name,business_profile_product_description,business_profile_support_address_city,business_profile_support_address_country,business_profile_support_address_line_1,business_profile_support_address_line_2,business_profile_support_address_postal_code,business_profile_support_address_state,business_profile_support_email,business_profile_support_phone,business_profile_support_url,business_profile_url,business_type,capabilities_afterpay_clearpay_payments,capabilities_au_becs_debit_payments,capabilities_bacs_debit_payments,capabilities_bancontact_payments,capabilities_card_issuing,capabilities_card_payments,capabilities_cartes_bancaires_payments,capabilities_eps_payments,capabilities_fpx_payments,capabilities_giropay_payments,capabilities_grabpay_payments,capabilities_ideal_payments,capabilities_jcb_payments,capabilities_legacy_payments,capabilities_oxxo_payments,capabilities_p_24_payments,capabilities_platform_payments,capabilities_sepa_debit_payments,capabilities_sofort_payments,capabilities_tax_reporting_us_1099_k,capabilities_tax_reporting_us_1099_misc,capabilities_transfers,charges_enabled,company_address_city,company_address_country,company_address_kana_city,company_address_kana_country,company_address_kana_line_1,company_address_kana_line_2,company_address_kana_postal_code,company_address_kana_state,company_address_kana_town,company_address_kanji_city,company_address_kanji_country,company_address_kanji_line_1,company_address_kanji_line_2,company_address_kanji_postal_code,company_address_kanji_state,company_address_kanji_town,company_address_line_1,company_address_line_2,company_address_postal_code,company_address_state,company_directors_provided,company_executives_provided,company_name,company_name_kana,company_name_kanji,company_owners_provided,company_phone,company_structure,company_tax_id_provided,company_tax_id_registrar,company_vat_id_provided,company_verification_document_back,company_verification_document_details,company_verification_document_details_code,company_verification_document_front,country,created,default_currency,details_submitted,email,individual_id,is_deleted,metadata,payouts_enabled,requirements_current_deadline,requirements_currently_due,requirements_disabled_reason,requirements_errors,requirements_eventually_due,requirements_past_due,requirements_pending_verification,settings_branding_icon,settings_branding_logo,settings_branding_primary_color,settings_card_payments_decline_on_avs_failure,settings_card_payments_decline_on_cvc_failure,settings_card_payments_statement_descriptor_prefix,settings_dashboard_display_name,settings_dashboard_timezone,settings_payments_statement_descriptor,settings_payments_statement_descriptor_kana,settings_payments_statement_descriptor_kanji,settings_payouts_debit_negative_balances,settings_payouts_schedule_delay_days,settings_payouts_schedule_interval,settings_payouts_schedule_monthly_anchor,settings_payouts_schedule_weekly_anchor,settings_payouts_statement_descriptor,tos_acceptance_date,tos_acceptance_ip,tos_acceptance_user_agent,type
+1234,2022-12-06 14:19:07,737,Hogwarts School of Witchcraft and Wizardry,,Hogwarts,US,123 Street,Apt 123,12345,CA,[email protected],,https://www.wizardingworld.com/,https://www.wizardingworld.com/,,active,,,active,,active,,active,,active,,active,,,,active,active,active,active,,,,TRUE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,US,,usd,TRUE,[email protected],,FALSE,,TRUE,,,,,,,,,file_asdfasgas,#0030ff,,,,Hogwarts Inc.,America/Los_Angeles,hogwarts.com,,,TRUE,7,daily,,,,,,,standard

integration_tests/seeds/invoice_data.csv

@@ -1,3 +1,3 @@
 id,_fivetran_synced,amount,currency,customer_id,date,description,discountable,invoice_id,is_deleted,livemode,period_end,period_start,proration,quantity,subscription_id,subscription_item_id,unit_amount
-sdfg54545,2021-02-18 14:16:37,45,usd,sdgfs84854,,description_here,TRUE,sdg545641,FALSE,TRUE,2021-02-19 14:16:37,2021-02-18 14:16:37,TRUE,45,sdkhg54645,kdshg455,5
+ab1111,2021-02-18 14:16:37,45,usd,sdgfs84854,,description_here,TRUE,sdg545641,FALSE,TRUE,2021-02-19 14:16:37,2021-02-18 14:16:37,TRUE,45,sdkhg54645,kdshg455,5
 si8jo54,2021-02-18 14:16:37,22,usd,dfd54654,,description_here,TRUE,dgjksg6486,FALSE,FALSE,2021-02-19 14:16:37,2021-02-18 14:16:37,FALSE,55,dghisd165748,sgdfs5464,6