Merge pull request #78 from fivetran/release/v0.13.0

Release/v0.13.0

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

.gitignore

@@ -3,4 +3,5 @@ target/
 dbt_modules/
 logs/
 env/
-dbt_packages/
+dbt_packages/
+package-lock.yml

CHANGELOG.md

@@ -1,3 +1,17 @@
+# dbt_stripe v0.13.0
+[PR #78](https://github.com/fivetran/dbt_stripe/pull/78) includes the following updates:
+
+## 🚨 Breaking Changes 🚨
+- Renamed folder `stripe_reports` to `stripe_financial_reports` to be more descriptive of the contents.
+  - ⚠️ If you are using folder names to scope out configs, runs, etc., you will need to update the folder name.
+
+## Bug fixes
+- Updated model `int_stripe__date_spine` to accommodate cases when model `stripe__balance_transactions` has no records. 
+  - Previously, the date spine relied on `stripe__balance_transactions` to set date bounds, which caused errors if empty. Now, the model defaults to a one-month range in such cases.
+
+## Under the hood
+- Updated structure of model `int_stripe__date_spine` for improved performance and maintainability.
+
 # dbt_stripe v0.12.0
 [PR #75](https://github.com/fivetran/dbt_stripe/pull/75) includes the following updates:
 

README.md

@@ -61,7 +61,7 @@ Include the following stripe package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/stripe
-    version: [">=0.12.0", "<0.13.0"]
+    version: [">=0.13.0", "<0.14.0"]
 
 ```
 Do **NOT** include the `stripe_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well. 

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 name: 'stripe'
 
-version: '0.12.0'
+version: '0.13.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   stripe:

integration_tests/.gitignore

@@ -2,4 +2,5 @@
 target/
 dbt_modules/
 logs/
-.DS_Store
+.DS_Store
+package-lock.yml

integration_tests/ci/sample.profiles.yml

@@ -45,10 +45,10 @@ integration_tests:
       schema: stripe_integrations_tests_11
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: stripe_integrations_tests_11
-      threads: 2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,14 @@
 config-version: 2
 
 name: 'stripe_integration_tests'
-version: '0.12.0'
+version: '0.13.0'
 
 profile: 'integration_tests'
 
+# For use with validations
+models:
+  +schema: "stripe_{{ var('directed_schema','dev') }}"
+
 vars:
   stripe_schema: stripe_integrations_tests_11
   stripe_source:

integration_tests/requirements.txt

@@ -1,9 +1,7 @@
-dbt-snowflake>=1.3.0,<2.0.0
-dbt-bigquery>=1.3.0,<2.0.0
-dbt-redshift>=1.3.0,<2.0.0
-dbt-postgres>=1.3.0,<2.0.0
-dbt-spark>=1.3.0,<2.0.0
-dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
-
-oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437
+dbt-snowflake>=1.3.0,<1.8.0
+dbt-bigquery>=1.3.0,<1.8.0
+dbt-redshift>=1.3.0,<1.8.0
+dbt-postgres>=1.3.0,<1.8.0
+dbt-spark>=1.3.0,<1.8.0
+dbt-spark[PyHive]>=1.3.0,<1.8.0
+dbt-databricks>=1.3.0,<1.8.0

integration_tests/tests/consistency/consistency_daily_overview.sql

@@ -0,0 +1,39 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the daily_overview end model matches the prior version
+-- and is aggregated on the date_index grain since the rollings totals will cause variation at the account_id grain
+-- the below iterates through the prod and dev names to reduce redudancy of logic
+with
+{% for prod_or_dev in ('prod', 'dev') %}
+    {% set cols = adapter.get_columns_in_relation(ref('stripe__daily_overview')) %}
+    {{ prod_or_dev }} as (
+        select
+            date_index,
+            source_relation
+            {% for col in cols if col.name not in ["account_id", "account_daily_id", "date_day", "date_week", "date_month", "date_year", "date_index", "source_relation"] %}
+                , floor(sum({{ col.name }})) as summed_{{ col.name }} -- floor and sum is to keep consistency between dev and prod aggs
+            {% endfor %}
+        from {{ target.schema }}_stripe_{{ prod_or_dev }}.stripe__daily_overview
+        group by 1,2 -- need to group to remove randomization stemming from rolling totals
+    ),
+{% endfor %}
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

integration_tests/tests/consistency/consistency_daily_overview_count.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure the rows counts are the same between versions
+with prod as (
+    select count(*) as prod_rows
+    from {{ target.schema }}_stripe_prod.stripe__daily_overview
+),
+
+dev as (
+    select count(*) as dev_rows
+    from {{ target.schema }}_stripe_dev.stripe__daily_overview
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from prod
+join dev
+    on prod.prod_rows != dev.dev_rows

integration_tests/tests/integrity/integrity_daily_overview.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure there is no fanout between the spine and the daily_overview
+with spine as (
+    select count(*) as spine_count
+    from {{ target.schema }}_stripe_dev.int_stripe__date_spine
+),
+
+daily_overview as (
+    select count(*) as daily_overview_count
+    from {{ target.schema }}_stripe_dev.stripe__daily_overview
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from spine
+join daily_overview
+    on spine.spine_count != daily_overview.daily_overview_count

models/intermediate/int_stripe__account_daily.sql

@@ -100,4 +100,4 @@ select
 from daily_account_balance_transactions
 left join daily_failed_charges
     on daily_account_balance_transactions.date_day = daily_failed_charges.date
-    and daily_account_balance_transactions.source_relation = daily_failed_charges.source_relation
+    and daily_account_balance_transactions.source_relation = daily_failed_charges.source_relation

models/intermediate/int_stripe__date_spine.sql

@@ -1,62 +1,43 @@
-
+-- depends_on: {{ ref('stripe__balance_transactions') }}
 with spine as (
 
     {% if execute %}
-    {% set first_date_query %}
-        select  min( balance_transaction_created_at ) as min_date from {{ ref('stripe__balance_transactions') }}
-    {% endset %}
-    {% set first_date = run_query(first_date_query).columns[0][0]|string %}
-
-        {% if target.type == 'postgres' %}
-            {% set first_date_adjust = "cast('" ~ first_date[0:10] ~ "' as date)" %}
 
-        {% else %}
-            {% set first_date_adjust = "'" ~ first_date[0:10] ~ "'" %}
+    {%- set first_date_query %}
+        select coalesce(
+            min(cast(balance_transaction_created_at as date)), 
+            cast({{ dbt.dateadd("month", -1, "current_date") }} as date)
+            ) as min_date
+        from {{ ref('stripe__balance_transactions') }}
+    {% endset -%}
 
-        {% endif %}
-
-    {% else %} {% set first_date_adjust = "'2009-01-01'" %}
-    {% endif %}
+    {%- set first_date = dbt_utils.get_single_value(first_date_query) %}
 
-    {% if execute %}
     {% set last_date_query %}
-        select  max( balance_transaction_created_at ) as max_date from {{ ref('stripe__balance_transactions') }}
-    {% endset %}
-
-    {% set current_date_query %}
-        select current_date
+        select coalesce(
+            greatest(max(cast(balance_transaction_created_at as date)), cast(current_date as date)),
+            cast(current_date as date)
+            ) as max_date
+        from {{ ref('stripe__balance_transactions') }}
     {% endset %}
 
-    {% if run_query(current_date_query).columns[0][0]|string < run_query(last_date_query).columns[0][0]|string %}
-
-    {% set last_date = run_query(last_date_query).columns[0][0]|string %}
-
-    {% else %} {% set last_date = run_query(current_date_query).columns[0][0]|string %}
-    {% endif %}
-
-    {% if target.type == 'postgres' %}
-        {% set last_date_adjust = "cast('" ~ last_date[0:10] ~ "' as date)" %}
+    {% set last_date = dbt_utils.get_single_value(last_date_query) %}
 
     {% else %}
-        {% set last_date_adjust = "'" ~ last_date[0:10] ~ "'" %}
 
-    {% endif %}
+    {% set first_date = 'dbt.dateadd("month", -1, "current_date")' %}
+    {% set last_date = 'dbt.current_timestamp_backcompat()' %}
+
     {% endif %}
 
     {{ dbt_utils.date_spine(
         datepart="day",
-        start_date=first_date_adjust,
-        end_date=dbt.dateadd("day", 1, last_date_adjust)
+        start_date="cast('" ~ first_date ~ "' as date)",
+        end_date=dbt.dateadd("day", 1, "cast('" ~ last_date  ~ "' as date)")
         )
     }}
 ),
 
-balance_transactions as (
-
-    select *
-    from {{ ref('stripe__balance_transactions') }}
-),
-
 account as (
 
     select *