Merge pull request #11 from fivetran/feature/v2_updates

Feature/v2 updates

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,7 @@
 contact_links:
+  - name: Provide feedback to our dbt package team
+    url: https://www.surveymonkey.com/r/DQ7K7WW
+    about: Fill out our survey form to provide valuable feedback to the Fivetran team developing and maintaining the dbt packages.
   - name: Ask a question during our office hours
     url: https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours
     about: Schedule time during the external office hours block with the Fivetran Analytics Engineering team for support

CHANGELOG.md

@@ -1,3 +1,35 @@
+# dbt_snapchat_ads v0.4.0
+PR [#11](https://github.com/fivetran/dbt_snapchat_ads/pull/11) applies the Ad Reporting V2 updates:
+
+## 🚨 Breaking Changes 🚨
+- Changes `snapchat_schema` and `snapchat_database` variable names to `snapchat_ads_schema` and `snapchat_ads_database` 
+- Updates model names to prefix with `snapchat_ads` and removes the `ad_adapter` model and dependencies on it
+## 🎉 Feature Enhancements 🎉
+
+- Addition of the following new end models. These models were added to provide further flexibility and ensure greater accuracy of your Snapchat Ads reporting. Additionally, these new end models will be leveraged in the respective downstream [dbt_ad_reporting](https://github.com/fivetran/dbt_ad_reporting) models.
+  - `snapchat_ads__ads_report`
+    - Each record in this table represents the daily performance at the ads level.
+  - `snapchat_ads__creative_report`
+    - Each record in this table represents the daily performance at the campaign level.
+  - `snapchat_ads__url_report`
+    - Each record in this table represents the daily performance at the url level.
+
+- Applies README standardization updates
+- Introduces the identifier variable for all source models
+- Casts all timestamp fields using dbt_utils.type_timestamp() and rounds all monetary fields
+- Inclusion of passthrough metrics:
+  - `snapchat_ads__ad_hourly_passthrough_metrics`
+  - `snapchat_ads__ad_squad_hourly_passthrough_metrics`
+  - `snapchat_ads__campaign_hourly_report_passthrough_metrics`
+> This applies to all passthrough columns within the `dbt_snapchat_ads_source` package and not just the `snapchat_ads__ad_hourly_passthrough_metrics` example.
+```yml
+vars:
+  snapchat_ads__ad_hourly_passthrough_metrics:
+    - name: "my_field_to_include" # Required: Name of the field within the source.
+      alias: "field_alias" # Optional: If you wish to alias the field within the staging model.
+```
+- Add enable configs for this specific ad platform, for use in the Ad Reporting rollup package 
+
 # dbt_snapchat_ads v0.3.1
 🎉 Fix creative_id bug [issue]](https://github.com/fivetran/dbt_snapchat_ads/issues/8) 🎉
 

DECISIONLOG.md

@@ -0,0 +1,8 @@
+# Snapchat Ads Decision Log
+## Ads Associated with Multiple Ad Groups and Multiple Campaigns
+An ad can be associated with multiple ad squads and therefore multiple campaigns on any given day. Therefore, `snapchat_ads__ad_report` doesn't contain ad_squad or campaign fields since it reports on an ad level. Similarly, `snapchat_ads__campaign_report` and `snapchat_ads__ad_squad_report` do not contain ad fields. This prevents overattributing performance to just one ad.
+
+## UTM Filtering
+This package contains a `snapchat_ads__url_report` which provides daily metrics for your utm compatible ads. It is important to note that not all Ads leverage utm parameters. Therefore, this package takes an opinionated approach to filter out any records that do not contain utm parameters or leverage a url within the ad.
+
+If you would like to leverage a report that contains all ads and their daily metrics, I would suggest you leverage the snpapchat_ads__ad_report which does not apply any filtering.

dbt_project.yml

@@ -1,17 +1,27 @@
 name: 'snapchat_ads'
-version: '0.3.1'
+version: '0.4.0'
 config-version: 2
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 vars:
   snapchat_ads:
-    creative_history: "{{ ref('stg_snapchat__creative_history') }}"
-    creative_url_tag_history: "{{ ref('stg_snapchat__creative_url_tag_history') }}"
-    ad_hourly_report: "{{ ref('stg_snapchat__ad_hourly_report') }}"
-    ad_account_history: "{{ ref('stg_snapchat__ad_account_history') }}"
-    ad_history: "{{ ref('stg_snapchat__ad_history') }}"
-    ad_squad_history: "{{ ref('stg_snapchat__ad_squad_history') }}"
-    campaign_history: "{{ ref('stg_snapchat__campaign_history') }}"
+    creative_history: "{{ ref('stg_snapchat_ads__creative_history') }}"
+    creative_url_tag_history: "{{ ref('stg_snapchat_ads__creative_url_tag_history') }}"
+    ad_hourly_report: "{{ ref('stg_snapchat_ads__ad_hourly_report') }}"
+    ad_account_history: "{{ ref('stg_snapchat_ads__ad_account_history') }}"
+    ad_history: "{{ ref('stg_snapchat_ads__ad_history') }}"
+    ad_squad_history: "{{ ref('stg_snapchat_ads__ad_squad_history') }}"
+    ad_squad_hourly_report: "{{ ref('stg_snapchat_ads__ad_squad_hourly_report') }}"
+    campaign_history: "{{ ref('stg_snapchat_ads__campaign_history') }}"
+    campaign_hourly_report: "{{ ref('stg_snapchat_ads__campaign_hourly_report') }}"
+
+  snapchat_ads__ad_hourly_passthrough_metrics: []
+  snapchat_ads__ad_squad_hourly_passthrough_metrics: []
+  snapchat_ads__campaign_hourly_report_passthrough_metrics: []
+
 models:
   snapchat_ads:
     +schema: snapchat_ads
     +materialized: table
+    intermediate:
+      materialized: ephemeral
+

integration_tests/ci/sample.profiles.yml

@@ -16,14 +16,14 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: snapchat_integration_tests
+      schema: snapchat_ads_integration_tests_4
       threads: 8
     bigquery:
       type: bigquery
       method: service-account
       keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}"
       project: 'dbt-package-testing'
-      schema: snapchat_integration_tests
+      schema: snapchat_ads_integration_tests_4
       threads: 8
     snowflake:
       type: snowflake
@@ -33,12 +33,12 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: snapchat_integration_tests
+      schema: snapchat_ads_integration_tests_4
       threads: 8
     spark:
       type: spark
       method: http
-      schema: snapchat_ads_integration_tests
+      schema: snapchat_ads_integration_tests_4
       host: "{{ env_var('CI_SPARK_DBT_HOST') }}"
       organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}"
       token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}"
@@ -54,7 +54,7 @@ integration_tests:
       password: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       port: 5432
       dbname: "{{ env_var('CI_POSTGRES_DBT_DATABASE') }}"
-      schema: snapchat_integration_tests
+      schema: snapchat_ads_integration_tests_4
       threads: 8
       keepalives_idle: 0
       sslmode: prefer

integration_tests/dbt_project.yml

@@ -1,24 +1,31 @@
 name: 'snapchat_ads_integration_tests'
-version: '0.3.1'
+version: '0.4.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars: 
   snapchat_ads_source:
-    ad_account_history:  "{{ ref('snapchat_ad_account_history_data') }}"
-    ad_history: "{{ ref('snapchat_ad_history_data') }}"
-    ad_hourly_report: "{{ ref('snapchat_ad_hourly_report_data') }}"
-    ad_squad_history: "{{ ref('snapchat_ad_squad_history_data') }}"
-    campaign_history: "{{ ref('snapchat_campaign_history_data') }}"
-    creative_history: "{{ ref('snapchat_creative_history_data') }}"
-    creative_url_tag_history: "{{ ref('snapchat_creative_url_tag_history_data') }}"
+    snapchat_ads_schema: snapchat_ads_integration_tests_4
+    snapchat_ads_ad_account_history_identifier:  "snapchat_ad_account_history_data"
+    snapchat_ads_ad_history_identifier: "snapchat_ad_history_data"
+    snapchat_ads_ad_hourly_report_identifier: "snapchat_ad_hourly_report_data"
+    snapchat_ads_ad_squad_history_identifier: "snapchat_ad_squad_history_data"
+    snapchat_ads_ad_squad_hourly_report_identifier: "snapchat_ad_squad_hourly_report_data"
+    snapchat_ads_campaign_history_identifier: "snapchat_campaign_history_data"
+    snapchat_ads_campaign_hourly_report_identifier: "snapchat_campaign_hourly_report_data"
+    snapchat_ads_creative_history_identifier: "snapchat_creative_history_data"
+    snapchat_ads_creative_url_tag_history_identifier: "snapchat_creative_url_tag_history_data"
 
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
 
 seeds:
   snapchat_ads_integration_tests:
+    +quote_columns: "{{ true if target.type in ('redshift', 'postgres') else false }}"
     snapchat_ad_hourly_report_data:
       +column_types:
-        date: timestamp
+        date: timestamp
+    snapchat_creative_url_tag_history_data:
+      +column_types:
+        updated_at: timestamp

integration_tests/requirements.txt

@@ -1,6 +1,6 @@
-dbt-snowflake~=1.0.0
-dbt-bigquery~=1.0.0
-dbt-redshift~=1.0.0
-dbt-postgres~=1.0.0
-dbt-spark~=1.0.0
-dbt-spark[PyHive]~=1.0.0
+dbt-snowflake>=1.0.0
+dbt-bigquery>=1.0.0
+dbt-redshift>=1.0.0
+dbt-postgres>=1.0.0
+dbt-spark>=1.0.0
+dbt-spark[PyHive]>=1.0.0

integration_tests/seeds/snapchat_ad_squad_hourly_report_data.csv

@@ -0,0 +1,11 @@
+ad_squad_id,date,_fivetran_synced,android_installs,attachment_avg_view_time_millis,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,conversion_add_billing,conversion_add_cart,conversion_app_opens,conversion_level_completes,conversion_page_views,conversion_purchases,conversion_purchases_value,conversion_save,conversion_searches,conversion_sign_ups,conversion_start_checkout,conversion_view_content,impressions,ios_installs,quartile_1,quartile_2,quartile_3,saves,screen_time_millis,shares,spend,story_completes,story_opens,swipe_up_percent,swipes,total_installs,video_views,view_completion,view_time_millis
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-23T04:00:00,2022-06-03T19:57:36.398,0,0,0,0,0,0,0,1031,1031,0,0,0,0,0,0,0,0,0,0,0,0,98,0,0,0,0,0,101017,0,231409,0,0,0,2,0,0,0,101017
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T21:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1192,1192,0,0,0,0,0,0,0,0,0,0,0,0,234,0,0,0,0,0,278915,0,923850,0,0,0,2,0,0,0,278915
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T19:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1154,1154,0,0,0,0,0,0,0,0,0,0,0,0,274,0,0,0,0,0,316254,0,1273081,0,0,0,2,0,0,0,316254
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T11:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1210,1210,0,0,0,0,0,0,0,0,0,0,0,0,218,0,0,0,0,0,263671,0,1205834,0,0,0,3,0,0,0,263671
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T22:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1011,1011,0,0,0,0,0,0,0,0,0,0,0,0,217,0,0,0,0,0,219347,0,695588,0,0,0,4,0,0,0,219347
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T16:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1086,1086,0,0,0,0,0,0,0,0,0,0,0,0,251,0,0,0,0,0,272598,0,1100768,0,0,0,2,0,0,0,272598
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T13:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1147,1147,0,0,0,0,0,0,0,0,0,0,0,0,154,0,0,0,0,0,176605,0,705311,0,0,0,3,0,0,0,176605
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T20:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1079,1079,0,0,0,0,0,0,0,0,0,0,0,0,292,0,0,0,0,0,315194,0,986431,0,0,0,3,0,0,0,315194
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T12:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1064,1064,0,0,0,0,0,0,0,0,0,0,0,0,183,0,0,0,0,0,194760,0,863491,0,0,0,2,0,0,0,194760
+858e06d4-d24b-4825-8255-5d22027fb024,2022-05-19T15:00:00,2022-06-03T19:57:36.394,0,0,0,0,0,0,0,1108,1108,0,0,0,0,0,0,0,0,0,0,0,0,210,0,0,0,0,0,232725,0,801580,0,0,0,6,0,0,0,232725

integration_tests/seeds/snapchat_campaign_hourly_report_data.csv

@@ -0,0 +1,11 @@
+campaign_id,date,_fivetran_synced,android_installs,attachment_avg_view_time_millis,attachment_quartile_1,attachment_quartile_2,attachment_quartile_3,attachment_total_view_time_millis,attachment_view_completion,avg_screen_time_millis,avg_view_time_millis,conversion_add_billing,conversion_add_cart,conversion_app_opens,conversion_level_completes,conversion_page_views,conversion_purchases,conversion_purchases_value,conversion_save,conversion_searches,conversion_sign_ups,conversion_start_checkout,conversion_view_content,impressions,ios_installs,quartile_1,quartile_2,quartile_3,saves,screen_time_millis,shares,spend,story_completes,story_opens,swipe_up_percent,swipes,total_installs,video_views,view_completion,view_time_millis
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T10:00:00,2022-06-03T19:57:39.338,0,0,0,0,0,0,0,656,656,0,0,0,0,0,0,0,0,0,0,0,0,250,0,0,0,0,0,163989,0,60215,0,0,0,0,0,0,0,163989
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T13:00:00,2022-06-03T19:57:39.338,0,0,0,0,0,0,0,707,707,0,0,0,0,0,0,0,0,0,0,0,0,608,0,0,0,0,0,429790,0,172732,0,0,0,0,0,0,0,429790
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T11:00:00,2022-06-03T19:57:39.338,0,4257,0,0,0,4257,0,742,742,0,0,0,0,0,0,0,0,0,0,0,0,362,0,0,0,0,0,268531,0,82993,0,0,0,1,0,0,0,268531
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T07:00:00,2022-06-03T19:57:39.338,0,0,0,0,0,0,0,735,735,0,0,0,0,0,0,0,0,0,0,0,0,559,0,0,0,0,0,411079,0,69263,0,0,0,0,0,0,0,411079
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T12:00:00,2022-06-03T19:57:39.338,0,0,0,0,0,0,0,664,664,0,0,0,0,0,0,0,0,0,0,0,0,337,0,0,0,0,0,223861,0,84483,0,0,0,0,0,0,0,223861
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T09:00:00,2022-06-03T19:57:39.338,0,1737,0,0,0,1737,0,799,799,0,0,0,0,0,0,0,0,0,0,0,0,117,0,0,0,0,0,93515,0,22118,0,0,0,1,0,0,0,93515
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T14:00:00,2022-06-03T19:57:39.338,0,7619,0,0,0,22858,0,756,756,0,0,0,0,0,0,0,0,0,0,0,0,1049,0,0,0,0,0,793257,0,298014,0,0,0,3,0,0,0,793257
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T05:00:00,2022-06-03T19:57:39.338,0,995,0,0,0,995,0,748,748,0,0,0,0,0,0,0,0,0,0,0,0,655,0,0,0,0,0,489728,0,72377,0,0,0,1,0,0,0,489728
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T15:00:00,2022-06-03T19:57:39.338,0,0,0,0,0,0,0,650,650,0,0,0,0,0,0,0,0,0,0,0,0,75,0,0,0,0,0,48760,0,20479,0,0,0,0,0,0,0,48760
+40ba77bb-289d-41d9-85bf-83f485908a5c,2022-05-29T06:00:00,2022-06-03T19:57:39.338,0,929,0,0,0,929,0,730,730,0,0,0,0,0,0,0,0,0,0,0,0,1130,0,0,0,0,0,824425,0,130814,0,0,0,1,0,0,0,824425