Bugfix/update passthrough coulmns

CHANGELOG.md

@@ -1,3 +1,10 @@
+# dbt_salesforce v0.9.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #38](https://github.com/fivetran/dbt_salesforce/pull/38) updates the old passthrough column methodology to allow for aliasing of any field names brought in. This is useful, for example, if you wish to bring in fields across different Salesforce objects that may have the same names and wish to alias them to avoid confusion, particularly if any of the objects are joined together.
+
+Additionally, the `salesforce__opportunity_enhanced` model has been updated regarding how custom fields passed through from the `user` table are dealt with. Since the `user` model is joined in two times, once as information about an opportunity owner and the other about an opportunity manager, to avoid ambiguity, custom fields passed through from the user table will be suffixed based on whether it belongs to a user who is an `_owner` or a `_manager`. 
+
 # dbt_salesforce v0.8.0
 
 ## 🚨 Breaking Changes 🚨:

README.md

@@ -121,24 +121,48 @@ models:
       +schema: my_new_schema_name # leave blank for just the target_schema
 ```
 ### Adding Passthrough Columns
-This package allows users to add additional columns to the `opportunity enhanced` model and `contact enhanced` model by using the below variables in you `dbt_project.yml` file. For the `opportunity enhanced` model, columns passed through in `opportunity_enhanced_pass_through_columns` must also be present in the upstream source `opportunity`, `account`, `user`, or `user_role` table. If you want to include a column from the `user` table, you must specify if you want it to be a field related to the opportunity_manager or opportunity_owner. 
+This package allows users to add additional columns to the `opportunity enhanced`, `opportunity_line_item_enhanced`, and `contact enhanced` model by using the below variables in your `dbt_project.yml` file. 
+
+For the `opportunity enhanced` model, it joins in the `user` model two times, once as information about an opportunity owner and the other about an opportunity manager. Therefore to avoid ambiguous columns from joining in the same model twice, custom fields passed through from the user table will be suffixed based on whether it belongs to a user who is an `_owner` or a `_manager`. 
 
 Additionally, you may add additional columns to the staging models. For example, for passing columns to `stg_salesforce__product_2` you would need to configure `product_2_pass_through_columns`.
 
 ```yml
 vars:
-  opportunity_enhanced_pass_through_columns: [account_custom_field_1, my_opp_custom_field, user_role_custom_field_1, opportunity_manager.user_custom_column_1]
-  account_pass_through_columns: [account_custom_field_1, account_custom_field_2]
-  user_pass_through_columns: [user_custom_column_1,user_custom_column_2]
-  contact_pass_through_columns: [contact_custom_field_1, contact_custom_field_2]
-  opportunity_pass_through_columns: [my_opp_custom_field]
-  lead_pass_through_columns: [lead_custom_field_1, lead_custom_field_2]
-  task_pass_through_columns: [task_custom_field_1, task_custom_field_2]
-  event_pass_through_columns: [event_custom_field_1, event_custom_field_2]
-  product_2_pass_through_columns: [product_2_custom_field_1, product_2_custom_field_2]
-  order_pass_through_columns: [order_custom_field_1, order_custom_field_2]
-  opportunity_line_item_pass_through_columns: [opportunity_line_item_custom_field_1, opportunity_line_item_custom_field_2]
-  user_role_pass_through_columns: [user_role_custom_field_1, user_role_custom_field_2]
+  salesforce__account_pass_through_columns: 
+    - name: "salesforce__account_field"
+      alias: "salesforce__account_field"
+  salesforce__contact_pass_through_columns: 
+    - name: "salesforce__contact_field"
+      alias: "contact_field_x"
+  salesforce__event_pass_through_columns: 
+    - name: "salesforce__event_field"
+  salesforce__lead_pass_through_columns: 
+    - name: "salesforce__lead_field"
+  salesforce__opportunity_pass_through_columns: 
+    - name: "salesforce__opportunity_field"
+      alias: "opportunity_field_x"
+  salesforce__opportunity_line_item_pass_through_columns: 
+    - name: "salesforce__opportunity_line_item_field"
+      alias: "opportunity_line_item_field_x"
+    - name: "field_name_2"
+  salesforce__order_pass_through_columns: 
+    - name: "salesforce__order_field"
+      alias: "order_field_x"
+    - name: "another_field"
+      alias: "field_abc"
+  salesforce__product_2_pass_through_columns: 
+    - name: "salesforce__product_2_field"
+      alias: "product_2_field_x"
+  salesforce__task_pass_through_columns: 
+    - name: "salesforce__task_field"
+      alias: "task_field_x"
+  salesforce__user_role_pass_through_columns: 
+    - name: "salesforce__user_role_field"
+      alias: "user_role_field_x"
+  salesforce__user_pass_through_columns: 
+    - name: "salesforce__user_field"
+
 ```
 
 ## (Optional) Step 5: Adding Formula Fields as Pass Through Columns
@@ -171,7 +195,7 @@ This dbt package is dependent on the following dbt packages. For more informatio
 ```yml
 packages:
     - package: fivetran/salesforce_source
-      version: [">=0.6.0", "<0.7.0"]
+      version: [">=0.7.0", "<0.8.0"]
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]
     - package: dbt-labs/dbt_utils

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'salesforce'
-version: '0.8.0'
+version: '0.9.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   salesforce:
@@ -20,18 +20,19 @@ vars:
     product_2: "{{ ref('stg_salesforce__product_2') }}"
     order: "{{ ref('stg_salesforce__order') }}"
     opportunity_line_item: "{{ ref('stg_salesforce__opportunity_line_item') }}"
-    opportunity_enhanced_pass_through_columns: []
-    account_pass_through_columns: []
-    contact_pass_through_columns: []
-    event_pass_through_columns: []
-    lead_pass_through_columns: []
-    opportunity_pass_through_columns: []
-    opportunity_line_item_pass_through_columns: []
-    order_pass_through_columns: []
-    product_2_pass_through_columns: []
-    task_pass_through_columns: []
-    user_role_pass_through_columns: []
-    user_pass_through_columns: []
+
+    salesforce__account_pass_through_columns: []
+    salesforce__contact_pass_through_columns: []
+    salesforce__event_pass_through_columns: []
+    salesforce__lead_pass_through_columns: []
+    salesforce__opportunity_pass_through_columns: []
+    salesforce__opportunity_line_item_pass_through_columns: []
+    salesforce__order_pass_through_columns: []
+    salesforce__product_2_pass_through_columns: []
+    salesforce__task_pass_through_columns: []
+    salesforce__user_role_pass_through_columns: []
+    salesforce__user_pass_through_columns: []
+
 clean-targets:
 - target
 - dbt_modules

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: salesforce_integrations_tests
+      schema: salesforce_integrations_tests_1
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: salesforce_integrations_tests
+      schema: salesforce_integrations_tests_1
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: salesforce_integrations_tests
+      schema: salesforce_integrations_tests_1
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: salesforce_integrations_tests
+      schema: salesforce_integrations_tests_1
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: salesforce_integrations_tests
+      schema: salesforce_integrations_tests_1
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'salesforce_integration_tests'
-version: '0.8.0'
+version: '0.9.0'
 config-version: 2
 
 profile: 'integration_tests'
@@ -23,7 +23,7 @@ vars:
     salesforce_product_2_identifier: "sf_product_2_data"
     salesforce_order_identifier: "sf_order_data"
     salesforce_opportunity_line_item_identifier: "sf_opportunity_line_item_data"
-    salesforce_schema: salesforce_integrations_tests
+    salesforce_schema: salesforce_integrations_tests_1
 
 seeds:
   salesforce_integration_tests:

macros/custom_persist_pass_through_columns.sql

@@ -0,0 +1,9 @@
+{% macro custom_persist_pass_through_columns(pass_through_variable, identifier=none, transform='', append_string='') %}
+
+{% if var(pass_through_variable, none) %}
+    {% for field in var(pass_through_variable) %}
+        , {{ transform ~ '(' ~ (identifier ~ '.' if identifier else '') ~ (field.alias if field.alias else field.name) ~ ')' }} as {{ field.alias if field.alias else field.name }}{{ append_string }}
+    {% endfor %}
+{% endif %}
+
+{% endmacro %}

models/salesforce__contact_enhanced.sql

@@ -54,20 +54,9 @@ select
     account.type as account_type
 
         --The below scripts allows for pass through columns.
-    {% if var('contact_pass_through_columns',[]) != [] %}
-    , contact.{{ var('contact_pass_through_columns') | join (", contact.")}}
-
-    {% endif %}
-
-    {% if var('account_pass_through_columns',[]) != [] %}
-    , account.{{ var('account_pass_through_columns') | join (", account.")}}
-
-    {% endif %}
-
-    {% if var('user_pass_through_columns',[]) != [] %}
-    , salesforce_user.{{ var('user_pass_through_columns') | join (", salesforce_user.")}}
-
-    {% endif %}
+    {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='salesforce__contact_pass_through_columns', identifier='contact') }}
+    {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='salesforce__account_pass_through_columns', identifier='account') }}
+    {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='salesforce__user_pass_through_columns', identifier='salesforce_user') }}
 
 from contact
 left join account 

models/salesforce__opportunity_enhanced.sql

@@ -4,7 +4,13 @@ with opportunity as (
     from {{ var('opportunity') }}
 ), 
 
-salesforce_user as (
+opportunity_owner as (
+
+    select *
+    from {{ var('user') }}  
+), 
+
+opportunity_manager as (
 
     select *
     from {{ var('user') }}  
@@ -69,18 +75,22 @@ add_fields as (
         case when is_closed_this_quarter then 1 else 0 end as closed_count_this_quarter
 
         --The below script allows for pass through columns.
+        {{ custom_persist_pass_through_columns(pass_through_variable='salesforce__opportunity_pass_through_columns', identifier='opportunity') }}
+        {{ custom_persist_pass_through_columns(pass_through_variable='salesforce__account_pass_through_columns', identifier='account') }}
+        {{ custom_persist_pass_through_columns(pass_through_variable='salesforce__user_pass_through_columns', identifier='opportunity_owner', append_string= '_owner') }}
+        {{ custom_persist_pass_through_columns(pass_through_variable='salesforce__user_pass_through_columns', identifier='opportunity_manager', append_string= '_manager') }}
 
-        {% if var('opportunity_enhanced_pass_through_columns',[]) != [] %}
-        , {{ var('opportunity_enhanced_pass_through_columns') | join (", ")}}
-
+        -- If using user_role table, the following will be included, otherwise it will not.
+        {% if var('salesforce__user_role_enabled', True) %}
+        {{ custom_persist_pass_through_columns(pass_through_variable='salesforce__user_role_pass_through_columns', identifier='user_role') }}
         {% endif %}
 
     from opportunity
     left join account 
         on opportunity.account_id = account.account_id
-    left join salesforce_user as opportunity_owner 
+    left join opportunity_owner 
         on opportunity.owner_id = opportunity_owner.user_id
-    left join salesforce_user as opportunity_manager 
+    left join opportunity_manager 
         on opportunity_owner.manager_id = opportunity_manager.user_id
 
     -- If using user_role table, the following will be included, otherwise it will not.

models/salesforce__opportunity_line_item_enhanced.sql

@@ -57,14 +57,10 @@ final as (
         {% endif %}
 
         --The below script allows for pass through columns.
-        {% if var('opportunity_line_item_pass_through_columns',[]) != [] %}
-        , oli.{{ var('opportunity_line_item_pass_through_columns') | join (", oli.")}}
-
-        {% endif %}
-
-        {% if var('product_2_pass_through_columns',[]) != [] %}
-        , product_2.{{ var('product_2_pass_through_columns') | join (", product_2.")}}
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='salesforce__opportunity_line_item_pass_through_columns', identifier='opportunity_line_item') }}
 
+        {% if var('salesforce__product_2_enabled', True) %}
+        {{ fivetran_utils.persist_pass_through_columns(pass_through_variable='salesforce__product_2_pass_through_columns', identifier='product_2') }}
         {% endif %}
 
     from opportunity_line_item as oli

packages.yml

@@ -1,3 +1,7 @@
 packages:
-  - package: fivetran/salesforce_source
-    version: [">=0.6.0", "<0.7.0"]
+  # - package: fivetran/salesforce_source
+  #   version: [">=0.7.0", "<0.8.0"]
+
+- git: https://github.com/fivetran/dbt_salesforce_source.git 
+  revision: bugfix/update_passthrough_columns
+  warn-unpinned: false