Feature/connector name update

CHANGELOG.md

@@ -1,3 +1,16 @@
+# dbt_fivetran_log v1.9.0
+[PR #132](https://github.com/fivetran/dbt_fivetran_log/pull/132) includes the following updates:
+
+## 🚨 Schema Changes 🚨
+- Following the [July 2024 Fivetran Platform connector update](https://fivetran.com/docs/logs/fivetran-platform/changelog#july2024), the `connector_name` field has been added to the `incremental_mar` source table. As a result, the following changes have been applied:
+  - A new tmp model `stg_fivetran_platform__incremental_mar_tmp` has been created. This is necessary to ensure column consistency in downstream `incremental_mar` models.
+  - The `get_incremental_mar_columns()` macro has been added to ensure all required columns are present in the `stg_fivetran_platform__incremental_mar` model.
+  - The `stg_fivetran_platform__incremental_mar` has been updated to reference both the aforementioned tmp model and macro to fill empty fields if any required field is not present in the source.
+  - The `connector_name` field in the `stg_fivetran_platform__incremental_mar` model is now defined by: `coalesce(connector_name, connector_id)`. This ensures the data model will use the appropriate field to define the `connector_name`.
+
+## Under the Hood
+- Updated integration test seed data within `integration_tests/seeds/incremental_mar.csv` to ensure new code updates are working as expected.
+
 # dbt_fivetran_log v1.8.0
 [PR #130](https://github.com/fivetran/dbt_fivetran_log/pull/130) includes the following updates:
 

README.md

@@ -69,7 +69,7 @@ Include the following Fivetran Platform package version range in your `packages.
 ```yaml
 packages:
   - package: fivetran/fivetran_log
-    version: [">=1.8.0", "<1.9.0"]
+    version: [">=1.9.0", "<1.10.0"]
 ```
 
 > Note that although the source connector is now "Fivetran Platform", the package retains the old name of "fivetran_log".

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.8.0'
+version: '1.9.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.8.0'
+version: '1.9.0'
 
 config-version: 2
 profile: 'integration_tests'

integration_tests/seeds/incremental_mar.csv

@@ -1,11 +1,11 @@
-connector_id,destination_id,free_type,measured_date,schema_name,sync_type,table_name,updated_at,_fivetran_synced,incremental_rows
-this_connector,ups_trampoline,PAID,2022-01-31 00:00:00,this_connector,UNKNOWN,my_table,2022-09-01 6:12:13,2022-11-09 20:09:26,46
-this_connector,ups_trampoline,PAID,2022-02-22 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-22 8:29:22,2022-11-07 20:10:24,1
-this_connector,television_trace,PAID,2022-03-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-01 21:52:24,2022-11-07 20:10:15,8
-this_connector,television_trace,PAID,2022-03-04 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-04 16:19:05,2022-11-07 20:10:16,1
-this_connector,television_trace,PAID,2022-07-29 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-29 23:05:17,2022-11-07 20:10:33,4
-this_connector,television_trace,PAID,2022-05-14 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-14 11:07:37,2022-11-07 20:10:24,1
-this_connector,television_trace,PAID,2021-09-19 00:00:00,that_connector,UNKNOWN,drop_table,2021-09-19 12:11:38,2022-11-07 20:09:55,0
-that_connector,television_trace,PAID,2022-06-08 00:00:00,that_connector,UNKNOWN,drop_table,2022-06-08 21:29:07,2022-11-07 20:10:25,2
-that_connector,television_trace,PAID,2022-03-05 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-05 10:55:00,2022-11-07 20:10:16,1
-that_connector,television_trace,PAID,2022-07-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-01 21:56:37,2022-11-07 20:10:30,1
+connector_name,connector_id,destination_id,free_type,measured_date,schema_name,sync_type,table_name,updated_at,_fivetran_synced,incremental_rows
+this_connector,,ups_trampoline,PAID,2022-01-31 00:00:00,this_connector,UNKNOWN,my_table,2022-09-01 6:12:13,2022-11-09 20:09:26,46
+this_connector,,ups_trampoline,PAID,2022-02-22 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-22 8:29:22,2022-11-07 20:10:24,1
+,this_connector,television_trace,PAID,2022-03-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-01 21:52:24,2022-11-07 20:10:15,8
+this_connector,,television_trace,PAID,2022-03-04 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-04 16:19:05,2022-11-07 20:10:16,1
+this_connector,,television_trace,PAID,2022-07-29 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-29 23:05:17,2022-11-07 20:10:33,4
+this_connector,,television_trace,PAID,2022-05-14 00:00:00,that_connector,UNKNOWN,drop_table,2022-05-14 11:07:37,2022-11-07 20:10:24,1
+this_connector,,television_trace,PAID,2021-09-19 00:00:00,that_connector,UNKNOWN,drop_table,2021-09-19 12:11:38,2022-11-07 20:09:55,0
+this_connector,,television_trace,PAID,2022-06-08 00:00:00,that_connector,UNKNOWN,drop_table,2022-06-08 21:29:07,2022-11-07 20:10:25,2
+,this_connector,television_trace,PAID,2022-03-05 00:00:00,that_connector,UNKNOWN,drop_table,2022-03-05 10:55:00,2022-11-07 20:10:16,1
+this_connector,,television_trace,PAID,2022-07-01 00:00:00,that_connector,UNKNOWN,drop_table,2022-07-01 21:56:37,2022-11-07 20:10:30,1

macros/get_incremental_mar_columns.sql

@@ -0,0 +1,19 @@
+{% macro get_incremental_mar_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "connector_id", "datatype": dbt.type_string()},
+    {"name": "connector_name", "datatype": dbt.type_string()},
+    {"name": "destination_id", "datatype": dbt.type_string()},
+    {"name": "free_type", "datatype": dbt.type_string()},
+    {"name": "measured_date", "datatype": dbt.type_timestamp()},
+    {"name": "schema_name", "datatype": dbt.type_string()},
+    {"name": "sync_type", "datatype": dbt.type_string()},
+    {"name": "table_name", "datatype": dbt.type_string()},
+    {"name": "incremental_rows", "datatype": dbt.type_int()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/staging/src_fivetran_platform.yml

@@ -20,7 +20,9 @@ sources:
           Each measurement is calculated cumulatively for the month and includes all types of mar (paid, free, etc.)
         columns:
           - name: connector_id
-            description: The *name* of the connector being measured. Note - this is erroneously named and will be fixed soon by Fivetran.
+            description: The *name* of the connector being measured. This may still be present in older Fivetran Platform connectors. However, all connectors setup after July 2024 will not have this field as it has been replaced with connector_name.
+          - name: connector_name
+            description: The name of the connector being measured.
           - name: destination_id
             description: Foreign key referencing the `destination` whose table is being measured.
           - name: free_type

models/staging/stg_fivetran_platform.yml

@@ -7,7 +7,7 @@ models:
       Each measurement is calculated cumulatively for the month and includes all types of mar (paid, free, etc.)
     columns:
       - name: connector_name
-        description: The *name* of the connector being measured. Note - this is erroneously named and will be fixed soon by Fivetran.
+        description: The name of the connector being measured.
       - name: destination_id
         description: Foreign key referencing the `destination` whose table is being measured.
       - name: free_type

models/staging/stg_fivetran_platform__incremental_mar.sql

@@ -1,13 +1,24 @@
 with base as (
 
     select * 
-    from {{ var('incremental_mar') }}
+    from {{ ref('stg_fivetran_platform__incremental_mar_tmp') }}
 ),
 
 fields as (
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_fivetran_platform__incremental_mar_tmp')),
+                staging_columns=get_incremental_mar_columns()
+            )
+        }}
+    from base
+),
+
+final as (
 
     select
-        connector_id as connector_name,
+        coalesce(connector_name, connector_id) as connector_name,
         destination_id,
         free_type,
         cast(measured_date as {{ dbt.type_timestamp() }}) as measured_date,
@@ -17,8 +28,8 @@ fields as (
         updated_at,
         _fivetran_synced,
         incremental_rows
-    from base
+    from fields
 )
 
 select * 
-from fields
+from final

models/staging/tmp/stg_fivetran_platform__incremental_mar_tmp.sql

@@ -0,0 +1,2 @@
+select *
+from {{ var('incremental_mar') }}