Merge pull request #112 from fivetran/feature/add-deleted-connector-s…

…tatuses Add deleted connector statuses to `fivetran_platform__connector_status`
fivetran · Jan 24, 2024 · a1a4662 · a1a4662
2 parents 28fa36f + e7d80b4
commit a1a4662
Show file tree

Hide file tree

Showing 13 changed files with 31 additions and 16 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,8 +1,18 @@
+# dbt_fivetran_log v1.4.3
+[PR #112](https://github.com/fivetran/dbt_fivetran_log/pull/112) includes the following updates:
+
+## Feature Updates
+- Updated logic for `connector_health` dimension in `fivetran_platform__connector_status` to show `deleted` for connectors that had been removed. Previously the connector would report the last known status before deletion, which is inaccurate based on the definition of this measure. 
+- Brought in the `is_deleted` dimension (based on the `_fivetran_deleted` value) to `stg_fivetran__platform__connector` to capture connectors that are deleted in the downstream `fivetran_platform__connector_status` model.
+
+## Under The Hood
+- Renamed `get_brand_columns` macro file to `get_connector_columns` to maintain consistency with the actual macro function within the file, and the `connector` source that the macro is drawing columns from. 
+
 # dbt_fivetran_log v1.4.2
 [PR #109](https://github.com/fivetran/dbt_fivetran_log/pull/109) includes the following updates:
 
 ## Bug Fixes
-- Adjusted the `stg_fivetran_platform__credits_used` and `stg_fivetran_platform__usage_cost` models to return empty tables (via a `limit 0`) if the respective `fivetran_platform__credits_pricing` and/or `fivetran_platform__usage_pricing` variables are disabled. This is to avoid Postgres data type errors if those tables are null. 
+- Adjusted the `stg_fivetran_platform__credits_used` and `stg_fivetran_platform__usage_cost` models to return empty tables (via a `limit 0` function, or `fetch/offset` function for SQL Server) if the respective `fivetran_platform__credits_pricing` and/or `fivetran_platform__usage_pricing` variables are disabled. This is to avoid Postgres data type errors if those tables are null. 
 
 ## Under the Hood
 - Included an additional test case within the integration tests where the `fivetran_platform__credits_pricing` variable is set to false and the `fivetran_platform__usage_pricing` variable is set to true in order to effectively test this scenario.
@@ -18,8 +28,6 @@
 ## Under the Hood
 - Included auto-releaser GitHub Actions workflow to automate future releases.
 - Updated the maintainer PR template to resemble the most up to date format.
-- Included a `quickstart.yml` file to allow for automated Quickstart data model deployments.
-
 
 # dbt_fivetran_log v1.4.0
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.4.2'
+version: '1.4.3'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.4.2'
+version: '1.4.3'
 
 config-version: 2
 profile: 'integration_tests'

diff --git a/integration_tests/seeds/connector.csv b/integration_tests/seeds/connector.csv
@@ -1,3 +1,3 @@
-connector_id,_fivetran_synced,connecting_user_id,connector_name,connector_type,destination_id,paused,signed_up
-this_connector,2021-03-12 21:03:45.994,,this_connector,s3,ups_trampoline,true,2020-08-31 19:00:22.913630
-this_connector,2021-03-12 21:03:45.991,off_ence,this_connector,s3,television_trace,true,2019-03-22 18:49:12.833910
+connector_id,_fivetran_synced,connecting_user_id,connector_name,connector_type,destination_id,paused,signed_up,_fivetran_deleted
+this_connector,2021-03-12 21:03:45.994,,this_connector,s3,ups_trampoline,true,2020-08-31 19:00:22.913630,false
+this_connector,2021-03-12 21:03:45.991,off_ence,this_connector,s3,television_trace,true,2019-03-22 18:49:12.833910,true
diff --git a/macros/get_brand_columns.sql → macros/get_connector_columns.sql b/macros/get_brand_columns.sql → macros/get_connector_columns.sql
diff --git a/models/fivetran_platform.yml b/models/fivetran_platform.yml
@@ -31,7 +31,7 @@ models:
         description: Name of the destination loading data from this connector.
 
       - name: connector_health
-        description: Status of the connector's data flow. Can be `"broken"`, `"incomplete"`, `"connected"`, `"paused"`, `"initial sync in progress"`, or `"priority first sync"`.
+        description: Status of the connector's data flow. Can be `"broken"`, `"deleted"`, `"incomplete"`, `"connected"`, `"paused"`, `"initial sync in progress"`, or `"priority first sync"`.
 
       - name: last_successful_sync_completed_at
         description: > 

diff --git a/models/fivetran_platform__connector_status.sql b/models/fivetran_platform__connector_status.sql
@@ -65,6 +65,7 @@ connector_metrics as (
         connector.destination_id,
         connector.is_paused,
         connector.set_up_at,
+        connector.is_deleted,
         max(case when connector_log.event_subtype = 'sync_start' then connector_log.created_at else null end) as last_sync_started_at,
 
         max(case when connector_log.event_subtype = 'sync_end' 
@@ -92,14 +93,17 @@ connector_metrics as (
     from connector 
     left join connector_log 
         on connector_log.connector_id = connector.connector_id
-    group by connector.connector_id, connector.connector_name, connector.connector_type, connector.destination_id, connector.is_paused, connector.set_up_at
+    group by connector.connector_id, connector.connector_name, connector.connector_type, connector.destination_id, connector.is_paused, connector.set_up_at, connector.is_deleted
 ),
 
 connector_health_status as (
 
     select
         *,
         case 
+            -- connector is deleted
+            when is_deleted {{ ' = 1' if target.type == 'sqlserver' }} then 'deleted'
+
             -- connector is paused
             when is_paused {{ ' = 1' if target.type == 'sqlserver' }} then 'paused'
 

diff --git a/models/staging/stg_fivetran_platform.yml b/models/staging/stg_fivetran_platform.yml
@@ -53,6 +53,8 @@ models:
         description: Boolean that is true if the connector's sync is currently paused.
       - name: set_up_at
         description: Timestamp of when the connection was set up.
+      - name: is_deleted
+        description: Boolean that is true if the connector has been deleted.
 
   - name: stg_fivetran_platform__credits_used
     description: Table of the credits used by the customer per month for each destination.

diff --git a/models/staging/stg_fivetran_platform__connector.sql b/models/staging/stg_fivetran_platform__connector.sql
@@ -25,7 +25,8 @@ final as (
         destination_id,
         connecting_user_id,
         paused as is_paused,
-        signed_up as set_up_at
+        signed_up as set_up_at,
+        coalesce(_fivetran_deleted,{{ ' 0 ' if target.type == 'sqlserver' else ' false'}}) as is_deleted
     from fields
 
     -- Only look at the most recent one