Merge pull request #24 from fivetran/feature/priority-first-syncs

Feature/priority first syncs
fivetran · Oct 25, 2021 · 344ba97 · 344ba97
2 parents ab6679d + f090032
commit 344ba97
Show file tree

Hide file tree

Showing 12 changed files with 78 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,20 @@
+# dbt_fivetran_log v0.4.1
+
+## 🚨 Breaking Changes
+- n/a
+
+## Bug Fixes
+- Added logic in the `fivetran_log__connector_status` model to accommodate the way [priority-first syncs](https://fivetran.com/docs/getting-started/feature#priorityfirstsync) are currently logged. Now, each connector's `connector_health` field can be one of the following values:
+    - "broken"
+    - "incomplete"
+    - "connected"
+    - "paused"
+    - "initial sync in progress"
+    - "priority first sync"
+Once your connector has completed its priority-first sync and begun syncing normally, it will be marked as `connected`.
+
+## Features
+- Added this changelog to capture iterations of the package!
+
+## Under the Hood
+- n/a
diff --git a/README.md b/README.md
@@ -46,6 +46,9 @@ packages:
     version: [">=0.4.0", "<0.5.0"]
 ```
 
+## Package Maintenance
+The Fivetran team maintaining this package **only** maintains the latest version. We highly recommend you keep your `packages.yml` updated with the [dbt hub latest version](https://hub.getdbt.com/fivetran/fivetran_log/latest/). You may refer to the [CHANGELOG](/CHANGELOG.md) and release notes for more information on changes across versions.
+
 ## Configuration
 By default, this package will run using your target database and the `fivetran_log` schema. If this is not where your Fivetran Log data is, add the following configuration to your `dbt_project.yml` file:
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'fivetran_log'
-version: '0.4.0'
+version: '0.4.1'
 
 require-dbt-version: ">=0.20.0"
 

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/graph.gpickle b/docs/graph.gpickle
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/partial_parse.msgpack b/docs/partial_parse.msgpack
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '0.4.0'
+version: '0.4.1'
 config-version: 2
 profile: 'integration_tests'
 

diff --git a/models/fivetran_log.yml b/models/fivetran_log.yml
@@ -31,7 +31,7 @@ models:
         description: Name of the destination loading data from this connector.
 
       - name: connector_health
-        description: Status of the connector's data flow. Can be `"broken"`, `"incomplete"`, `"connected"`, `"paused"`, or `"initial sync in progress"`.
+        description: Status of the connector's data flow. Can be `"broken"`, `"incomplete"`, `"connected"`, `"paused"`, `"initial sync in progress"`, or `"priority first sync"`.
 
       - name: last_sync_started_at
         description: > 

diff --git a/models/fivetran_log__connector_status.sql b/models/fivetran_log__connector_status.sql
@@ -7,7 +7,12 @@ with connector_log as (
     where event_type = 'SEVERE'
         or event_type = 'WARNING'
         or event_subtype like 'sync%'
+        or (event_subtype = 'status' 
+            and {{ fivetran_utils.json_extract(string="message_data", string_path="status") }} ='RESCHEDULED'
+            and {{ fivetran_utils.json_extract(string="message_data", string_path="reason") }} like '%intended behavior%'
+            ) -- for priority-first syncs. these should be captured by event_type = 'WARNING' but let's make sure
 
+        -- whole reason is "We have rescheduled the connector to force flush data from the forward sync into your destination. This is intended behavior and means that the connector is working as expected."
 ),
 
 schema_changes as (
@@ -49,7 +54,16 @@ connector_metrics as (
         connector.is_paused,
         connector.set_up_at,
         max(case when connector_log.event_subtype = 'sync_start' then connector_log.created_at else null end) as last_sync_started_at,
-        max(case when connector_log.event_subtype = 'sync_end' then connector_log.created_at else null end) as last_sync_completed_at,
+
+        max(case when connector_log.event_subtype = 'sync_end' 
+            then connector_log.created_at else null end) as last_sync_completed_at,
+
+        max(case when connector_log.event_subtype = 'status'
+                and {{ fivetran_utils.json_extract(string="connector_log.message_data", string_path="status") }} ='RESCHEDULED'
+                and {{ fivetran_utils.json_extract(string="connector_log.message_data", string_path="reason") }} like '%intended behavior%'
+            then connector_log.created_at else null end) as last_priority_first_sync_completed_at,
+
+
         max(case when connector_log.event_type = 'SEVERE' then connector_log.created_at else null end) as last_error_at,
         max(case when event_type = 'WARNING' then connector_log.created_at else null end) as last_warning_at
 
@@ -65,13 +79,27 @@ connector_health as (
     select
         *,
         case 
-            -- there has not been a sync
+            -- connector is paused
             when is_paused then 'paused'
+
+            -- a sync has never been attempted
             when last_sync_started_at is null then 'incomplete'
+
+            -- a priority-first sync has occurred, but a normal sync has not
+            when last_priority_first_sync_completed_at is not null and last_sync_completed_at is null then 'priority first sync'
+
+            -- a priority sync has occurred more recently than a normal one (may occurr if the connector has been paused and resumed)
+            when last_priority_first_sync_completed_at > last_sync_completed_at then 'priority first sync'
+
+            -- a sync has been attempted, but not completed, and it's not due to errors. also a priority-first sync hasn't
             when last_sync_completed_at is null and last_error_at is null then 'initial sync in progress'
 
+            -- there's been an error since the connector last completed a sync
             when last_error_at > last_sync_completed_at then 'broken'
+
+            -- there's never been a successful sync and there have been errors
             when last_sync_completed_at is null and last_error_at is not null then 'broken'
+
         else 'connected' end as connector_health
 
     from connector_metrics
@@ -97,9 +125,13 @@ connector_recent_logs as (
     left join connector_log 
         on connector_log.connector_id = connector_health.connector_id
         -- limiting relevance to since the last successful sync completion (if there has been one)
-        and connector_log.created_at > coalesce(connector_health.last_sync_completed_at, '2000-01-01') 
-        -- only looking at erors and warnings (excluding syncs)
-        and connector_log.event_type != 'INFO'  
+        and connector_log.created_at > coalesce(connector_health.last_sync_completed_at, connector_health.last_priority_first_sync_completed_at, '2000-01-01') 
+        -- only looking at errors and warnings (excluding syncs - both normal and priority first)
+        and connector_log.event_type != 'INFO' 
+        -- need to explicitly avoid priority first statuses because they are of event_type WARNING
+        and not (connector_log.event_subtype = 'status' 
+            and {{ fivetran_utils.json_extract(string="connector_log.message_data", string_path="status") }} ='RESCHEDULED'
+            and {{ fivetran_utils.json_extract(string="connector_log.message_data", string_path="reason") }} like '%intended behavior%')
 
     group by 1,2,3,4,5,6,7,8,9,10,11 -- de-duping error messages