Update api_calls to extract_summary (#138)

* updates * update changelog * update case when * update seed * regen docs * update test * update changelog * update readme * update comment
fivetran · Nov 21, 2024 · 244f2c5 · 244f2c5
1 parent d3ae73d
commit 244f2c5
Show file tree

Hide file tree

Showing 11 changed files with 56 additions and 26 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,14 @@
+# dbt_fivetran_log v1.9.1
+[PR #138](https://github.com/fivetran/dbt_fivetran_log/pull/138) includes the following updates:
+
+## Features
+- For Fivetran Platform Connectors created after November 2024, Fivetran has deprecated the `api_call` event in favor of `extract_summary` ([release notes](https://fivetran.com/docs/logs/changelog)).
+- Accordingly, we have updated the `fivetran_platform__connector_daily_events` model to support the new `extract_summary` event while maintaining backward compatibility with the `api_call` event for connectors created before November 2024. 
+
+## Under the Hood
+- Replaced the deprecated `dbt.current_timestamp_backcompat()` function with `dbt.current_timestamp()` to ensure all timestamps are captured in UTC.
+- Updated `fivetran_platform__connector_daily_events` to support running `dbt compile` prior to the initial `dbt run` on a new schema.
+
 # dbt_fivetran_log v1.9.0
 [PR #132](https://github.com/fivetran/dbt_fivetran_log/pull/132) includes the following updates:
 

diff --git a/README.md b/README.md
@@ -1,4 +1,6 @@
-<p align="center">
+# Fivetran Platform dbt Package ([Docs](https://fivetran.github.io/dbt_fivetran_log/))
+
+<p align="left">
     <a alt="License"
         href="https://github.com/fivetran/dbt_fivetran_log/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
@@ -13,7 +15,6 @@
         <img src="https://img.shields.io/badge/Fivetran_Quickstart_Compatible%3F-yes-green.svg" /></a>
 </p>
 
-# Fivetran Platform dbt Package ([Docs](https://fivetran.github.io/dbt_fivetran_log/))
 ## What does this dbt package do?
 - Generates a comprehensive data dictionary of your Fivetran Platform connector (previously called Fivetran Log) data via the [dbt docs site](https://fivetran.github.io/dbt_fivetran_log/)
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.9.0'
+version: '1.9.1'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.9.0'
+version: '1.9.1'
 
 config-version: 2
 profile: 'integration_tests'

diff --git a/integration_tests/seeds/log.csv b/integration_tests/seeds/log.csv
@@ -29,4 +29,6 @@ ZZLSXVYxirh8aKlgdCzo5fvd71g=,2021-12-10 14:26:42.029,2021-12-10 20:30:53.949,thi
 D7UqnKYn6OT04HkUcPNjXA95ttI=,2021-12-10 14:26:29.719,2021-12-10 20:30:53.878,this_connector,INFO,"{""table"":""media_insights""}",write_to_table_start,123,456abc
 V3H62Gj5c6nB8bY+OdxMHvp/hNs=,2021-12-10 14:26:05.907,2021-12-10 20:30:53.778,this_connector,INFO,,sync_start,123,456abc
 jkLxnPUfQ/9mHaWCmxUQEhul8ZA=,2023-11-09 11:31:31.579000,2023-11-09 15:55:12.560000,protestations_mourned,INFO,"{""actor"":""[email protected]"",""properties"":{""endpoint"":""************"",""apiKey"":""************"",""customEvents"":[],""syncMode"":""AllEvents"",""isE2ETest"":false,""events"":[],""customEventSyncMode"":""AllEvents""},""id"":""iterable""}",create_connector,,
-R7UqnKYn6OT04HkUcPNjXA95qqI=,2021-12-10 16:26:29.719,2021-12-10 20:30:53.878,intrinsic_departed,INFO,says actor but not a json,status,,456abc
+R7UqnKYn6OT04HkUcPNjXA95qqI=,2021-12-10 16:26:29.719,2021-12-10 20:30:53.878,intrinsic_departed,INFO,says actor but not a json,status,,456abc
+D7UqnKYn6OT04HkUcPNjXA95qqp=,2024-11-18 16:26:29.719,2024-11-19 20:30:53.878,aft_gleeful,INFO,"{""total_queries"":15,""total_rows"":4810}",extract_summary,,456abc
+h7UqnKYn6OT04HkUcPNjXA95qlp=,2024-11-19 16:26:29.719,2024-11-19 20:30:53.878,aft_gleeful,INFO,"{""total_queries"":5,""total_rows"":4810}",extract_summary,,456abc
diff --git a/integration_tests/tests/integrity/row_count__connector_daily_events.sql b/integration_tests/tests/integrity/row_count__connector_daily_events.sql
@@ -37,7 +37,7 @@ spine as (
         {{ fivetran_utils.fivetran_date_spine(
             datepart = "day", 
             start_date =  "cast('" ~ first_date[0:10] ~ "' as date)", 
-            end_date = dbt.dateadd("week", 1, dbt.date_trunc('day', dbt.current_timestamp_backcompat() if target.type != 'sqlserver' else dbt.current_timestamp())) 
+            end_date = dbt.dateadd("week", 1, dbt.date_trunc('day', dbt.current_timestamp())) 
             ) 
         }} 
     ) as date_spine

diff --git a/models/fivetran_platform__connector_daily_events.sql b/models/fivetran_platform__connector_daily_events.sql
@@ -7,39 +7,56 @@ with connector as (
 ),
 
 -- grab api calls, schema changes, and record modifications
+
 log_events as (
 
     select 
         connector_id,
         cast( {{ dbt.date_trunc('day', 'created_at') }} as date) as date_day,
-        case 
-            when event_subtype in ('create_table', 'alter_table', 'create_schema', 'change_schema_config') then 'schema_change' 
-            else event_subtype end as event_subtype,
-
-        sum(case when event_subtype = 'records_modified' then cast( {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_bigint()}} )
-        else 1 end) as count_events 
+        event_subtype,
+        message_data
 
     from {{ ref('stg_fivetran_platform__log') }}
 
-    where event_subtype in ('api_call', 
-                            'records_modified', 
-                            'create_table', 'alter_table', 'create_schema', 'change_schema_config') -- all schema changes
-
+    where event_subtype in (
+        'api_call', 'extract_summary', 'records_modified', 'create_table', 'alter_table',
+        'create_schema', 'change_schema_config') -- all relevant event subtypes
         and connector_id is not null
+),
+
+agg_log_events as (
 
-    group by connector_id, cast( {{ dbt.date_trunc('day', 'created_at') }} as date), event_subtype
+    select 
+        connector_id,
+        date_day,
+        case 
+            when event_subtype in ('create_table', 'alter_table', 'create_schema', 'change_schema_config') then 'schema_change' 
+            else event_subtype end as event_subtype,
+
+        sum(
+            case 
+                when event_subtype = 'records_modified' 
+                then cast({{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_bigint()}} )
+                when event_subtype = 'extract_summary'
+                then cast({{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['total_queries']) }} as {{ dbt.type_bigint()}})
+                else 1
+                end
+            ) as count_events
+
+    from log_events
+    group by connector_id, date_day, event_subtype
 ),
 
 pivot_out_events as (
 
     select
         connector_id,
         date_day,
-        max(case when event_subtype = 'api_call' then count_events else 0 end) as count_api_calls,
+        max(case when event_subtype = 'api_call' or event_subtype = 'extract_summary' then count_events else 0 end) as count_api_calls,
         max(case when event_subtype = 'records_modified' then count_events else 0 end) as count_record_modifications,
         max(case when event_subtype = 'schema_change' then count_events else 0 end) as count_schema_changes
 
-    from log_events
+    from agg_log_events
     group by connector_id, date_day
 ), 
 
@@ -63,7 +80,7 @@ connector_event_counts as (
 
 spine as (
 
-    {% if execute %}
+    {% if execute and flags.WHICH in ('run', 'build') %}
     {% set first_date_query %}
         select  min( signed_up ) as min_date from {{ var('connector') }}
     {% endset %}
@@ -78,7 +95,7 @@ spine as (
         {{ fivetran_utils.fivetran_date_spine(
             datepart = "day", 
             start_date =  "cast('" ~ first_date[0:10] ~ "' as date)", 
-            end_date = dbt.dateadd("week", 1, dbt.date_trunc('day', dbt.current_timestamp_backcompat() if target.type != 'sqlserver' else dbt.current_timestamp())) 
+            end_date = dbt.dateadd("week", 1, dbt.date_trunc('day', dbt.current_timestamp())) 
             ) 
         }} 
     ) as date_spine
@@ -138,7 +155,7 @@ final as (
     select *
     from join_event_history
 
-    where date_day <= cast({{ dbt.current_timestamp_backcompat() if target.type != 'sqlserver' else dbt.current_timestamp() }} as date)
+    where date_day <= cast({{ dbt.current_timestamp() }} as date)
 )
 
 select *

diff --git a/models/fivetran_platform__connector_status.sql b/models/fivetran_platform__connector_status.sql
@@ -47,7 +47,7 @@ schema_changes as (
     from {{ ref('stg_fivetran_platform__log') }}
 
     where 
-        {{ dbt.datediff('created_at', dbt.current_timestamp_backcompat() if target.type != 'sqlserver' else dbt.current_timestamp(), 'day') }} <= 30
+        {{ dbt.datediff('created_at', dbt.current_timestamp(), 'day') }} <= 30
         and event_subtype in ('create_table', 'alter_table', 'create_schema', 'change_schema_config')
 
     group by connector_id