From f4863816a5161371c14ae2bd3d18b467864ca84f Mon Sep 17 00:00:00 2001 From: amadrizwan Date: Tue, 23 Nov 2021 17:00:39 +0100 Subject: [PATCH 1/3] adding WHERE filter to events table through var --- dbt_project.yml | 11 +++++++---- models/sessions/jitsu_events.sql | 6 +++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 8af6483..f342102 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,6 @@ +profile: dbt-clickhouse name: 'jitsu' -version: '0.1.1' +version: '0.1.2' require-dbt-version: ">=0.20.0" config-version: 2 @@ -22,7 +23,7 @@ clean-targets: # directories to be removed by `dbt clean` # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - materialized: "{{var('jitsu_model_materialized', 'incremental')}}" + materialized: "{{var('jitsu_model_materialized', 'table')}}" pre-hook: - "{{ onrunstart() }}" # On clickhouse enables allow_experimental_window_functions flag @@ -33,7 +34,7 @@ dispatch: vars: # location of source data table - jitsu_events_table: + jitsu_events_table: "{{ source('jitsu', 'pageviews') }}" # length of trailing window in minutes for session attribution. # We recommend enabling builtin Retrospective User Recognition feature for user stitching @@ -49,4 +50,6 @@ vars: # define them here. Columns will be included in the `jitsu_sessions` # model as `first_` and `last_`. jitsu_pass_through_columns: [] - + + jitsu_events_table_filter: "event_type='pageview'" + jitsu_model_materialized: 'table' diff --git a/models/sessions/jitsu_events.sql b/models/sessions/jitsu_events.sql index ccb70b6..e8291cc 100644 --- a/models/sessions/jitsu_events.sql +++ b/models/sessions/jitsu_events.sql @@ -1,5 +1,9 @@ {{ config( materialized = 'view' )}} +{% if var('jitsu_events_table') is defined %} + select * from {{var('jitsu_events_table')}} where user_anonymous_id is not null and {{var('jitsu_events_table_filter')}} +{% else %} + select * from {{var('jitsu_events_table')}} where user_anonymous_id is not null +{% endif %} -select * from {{var('jitsu_events_table')}} where user_anonymous_id is not null \ No newline at end of file From 4df9d195298a7576651fa5391a1f873814333b2c Mon Sep 17 00:00:00 2001 From: amadrizwan Date: Tue, 23 Nov 2021 17:17:24 +0100 Subject: [PATCH 2/3] change agg from using window to group by --- models/sessions/jitsu_sessions.sql | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/models/sessions/jitsu_sessions.sql b/models/sessions/jitsu_sessions.sql index f8ca050..59bdaab 100644 --- a/models/sessions/jitsu_sessions.sql +++ b/models/sessions/jitsu_sessions.sql @@ -54,24 +54,26 @@ referrer_mapping as ( agg as ( - select distinct + select session_id, - user_anonymous_id, - last_value(coalesce(user_anonymous_id, user_internal_id, user_email)) over ({{window_clause}}) as blended_user_id, - min(_timestamp) over ( {{partition_by}} ) as session_start_timestamp, - max(_timestamp) over ( {{partition_by}} ) as session_end_timestamp, - count(*) over ( {{partition_by}} ) as pageviews, + coalesce(user_anonymous_id, user_id, user_email) as blended_user_id, + max(user_anonymous_id) as user_anonymous_id, + max(user_id) as user_id, + max(user_email) as user_email, + min(_timestamp) as session_start_timestamp, + max(_timestamp) session_end_timestamp, + count() as pageviews, {% for (key, value) in first_values.items() %} - first_value({{key}}) over ({{window_clause}}) as {{value}}, + min({{key}}) as {{value}}, {% endfor %} {% for (key, value) in last_values.items() %} - last_value({{key}}) over ({{window_clause}}) as {{value}}{% if not loop.last %},{% endif %} + max({{key}}) as {{value}}{% if not loop.last %},{% endif %} {% endfor %} - from pageviews_sessionized + from pageviews_sessionized GROUP BY session_id ), @@ -104,7 +106,6 @@ tiers as ( from diffs ), - mapped as ( select @@ -119,3 +120,4 @@ mapped as ( ) select * from mapped + From 464d7bbc1927a15d6f7336488be56552383c9db3 Mon Sep 17 00:00:00 2001 From: Amad Ali Date: Wed, 24 Nov 2021 10:12:10 +0100 Subject: [PATCH 3/3] Update jitsu_events.sql --- models/sessions/jitsu_events.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/sessions/jitsu_events.sql b/models/sessions/jitsu_events.sql index e8291cc..e8cdd3d 100644 --- a/models/sessions/jitsu_events.sql +++ b/models/sessions/jitsu_events.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'view' )}} -{% if var('jitsu_events_table') is defined %} +{% if var('jitsu_events_table_filter') is defined %} select * from {{var('jitsu_events_table')}} where user_anonymous_id is not null and {{var('jitsu_events_table_filter')}} {% else %} select * from {{var('jitsu_events_table')}} where user_anonymous_id is not null