From a2099d98c14971016559620083eb9912438ebf12 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 25 Sep 2024 12:03:25 -0400 Subject: [PATCH 1/4] Update to use configured dbt_valid_to_current_date --- .../materializations/snapshots/helpers.sql | 18 +++++++++++++++--- .../snapshots/snapshot_merge.sql | 6 +++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 8d982855..6f13ddb9 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -40,6 +40,11 @@ {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% if config.get('dbt_valid_to_current_date') %} + {% set valid_to_expr = " = " ~ config.get('dbt_valid_to_current_date') %} + {% else %} + {% set valid_to_expr = " is null " %} + {% endif %} with snapshot_query as ( @@ -53,7 +58,7 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ columns.dbt_valid_to }} is null + where {{ columns.dbt_valid_to }} {{ valid_to_expr }} ), @@ -64,7 +69,7 @@ {{ strategy.unique_key }} as dbt_unique_key, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} from snapshot_query @@ -166,7 +171,7 @@ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }} + {{ get_dbt_valid_to_current(strategy, columns) }} from ( {{ sql }} ) sbq @@ -210,3 +215,10 @@ {% endif %} {% endif %} {% endmacro %} + +{% macro get_dbt_valid_to_current(strategy, columns) %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current_date') or "null" %} + {% do log("dbt_valid_to_current = " ~ dbt_valid_to_current, info=true) %} + coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) + as {{ columns.dbt_valid_to }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 74494ed2..ee57c429 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -14,7 +14,11 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current_date") %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current_date') }} + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} From 7a9e069a386c4b9422569fb9a2e4b83b75f95a20 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 25 Sep 2024 18:18:34 -0400 Subject: [PATCH 2/4] Change name of config field --- .../macros/materializations/snapshots/helpers.sql | 6 +++--- .../macros/materializations/snapshots/snapshot_merge.sql | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 6f13ddb9..1183c9f4 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -40,8 +40,8 @@ {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} - {% if config.get('dbt_valid_to_current_date') %} - {% set valid_to_expr = " = " ~ config.get('dbt_valid_to_current_date') %} + {% if config.get('dbt_valid_to_current') %} + {% set valid_to_expr = " = " ~ config.get('dbt_valid_to_current') %} {% else %} {% set valid_to_expr = " is null " %} {% endif %} @@ -217,7 +217,7 @@ {% endmacro %} {% macro get_dbt_valid_to_current(strategy, columns) %} - {% set dbt_valid_to_current = config.get('dbt_valid_to_current_date') or "null" %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} {% do log("dbt_valid_to_current = " ~ dbt_valid_to_current, info=true) %} coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) as {{ columns.dbt_valid_to }} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index ee57c429..fcd892f9 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -14,8 +14,8 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - {% if config.get("dbt_valid_to_current_date") %} - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current_date') }} + {% if config.get("dbt_valid_to_current") %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null {% endif %} From 7777be43d96751f4ab95547e921e188430ac3794 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 27 Sep 2024 13:25:57 -0400 Subject: [PATCH 3/4] Check for both dbt_valid_to_current and null --- .../macros/materializations/snapshots/helpers.sql | 14 +++++++------- .../materializations/snapshots/snapshot_merge.sql | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 1183c9f4..52fdb9bf 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -40,11 +40,6 @@ {% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%} {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} - {% if config.get('dbt_valid_to_current') %} - {% set valid_to_expr = " = " ~ config.get('dbt_valid_to_current') %} - {% else %} - {% set valid_to_expr = " is null " %} - {% endif %} with snapshot_query as ( @@ -58,7 +53,13 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ columns.dbt_valid_to }} {{ valid_to_expr }} + where + {% if config.get('dbt_valid_to_current') %} + {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} + ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} ), @@ -218,7 +219,6 @@ {% macro get_dbt_valid_to_current(strategy, columns) %} {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} - {% do log("dbt_valid_to_current = " ~ dbt_valid_to_current, info=true) %} coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) as {{ columns.dbt_valid_to }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index fcd892f9..cf787e4f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -15,7 +15,8 @@ when matched {% if config.get("dbt_valid_to_current") %} - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} + and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null {% endif %} From 3162f51d7fb0c2205578faf123be2f4beffd1701 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 27 Sep 2024 13:42:55 -0400 Subject: [PATCH 4/4] changie --- .changes/unreleased/Features-20240927-134248.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240927-134248.yaml diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml new file mode 100644 index 00000000..0b456244 --- /dev/null +++ b/.changes/unreleased/Features-20240927-134248.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable setting current value of dbt_valid_to +time: 2024-09-27T13:42:48.654556-04:00 +custom: + Author: gshank + Issue: "320"