Merge branch 'main' into qmalcolm--330-always-ensure-valid-materializ…

…ation
dbt-labs · Oct 17, 2024 · c7d9ea5 · c7d9ea5
2 parents 96e7854 + cc526ba
commit c7d9ea5
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 10 deletions.
diff --git a/.changes/1.7.1.md b/.changes/1.7.1.md
@@ -0,0 +1,5 @@
+## dbt-adapters 1.7.1 - October 15, 2024
+
+### Features
+
+- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320))
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -18,7 +18,7 @@ repos:
     -   id: dbt-core-in-adapters-check
 
 -   repo: https://github.com/psf/black
-    rev: 24.4.0
+    rev: 24.8.0
     hooks:
     -   id: black
         args:
@@ -30,7 +30,7 @@ repos:
         -   --force-exclude=dbt/adapters/events/adapter_types_pb2.py
 
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
     -   id: flake8
         exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/
@@ -41,7 +41,7 @@ repos:
         -   --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.11.2
     hooks:
     -   id: mypy
         exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,12 +11,18 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
 
 - dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))
 
-
-
 ## dbt-adapters 1.10.1 - September 16, 2024
 
 ## dbt-adapters 1.10.0 - September 12, 2024
 
+## dbt-adapters 1.7.1 - October 15, 2024
+
+### Features
+
+- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320))
+
+
+
 ## dbt-adapters 1.7.0 - September 19, 2024
 
 ### Features

diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py
@@ -1 +1 @@
-version = "1.7.0"
+version = "1.7.1"
diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql
@@ -53,7 +53,13 @@
             {{ strategy.unique_key }} as dbt_unique_key
 
         from {{ target_relation }}
-        where {{ columns.dbt_valid_to }} is null
+        where
+            {% if config.get('dbt_valid_to_current') %}
+               {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #}
+               ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null)
+            {% else %}
+                {{ columns.dbt_valid_to }} is null
+            {% endif %}
 
     ),
 
@@ -64,7 +70,7 @@
             {{ strategy.unique_key }} as dbt_unique_key,
             {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
             {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
-            nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},
+            {{ get_dbt_valid_to_current(strategy, columns) }},
             {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}
 
         from snapshot_query
@@ -166,7 +172,7 @@
         {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},
         {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},
         {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},
-        nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}
+        {{ get_dbt_valid_to_current(strategy, columns) }}
     from (
         {{ sql }}
     ) sbq
@@ -210,3 +216,9 @@
   {%   endif %}
   {% endif %}
 {% endmacro %}
+
+{% macro get_dbt_valid_to_current(strategy, columns) %}
+  {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %}
+  coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})
+  as {{ columns.dbt_valid_to }}
+{% endmacro %}
diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql
@@ -14,7 +14,12 @@
     on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
 
     when matched
-     and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+     {% if config.get("dbt_valid_to_current") %}
+       and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or
+            DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null)
+     {% else %}
+       and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+     {% endif %}
      and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
         then update
         set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}