diff --git a/.changes/1.7.1.md b/.changes/1.7.1.md new file mode 100644 index 00000000..4acda7f8 --- /dev/null +++ b/.changes/1.7.1.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f2a03f7..aa942447 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: dbt-core-in-adapters-check - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.8.0 hooks: - id: black args: @@ -30,7 +30,7 @@ repos: - --force-exclude=dbt/adapters/events/adapter_types_pb2.py - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/ @@ -41,7 +41,7 @@ repos: - --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.11.2 hooks: - id: mypy exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb37ad5..8e9f64fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,18 @@ and is generated by [Changie](https://github.com/miniscruff/changie). - dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315)) - - ## dbt-adapters 1.10.1 - September 16, 2024 ## dbt-adapters 1.10.0 - September 12, 2024 +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) + + + ## dbt-adapters 1.7.0 - September 19, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index a55413d1..116d5667 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.7.0" +version = "1.7.1" diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 8d982855..52fdb9bf 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,7 +53,13 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ columns.dbt_valid_to }} is null + where + {% if config.get('dbt_valid_to_current') %} + {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} + ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} ), @@ -64,7 +70,7 @@ {{ strategy.unique_key }} as dbt_unique_key, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} from snapshot_query @@ -166,7 +172,7 @@ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }} + {{ get_dbt_valid_to_current(strategy, columns) }} from ( {{ sql }} ) sbq @@ -210,3 +216,9 @@ {% endif %} {% endif %} {% endmacro %} + +{% macro get_dbt_valid_to_current(strategy, columns) %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} + coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) + as {{ columns.dbt_valid_to }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 74494ed2..cf787e4f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -14,7 +14,12 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current") %} + and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}