From 4df6e54b8214da49f60aa0486ede477de55359d9 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 10 Oct 2024 15:17:20 -0400 Subject: [PATCH 1/3] Enable setting current value of dbt_valid_to (#321) --- .../unreleased/Features-20240927-134248.yaml | 6 ++++++ .../materializations/snapshots/helpers.sql | 18 +++++++++++++++--- .../snapshots/snapshot_merge.sql | 7 ++++++- 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Features-20240927-134248.yaml diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml new file mode 100644 index 00000000..0b456244 --- /dev/null +++ b/.changes/unreleased/Features-20240927-134248.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable setting current value of dbt_valid_to +time: 2024-09-27T13:42:48.654556-04:00 +custom: + Author: gshank + Issue: "320" diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 8d982855..52fdb9bf 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,7 +53,13 @@ {{ strategy.unique_key }} as dbt_unique_key from {{ target_relation }} - where {{ columns.dbt_valid_to }} is null + where + {% if config.get('dbt_valid_to_current') %} + {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} + ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {% else %} + {{ columns.dbt_valid_to }} is null + {% endif %} ), @@ -64,7 +70,7 @@ {{ strategy.unique_key }} as dbt_unique_key, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}, + {{ get_dbt_valid_to_current(strategy, columns) }}, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }} from snapshot_query @@ -166,7 +172,7 @@ {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, - nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }} + {{ get_dbt_valid_to_current(strategy, columns) }} from ( {{ sql }} ) sbq @@ -210,3 +216,9 @@ {% endif %} {% endif %} {% endmacro %} + +{% macro get_dbt_valid_to_current(strategy, columns) %} + {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} + coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) + as {{ columns.dbt_valid_to }} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 74494ed2..cf787e4f 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -14,7 +14,12 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current") %} + and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} From 7d187c4dbe6aa159b7a28472af869204ef96cb94 Mon Sep 17 00:00:00 2001 From: FishtownBuildBot <77737458+FishtownBuildBot@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:35:24 -0400 Subject: [PATCH 2/3] [Automated] Update pre-commit hooks (#297) Co-authored-by: Mike Alfare Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f2a03f7..aa942447 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: dbt-core-in-adapters-check - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.8.0 hooks: - id: black args: @@ -30,7 +30,7 @@ repos: - --force-exclude=dbt/adapters/events/adapter_types_pb2.py - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/ @@ -41,7 +41,7 @@ repos: - --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.11.2 hooks: - id: mypy exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py From 3b6ad1f93e731b5046651ddecb635113a5c1cbe7 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 15 Oct 2024 23:03:35 +0000 Subject: [PATCH 3/3] Bumping version to 1.7.1 and generate changelog --- .changes/1.7.1.md | 5 +++++ .changes/unreleased/Features-20240927-134248.yaml | 6 ------ CHANGELOG.md | 10 ++++++++-- dbt/adapters/__about__.py | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 .changes/1.7.1.md delete mode 100644 .changes/unreleased/Features-20240927-134248.yaml diff --git a/.changes/1.7.1.md b/.changes/1.7.1.md new file mode 100644 index 00000000..4acda7f8 --- /dev/null +++ b/.changes/1.7.1.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml deleted file mode 100644 index 0b456244..00000000 --- a/.changes/unreleased/Features-20240927-134248.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Enable setting current value of dbt_valid_to -time: 2024-09-27T13:42:48.654556-04:00 -custom: - Author: gshank - Issue: "320" diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb37ad5..8e9f64fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,18 @@ and is generated by [Changie](https://github.com/miniscruff/changie). - dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315)) - - ## dbt-adapters 1.10.1 - September 16, 2024 ## dbt-adapters 1.10.0 - September 12, 2024 +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) + + + ## dbt-adapters 1.7.0 - September 19, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index a55413d1..116d5667 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.7.0" +version = "1.7.1"