From a109d47a5b9eafa7854847954ae1e3c6cf1e4452 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 22 Sep 2022 10:15:30 -0700 Subject: [PATCH 01/14] consolidate timestamp macros --- dbt/include/snowflake/macros/adapters.sql | 15 --------------- dbt/include/snowflake/timestamps.sql | 16 ++++++++++++++++ tests/functional/adapter/test_timestamps.py | 5 +++++ 3 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 dbt/include/snowflake/timestamps.sql create mode 100644 tests/functional/adapter/test_timestamps.py diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index e956982c5..cc0cfb930 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -161,21 +161,6 @@ {{ return(load_result('check_schema_exists').table) }} {%- endmacro %} -{% macro snowflake__current_timestamp() -%} - convert_timezone('UTC', current_timestamp()) -{%- endmacro %} - - -{% macro snowflake__snapshot_string_as_time(timestamp) -%} - {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} - {{ return(result) }} -{%- endmacro %} - - -{% macro snowflake__snapshot_get_time() -%} - to_timestamp_ntz({{ current_timestamp() }}) -{%- endmacro %} - {% macro snowflake__rename_relation(from_relation, to_relation) -%} {% call statement('rename_relation') -%} diff --git a/dbt/include/snowflake/timestamps.sql b/dbt/include/snowflake/timestamps.sql new file mode 100644 index 000000000..57549aab4 --- /dev/null +++ b/dbt/include/snowflake/timestamps.sql @@ -0,0 +1,16 @@ +{% macro snowflake__current_timestamp() -%} + convert_timezone('UTC', current_timestamp()) +{%- endmacro %} + +{% macro snowflake__snapshot_string_as_time(timestamp) -%} + {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} + {{ return(result) }} +{%- endmacro %} + +{% macro snowflake__snapshot_get_time() -%} + to_timestamp_ntz({{ current_timestamp() }}) +{%- endmacro %} + +{% macro snowflake__current_timestamp_backcompat() %} + convert_timezone('UTC', {{current_timestamp()}}) +{% endmacro %} \ No newline at end of file diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py new file mode 100644 index 000000000..382e03075 --- /dev/null +++ b/tests/functional/adapter/test_timestamps.py @@ -0,0 +1,5 @@ +from tests.adapter.dbt.tests.adapter.basic.test_timestamps import TestCurrentTimestamps + + +class TestCurrentTimestampSnowflake(TestCurrentTimestamps): + pass \ No newline at end of file From 731c013dc27331e8f5383397b4deeb5a16f52aea Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 11:16:28 -0700 Subject: [PATCH 02/14] add testing changes --- .../snowflake/macros/utils/timestamps.sql | 16 ++++++++++++++++ dev-requirements.txt | 6 ++++-- tests/functional/adapter/test_timestamps.py | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 dbt/include/snowflake/macros/utils/timestamps.sql diff --git a/dbt/include/snowflake/macros/utils/timestamps.sql b/dbt/include/snowflake/macros/utils/timestamps.sql new file mode 100644 index 000000000..2966c93d7 --- /dev/null +++ b/dbt/include/snowflake/macros/utils/timestamps.sql @@ -0,0 +1,16 @@ +{% macro snowflake__current_timestamp() -%} + convert_timezone('UTC', current_timestamp()) +{%- endmacro %} + +{% macro snowflake__snapshot_string_as_time(timestamp) -%} + {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} + {{ return(result) }} +{%- endmacro %} + +{% macro snowflake__snapshot_get_time() -%} + to_timestamp_ntz({{ current_timestamp() }}) +{%- endmacro %} + +{% macro snowflake__current_timestamp_backcompat() %} + convert_timezone('UTC', {{current_timestamp()}}) +{% endmacro %} diff --git a/dev-requirements.txt b/dev-requirements.txt index 85560e655..b4f353bfd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,10 +1,11 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 click~=8.1.3 +cryptography bumpversion flake8 flaky @@ -19,6 +20,7 @@ pytest-logbook pytest-csv pytest-xdist pytz +snowflake-connector-python tox>=3.13 types-pytz types-requests diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index 382e03075..2b6212bec 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -1,5 +1,5 @@ -from tests.adapter.dbt.tests.adapter.basic.test_timestamps import TestCurrentTimestamps +from dbt.tests.adapter.utils.test_timestamps import TestCurrentTimestamps class TestCurrentTimestampSnowflake(TestCurrentTimestamps): - pass \ No newline at end of file + pass From 02f2c500f04289de8404e36751a9e977794b3748 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 11:16:35 -0700 Subject: [PATCH 03/14] deleting extra timestamps.sql --- dbt/include/snowflake/timestamps.sql | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 dbt/include/snowflake/timestamps.sql diff --git a/dbt/include/snowflake/timestamps.sql b/dbt/include/snowflake/timestamps.sql deleted file mode 100644 index 57549aab4..000000000 --- a/dbt/include/snowflake/timestamps.sql +++ /dev/null @@ -1,16 +0,0 @@ -{% macro snowflake__current_timestamp() -%} - convert_timezone('UTC', current_timestamp()) -{%- endmacro %} - -{% macro snowflake__snapshot_string_as_time(timestamp) -%} - {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} - {{ return(result) }} -{%- endmacro %} - -{% macro snowflake__snapshot_get_time() -%} - to_timestamp_ntz({{ current_timestamp() }}) -{%- endmacro %} - -{% macro snowflake__current_timestamp_backcompat() %} - convert_timezone('UTC', {{current_timestamp()}}) -{% endmacro %} \ No newline at end of file From 9d8b024839e66e666b5a072d0f559224b1294236 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 11:23:25 -0700 Subject: [PATCH 04/14] changie entry --- .changes/unreleased/Breaking Changes-20220923-112314.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Breaking Changes-20220923-112314.yaml diff --git a/.changes/unreleased/Breaking Changes-20220923-112314.yaml b/.changes/unreleased/Breaking Changes-20220923-112314.yaml new file mode 100644 index 000000000..bd6a96f15 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20220923-112314.yaml @@ -0,0 +1,7 @@ +kind: Breaking Changes +body: Add timestamps.sql to consolidate timestamp macros +time: 2022-09-23T11:23:14.312738-07:00 +custom: + Author: colin-rogers-dbt + Issue: "5521" + PR: "273" From 105d46bf9b506b5723233bd91b96fc7d9f759d8d Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 23 Sep 2022 15:35:51 -0700 Subject: [PATCH 05/14] fix backcompat --- .../snowflake/macros/utils/timestamps.sql | 6 ++++- tests/functional/adapter/test_timestamps.py | 22 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/dbt/include/snowflake/macros/utils/timestamps.sql b/dbt/include/snowflake/macros/utils/timestamps.sql index 2966c93d7..c2a1807e5 100644 --- a/dbt/include/snowflake/macros/utils/timestamps.sql +++ b/dbt/include/snowflake/macros/utils/timestamps.sql @@ -2,6 +2,10 @@ convert_timezone('UTC', current_timestamp()) {%- endmacro %} +{% macro snowflake__current_timestamp_in_utc() -%} + convert_timezone('UTC', current_timestamp()) +{%- endmacro %} + {% macro snowflake__snapshot_string_as_time(timestamp) -%} {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} {{ return(result) }} @@ -12,5 +16,5 @@ {%- endmacro %} {% macro snowflake__current_timestamp_backcompat() %} - convert_timezone('UTC', {{current_timestamp()}}) + current_timestamp()::timestamp_ntz {% endmacro %} diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index 2b6212bec..ab4b5826a 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -1,5 +1,21 @@ -from dbt.tests.adapter.utils.test_timestamps import TestCurrentTimestamps +import pytest +from dbt.tests.adapter.utils import test_timestamps +_MODEL_CURRENT_TIMESTAMP = """ +SELECT {{current_timestamp()}} as current_timestamp, + {{current_timestamp_in_utc()}} as current_timestamp_in_utc, + {{current_timestamp_backcompat()}} as current_timestamp_backcompat +""" -class TestCurrentTimestampSnowflake(TestCurrentTimestamps): - pass +class TestCurrentTimestampSnowflake(test_timestamps.TestCurrentTimestamps): + @pytest.fixture(scope="class") + def models(self): + return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} + + @pytest.fixture(scope="class") + def expected_schema(self): + return { + "CURRENT_TIMESTAMP": "TIMESTAMP_TZ", + "CURRENT_TIMESTAMP_IN_UTC": "TIMESTAMP_TZ", + "CURRENT_TIMESTAMP_BACKCOMPAT": "TIMESTAMP_NTZ", + } From c07d9c68d93e58c42e5b048913fce9325ef80cb4 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:33:05 -0700 Subject: [PATCH 06/14] Update Breaking Changes-20220923-112314.yaml --- .changes/unreleased/Breaking Changes-20220923-112314.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/unreleased/Breaking Changes-20220923-112314.yaml b/.changes/unreleased/Breaking Changes-20220923-112314.yaml index bd6a96f15..ed9f67bbb 100644 --- a/.changes/unreleased/Breaking Changes-20220923-112314.yaml +++ b/.changes/unreleased/Breaking Changes-20220923-112314.yaml @@ -1,7 +1,7 @@ -kind: Breaking Changes +kind: Feature body: Add timestamps.sql to consolidate timestamp macros time: 2022-09-23T11:23:14.312738-07:00 custom: Author: colin-rogers-dbt - Issue: "5521" + Issue: "376" PR: "273" From 7c6617fcca4ae8747aafa4dcb8feba5bf462ddf2 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:16:19 -0700 Subject: [PATCH 07/14] Update dev-requirements.txt --- dev-requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index b4f353bfd..40bee7d8b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,7 +5,6 @@ git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt black==22.8.0 click~=8.1.3 -cryptography bumpversion flake8 flaky @@ -20,7 +19,6 @@ pytest-logbook pytest-csv pytest-xdist pytz -snowflake-connector-python tox>=3.13 types-pytz types-requests From c33d72e0f8b6adb2e42c38e1df2b818a789ba60d Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 28 Sep 2022 14:35:23 -0700 Subject: [PATCH 08/14] remove current_timestamp_in_utc --- dbt/include/snowflake/macros/utils/timestamps.sql | 14 +++++++------- tests/functional/adapter/test_timestamps.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbt/include/snowflake/macros/utils/timestamps.sql b/dbt/include/snowflake/macros/utils/timestamps.sql index c2a1807e5..6c4878352 100644 --- a/dbt/include/snowflake/macros/utils/timestamps.sql +++ b/dbt/include/snowflake/macros/utils/timestamps.sql @@ -2,13 +2,9 @@ convert_timezone('UTC', current_timestamp()) {%- endmacro %} -{% macro snowflake__current_timestamp_in_utc() -%} - convert_timezone('UTC', current_timestamp()) -{%- endmacro %} - {% macro snowflake__snapshot_string_as_time(timestamp) -%} - {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} - {{ return(result) }} + {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%} + {{ return(result) }} {%- endmacro %} {% macro snowflake__snapshot_get_time() -%} @@ -16,5 +12,9 @@ {%- endmacro %} {% macro snowflake__current_timestamp_backcompat() %} - current_timestamp()::timestamp_ntz + current_timestamp()::timestamp_ntz +{% endmacro %} + +{% macro snowflake__current_timestamp_in_utc_backcompat() %} + convert_timezone('UTC', {{ snowflake__current_timestamp_backcompat() }})::{{ type_timestamp() }} {% endmacro %} diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index ab4b5826a..49b96ba75 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -3,7 +3,7 @@ _MODEL_CURRENT_TIMESTAMP = """ SELECT {{current_timestamp()}} as current_timestamp, - {{current_timestamp_in_utc()}} as current_timestamp_in_utc, + {{current_timestamp_in_utc_backcompat()}} as current_timestamp_in_utc_backcompat, {{current_timestamp_backcompat()}} as current_timestamp_backcompat """ @@ -16,6 +16,6 @@ def models(self): def expected_schema(self): return { "CURRENT_TIMESTAMP": "TIMESTAMP_TZ", - "CURRENT_TIMESTAMP_IN_UTC": "TIMESTAMP_TZ", + "CURRENT_TIMESTAMP_IN_UTC_BACKCOMPAT": "TIMESTAMP_NTZ", "CURRENT_TIMESTAMP_BACKCOMPAT": "TIMESTAMP_NTZ", } From 331d14dd56b493e59d6e63a3cde50c7cae078b52 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 29 Sep 2022 11:10:40 -0700 Subject: [PATCH 09/14] fix backcompat --- dbt/include/snowflake/macros/utils/timestamps.sql | 2 +- tests/functional/adapter/test_timestamps.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dbt/include/snowflake/macros/utils/timestamps.sql b/dbt/include/snowflake/macros/utils/timestamps.sql index 6c4878352..ecbc9940b 100644 --- a/dbt/include/snowflake/macros/utils/timestamps.sql +++ b/dbt/include/snowflake/macros/utils/timestamps.sql @@ -12,7 +12,7 @@ {%- endmacro %} {% macro snowflake__current_timestamp_backcompat() %} - current_timestamp()::timestamp_ntz + current_timestamp::{{ type_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc_backcompat() %} diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index 49b96ba75..f63793540 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -19,3 +19,11 @@ def expected_schema(self): "CURRENT_TIMESTAMP_IN_UTC_BACKCOMPAT": "TIMESTAMP_NTZ", "CURRENT_TIMESTAMP_BACKCOMPAT": "TIMESTAMP_NTZ", } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + select convert_timezone('UTC', current_timestamp()) as current_timestamp, + convert_timezone('UTC', current_timestamp::TIMESTAMP)::TIMESTAMP as current_timestamp_in_utc_backcompat, + current_timestamp::TIMESTAMP as current_timestamp_backcompat + """ From 1896507ceb00586b230ec3b76fe7c6fa208b63d0 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 09:08:22 -0700 Subject: [PATCH 10/14] change test import --- tests/functional/adapter/test_timestamps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index f63793540..e278faa90 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -7,7 +7,8 @@ {{current_timestamp_backcompat()}} as current_timestamp_backcompat """ -class TestCurrentTimestampSnowflake(test_timestamps.TestCurrentTimestamps): + +class TestCurrentTimestampSnowflake(test_timestamps.BaseCurrentTimestamps): @pytest.fixture(scope="class") def models(self): return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} From ed0140139d0bdb9b17863d0b3795bc328cc4778a Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 14:24:44 -0700 Subject: [PATCH 11/14] fix BaseCurrentTimestamps import --- tests/functional/adapter/test_timestamps.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py index e278faa90..d120100a9 100644 --- a/tests/functional/adapter/test_timestamps.py +++ b/tests/functional/adapter/test_timestamps.py @@ -1,5 +1,6 @@ +import imp import pytest -from dbt.tests.adapter.utils import test_timestamps +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps _MODEL_CURRENT_TIMESTAMP = """ SELECT {{current_timestamp()}} as current_timestamp, @@ -8,7 +9,7 @@ """ -class TestCurrentTimestampSnowflake(test_timestamps.BaseCurrentTimestamps): +class TestCurrentTimestampSnowflake(BaseCurrentTimestamps): @pytest.fixture(scope="class") def models(self): return {"get_current_timestamp.sql": _MODEL_CURRENT_TIMESTAMP} From 5ce70e598929ff799a7f96e4dbb61983eb1c2f39 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:34:08 -0700 Subject: [PATCH 12/14] update dev-requirements --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 40bee7d8b..85560e655 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@consolidate_timestamp_logic#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 click~=8.1.3 From 09b62d670e61e1eb52bbc1eb62cb35b3810885e9 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:40:44 -0700 Subject: [PATCH 13/14] Update change log body --- ...hanges-20220923-112314.yaml => Feature-20220923-112314.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .changes/unreleased/{Breaking Changes-20220923-112314.yaml => Feature-20220923-112314.yaml} (62%) diff --git a/.changes/unreleased/Breaking Changes-20220923-112314.yaml b/.changes/unreleased/Feature-20220923-112314.yaml similarity index 62% rename from .changes/unreleased/Breaking Changes-20220923-112314.yaml rename to .changes/unreleased/Feature-20220923-112314.yaml index ed9f67bbb..ac76e36ac 100644 --- a/.changes/unreleased/Breaking Changes-20220923-112314.yaml +++ b/.changes/unreleased/Feature-20220923-112314.yaml @@ -1,5 +1,5 @@ kind: Feature -body: Add timestamps.sql to consolidate timestamp macros +body: Migrate dbt-utils current_timestamp macros into core + adapters time: 2022-09-23T11:23:14.312738-07:00 custom: Author: colin-rogers-dbt From 8ce23ecc0a5b75977713e15a18b854fae1f58e49 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 30 Sep 2022 15:42:44 -0700 Subject: [PATCH 14/14] fix changie log --- .changes/unreleased/Feature-20220923-112314.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Feature-20220923-112314.yaml b/.changes/unreleased/Feature-20220923-112314.yaml index ac76e36ac..3810f8f34 100644 --- a/.changes/unreleased/Feature-20220923-112314.yaml +++ b/.changes/unreleased/Feature-20220923-112314.yaml @@ -3,5 +3,5 @@ body: Migrate dbt-utils current_timestamp macros into core + adapters time: 2022-09-23T11:23:14.312738-07:00 custom: Author: colin-rogers-dbt - Issue: "376" + Issue: "276" PR: "273"