From 3f2386df0ae90534ad5947f616340419a6cc97f0 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 11 Apr 2024 16:04:28 -0400 Subject: [PATCH 01/66] Changes to support unit test sql format (#146) --- .changes/unreleased/Features-20240409-084844.yaml | 6 ++++++ .../global_project/macros/materializations/tests/unit.sql | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Features-20240409-084844.yaml diff --git a/.changes/unreleased/Features-20240409-084844.yaml b/.changes/unreleased/Features-20240409-084844.yaml new file mode 100644 index 00000000..4fceebda --- /dev/null +++ b/.changes/unreleased/Features-20240409-084844.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support for sql fixtures in unit testing +time: 2024-04-09T08:48:44.441106-04:00 +custom: + Author: gshank + Issue: "148" diff --git a/dbt/include/global_project/macros/materializations/tests/unit.sql b/dbt/include/global_project/macros/materializations/tests/unit.sql index 6d7b632c..78c6f6bc 100644 --- a/dbt/include/global_project/macros/materializations/tests/unit.sql +++ b/dbt/include/global_project/macros/materializations/tests/unit.sql @@ -3,6 +3,7 @@ {% set relations = [] %} {% set expected_rows = config.get('expected_rows') %} + {% set expected_sql = config.get('expected_sql') %} {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %} {%- set target_relation = this.incorporate(type='table') -%} @@ -11,10 +12,13 @@ {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%} {%- set column_name_to_data_types = {} -%} {%- for column in columns_in_relation -%} - {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} + {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%} {%- endfor -%} - {% set unit_test_sql = get_unit_test_sql(sql, get_expected_sql(expected_rows, column_name_to_data_types), tested_expected_column_names) %} + {% if not expected_sql %} + {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %} + {% endif %} + {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %} {% call statement('main', fetch_result=True) -%} From 43eb675facf55e1d2f77a5997ebbd40bf77aeb62 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:57:46 -0400 Subject: [PATCH 02/66] Update dependabot config to cover GHA (#161) --- .../unreleased/Under the Hood-20240410-184109.yaml | 6 ++++++ .github/dependabot.yml | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Under the Hood-20240410-184109.yaml diff --git a/.changes/unreleased/Under the Hood-20240410-184109.yaml b/.changes/unreleased/Under the Hood-20240410-184109.yaml new file mode 100644 index 00000000..c0fc5b92 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240410-184109.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Update dependabot config to cover GHA +time: 2024-04-10T18:41:09.719652-04:00 +custom: + Author: mikealfare + Issue: "161" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a6f3449..02f010c7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,17 @@ version: 2 updates: - # python dependencies - package-ecosystem: "pip" directory: "/" schedule: interval: "daily" rebase-strategy: "disabled" + - package-ecosystem: "pip" + directory: "/dbt-tests-adapter" + schedule: + interval: "daily" + rebase-strategy: "disabled" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + rebase-strategy: "disabled" From 67315c7b3082306a9a5dbdcb67210a4531dfbe6f Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Fri, 12 Apr 2024 12:16:50 -0600 Subject: [PATCH 03/66] Debug log when `type_code` fails to convert to a `data_type` (#135) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/unreleased/Features-20240323-160251.yaml | 6 ++++++ dbt/adapters/events/adapter_types.proto | 10 ++++++++++ dbt/adapters/events/adapter_types_pb2.py | 7 +++++-- dbt/adapters/events/types.py | 13 +++++++++++++ tests/unit/test_events.py | 1 + 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Features-20240323-160251.yaml diff --git a/.changes/unreleased/Features-20240323-160251.yaml b/.changes/unreleased/Features-20240323-160251.yaml new file mode 100644 index 00000000..c4b59464 --- /dev/null +++ b/.changes/unreleased/Features-20240323-160251.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Debug log when `type_code` fails to convert to a `data_type` +time: 2024-03-23T16:02:51.306658-06:00 +custom: + Author: dbeatty10 + Issue: "8912" diff --git a/dbt/adapters/events/adapter_types.proto b/dbt/adapters/events/adapter_types.proto index aa0b507c..69d64325 100644 --- a/dbt/adapters/events/adapter_types.proto +++ b/dbt/adapters/events/adapter_types.proto @@ -515,3 +515,13 @@ message ConstraintNotSupportedMsg { AdapterCommonEventInfo info = 1; ConstraintNotSupported data = 2; } + +// E050 +message TypeCodeNotFound { + int32 type_code = 1; +} + +message TypeCodeNotFoundMsg { + AdapterCommonEventInfo info = 1; + TypeCodeNotFound data = 2; +} diff --git a/dbt/adapters/events/adapter_types_pb2.py b/dbt/adapters/events/adapter_types_pb2.py index 5d41b719..bfd44080 100644 --- a/dbt/adapters/events/adapter_types_pb2.py +++ b/dbt/adapters/events/adapter_types_pb2.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: adapter_types.proto -# Protobuf Python Version: 4.25.2 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -16,7 +15,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x61\x64\x61pter_types.proto\x12\x0bproto_types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xab\x02\n\x16\x41\x64\x61pterCommonEventInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0b\n\x03msg\x18\x03 \x01(\t\x12\r\n\x05level\x18\x04 \x01(\t\x12\x15\n\rinvocation_id\x18\x05 \x01(\t\x12\x0b\n\x03pid\x18\x06 \x01(\x05\x12\x0e\n\x06thread\x18\x07 \x01(\t\x12&\n\x02ts\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x05\x65xtra\x18\t \x03(\x0b\x32..proto_types.AdapterCommonEventInfo.ExtraEntry\x12\x10\n\x08\x63\x61tegory\x18\n \x01(\t\x1a,\n\nExtraEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x13\x41\x64\x61pterNodeRelation\x12\x10\n\x08\x64\x61tabase\x18\n \x01(\t\x12\x0e\n\x06schema\x18\x0b \x01(\t\x12\r\n\x05\x61lias\x18\x0c \x01(\t\x12\x15\n\rrelation_name\x18\r \x01(\t\"\x9f\x02\n\x0f\x41\x64\x61pterNodeInfo\x12\x11\n\tnode_path\x18\x01 \x01(\t\x12\x11\n\tnode_name\x18\x02 \x01(\t\x12\x11\n\tunique_id\x18\x03 \x01(\t\x12\x15\n\rresource_type\x18\x04 \x01(\t\x12\x14\n\x0cmaterialized\x18\x05 \x01(\t\x12\x13\n\x0bnode_status\x18\x06 \x01(\t\x12\x17\n\x0fnode_started_at\x18\x07 \x01(\t\x12\x18\n\x10node_finished_at\x18\x08 \x01(\t\x12%\n\x04meta\x18\t \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x37\n\rnode_relation\x18\n \x01(\x0b\x32 .proto_types.AdapterNodeRelation\"G\n\x0fReferenceKeyMsg\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\x12\n\nidentifier\x18\x03 \x01(\t\"?\n\x19\x41\x64\x61pterDeprecationWarning\x12\x10\n\x08old_name\x18\x01 \x01(\t\x12\x10\n\x08new_name\x18\x02 \x01(\t\"\x87\x01\n\x1c\x41\x64\x61pterDeprecationWarningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x34\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32&.proto_types.AdapterDeprecationWarning\"!\n\x1f\x43ollectFreshnessReturnSignature\"\x93\x01\n\"CollectFreshnessReturnSignatureMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12:\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32,.proto_types.CollectFreshnessReturnSignature\"\x8e\x01\n\x11\x41\x64\x61pterEventDebug\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\"w\n\x14\x41\x64\x61pterEventDebugMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.AdapterEventDebug\"\x8d\x01\n\x10\x41\x64\x61pterEventInfo\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\"u\n\x13\x41\x64\x61pterEventInfoMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.AdapterEventInfo\"\x90\x01\n\x13\x41\x64\x61pterEventWarning\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\"{\n\x16\x41\x64\x61pterEventWarningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.AdapterEventWarning\"\xa0\x01\n\x11\x41\x64\x61pterEventError\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\x12\x10\n\x08\x65xc_info\x18\x05 \x01(\t\"w\n\x14\x41\x64\x61pterEventErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.AdapterEventError\"f\n\rNewConnection\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_type\x18\x02 \x01(\t\x12\x11\n\tconn_name\x18\x03 \x01(\t\"o\n\x10NewConnectionMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.NewConnection\"=\n\x10\x43onnectionReused\x12\x11\n\tconn_name\x18\x01 \x01(\t\x12\x16\n\x0eorig_conn_name\x18\x02 \x01(\t\"u\n\x13\x43onnectionReusedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.ConnectionReused\"0\n\x1b\x43onnectionLeftOpenInCleanup\x12\x11\n\tconn_name\x18\x01 \x01(\t\"\x8b\x01\n\x1e\x43onnectionLeftOpenInCleanupMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x36\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32(.proto_types.ConnectionLeftOpenInCleanup\".\n\x19\x43onnectionClosedInCleanup\x12\x11\n\tconn_name\x18\x01 \x01(\t\"\x87\x01\n\x1c\x43onnectionClosedInCleanupMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x34\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32&.proto_types.ConnectionClosedInCleanup\"f\n\x0eRollbackFailed\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\x12\x10\n\x08\x65xc_info\x18\x03 \x01(\t\"q\n\x11RollbackFailedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.RollbackFailed\"V\n\x10\x43onnectionClosed\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"u\n\x13\x43onnectionClosedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.ConnectionClosed\"X\n\x12\x43onnectionLeftOpen\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"y\n\x15\x43onnectionLeftOpenMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.ConnectionLeftOpen\"N\n\x08Rollback\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"e\n\x0bRollbackMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12#\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x15.proto_types.Rollback\"@\n\tCacheMiss\x12\x11\n\tconn_name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\"g\n\x0c\x43\x61\x63heMissMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x16.proto_types.CacheMiss\"b\n\rListRelations\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12/\n\trelations\x18\x03 \x03(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"o\n\x10ListRelationsMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.ListRelations\"g\n\x0e\x43onnectionUsed\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_type\x18\x02 \x01(\t\x12\x11\n\tconn_name\x18\x03 \x01(\t\"q\n\x11\x43onnectionUsedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.ConnectionUsed\"[\n\x08SQLQuery\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\x12\x0b\n\x03sql\x18\x03 \x01(\t\"e\n\x0bSQLQueryMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12#\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x15.proto_types.SQLQuery\"b\n\x0eSQLQueryStatus\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0e\n\x06status\x18\x02 \x01(\t\x12\x0f\n\x07\x65lapsed\x18\x03 \x01(\x02\"q\n\x11SQLQueryStatusMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.SQLQueryStatus\"O\n\tSQLCommit\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"g\n\x0cSQLCommitMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x16.proto_types.SQLCommit\"a\n\rColTypeChange\x12\x11\n\torig_type\x18\x01 \x01(\t\x12\x10\n\x08new_type\x18\x02 \x01(\t\x12+\n\x05table\x18\x03 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"o\n\x10\x43olTypeChangeMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.ColTypeChange\"@\n\x0eSchemaCreation\x12.\n\x08relation\x18\x01 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"q\n\x11SchemaCreationMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.SchemaCreation\"<\n\nSchemaDrop\x12.\n\x08relation\x18\x01 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"i\n\rSchemaDropMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.proto_types.SchemaDrop\"\xde\x01\n\x0b\x43\x61\x63heAction\x12\x0e\n\x06\x61\x63tion\x18\x01 \x01(\t\x12-\n\x07ref_key\x18\x02 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\x12/\n\tref_key_2\x18\x03 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\x12/\n\tref_key_3\x18\x04 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\x12.\n\x08ref_list\x18\x05 \x03(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"k\n\x0e\x43\x61\x63heActionMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12&\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.proto_types.CacheAction\"\x98\x01\n\x0e\x43\x61\x63heDumpGraph\x12\x33\n\x04\x64ump\x18\x01 \x03(\x0b\x32%.proto_types.CacheDumpGraph.DumpEntry\x12\x14\n\x0c\x62\x65\x66ore_after\x18\x02 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x03 \x01(\t\x1a+\n\tDumpEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"q\n\x11\x43\x61\x63heDumpGraphMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.CacheDumpGraph\"B\n\x11\x41\x64\x61pterRegistered\x12\x14\n\x0c\x61\x64\x61pter_name\x18\x01 \x01(\t\x12\x17\n\x0f\x61\x64\x61pter_version\x18\x02 \x01(\t\"w\n\x14\x41\x64\x61pterRegisteredMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.AdapterRegistered\"!\n\x12\x41\x64\x61pterImportError\x12\x0b\n\x03\x65xc\x18\x01 \x01(\t\"y\n\x15\x41\x64\x61pterImportErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.AdapterImportError\"#\n\x0fPluginLoadError\x12\x10\n\x08\x65xc_info\x18\x01 \x01(\t\"s\n\x12PluginLoadErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12*\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1c.proto_types.PluginLoadError\"a\n\x14NewConnectionOpening\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x18\n\x10\x63onnection_state\x18\x02 \x01(\t\"}\n\x17NewConnectionOpeningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12/\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32!.proto_types.NewConnectionOpening\"8\n\rCodeExecution\x12\x11\n\tconn_name\x18\x01 \x01(\t\x12\x14\n\x0c\x63ode_content\x18\x02 \x01(\t\"o\n\x10\x43odeExecutionMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.CodeExecution\"6\n\x13\x43odeExecutionStatus\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x65lapsed\x18\x02 \x01(\x02\"{\n\x16\x43odeExecutionStatusMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.CodeExecutionStatus\"%\n\x16\x43\x61talogGenerationError\x12\x0b\n\x03\x65xc\x18\x01 \x01(\t\"\x81\x01\n\x19\x43\x61talogGenerationErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x31\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32#.proto_types.CatalogGenerationError\"-\n\x13WriteCatalogFailure\x12\x16\n\x0enum_exceptions\x18\x01 \x01(\x05\"{\n\x16WriteCatalogFailureMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.WriteCatalogFailure\"\x1e\n\x0e\x43\x61talogWritten\x12\x0c\n\x04path\x18\x01 \x01(\t\"q\n\x11\x43\x61talogWrittenMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.CatalogWritten\"\x14\n\x12\x43\x61nnotGenerateDocs\"y\n\x15\x43\x61nnotGenerateDocsMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.CannotGenerateDocs\"\x11\n\x0f\x42uildingCatalog\"s\n\x12\x42uildingCatalogMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12*\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1c.proto_types.BuildingCatalog\"-\n\x18\x44\x61tabaseErrorRunningHook\x12\x11\n\thook_type\x18\x01 \x01(\t\"\x85\x01\n\x1b\x44\x61tabaseErrorRunningHookMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x33\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32%.proto_types.DatabaseErrorRunningHook\"4\n\x0cHooksRunning\x12\x11\n\tnum_hooks\x18\x01 \x01(\x05\x12\x11\n\thook_type\x18\x02 \x01(\t\"m\n\x0fHooksRunningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\'\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x19.proto_types.HooksRunning\"T\n\x14\x46inishedRunningStats\x12\x11\n\tstat_line\x18\x01 \x01(\t\x12\x11\n\texecution\x18\x02 \x01(\t\x12\x16\n\x0e\x65xecution_time\x18\x03 \x01(\x02\"}\n\x17\x46inishedRunningStatsMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12/\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32!.proto_types.FinishedRunningStats\"<\n\x15\x43onstraintNotEnforced\x12\x12\n\nconstraint\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x61pter\x18\x02 \x01(\t\"\x7f\n\x18\x43onstraintNotEnforcedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\".proto_types.ConstraintNotEnforced\"=\n\x16\x43onstraintNotSupported\x12\x12\n\nconstraint\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x61pter\x18\x02 \x01(\t\"\x81\x01\n\x19\x43onstraintNotSupportedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x31\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32#.proto_types.ConstraintNotSupportedb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x61\x64\x61pter_types.proto\x12\x0bproto_types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xab\x02\n\x16\x41\x64\x61pterCommonEventInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0b\n\x03msg\x18\x03 \x01(\t\x12\r\n\x05level\x18\x04 \x01(\t\x12\x15\n\rinvocation_id\x18\x05 \x01(\t\x12\x0b\n\x03pid\x18\x06 \x01(\x05\x12\x0e\n\x06thread\x18\x07 \x01(\t\x12&\n\x02ts\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x05\x65xtra\x18\t \x03(\x0b\x32..proto_types.AdapterCommonEventInfo.ExtraEntry\x12\x10\n\x08\x63\x61tegory\x18\n \x01(\t\x1a,\n\nExtraEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x13\x41\x64\x61pterNodeRelation\x12\x10\n\x08\x64\x61tabase\x18\n \x01(\t\x12\x0e\n\x06schema\x18\x0b \x01(\t\x12\r\n\x05\x61lias\x18\x0c \x01(\t\x12\x15\n\rrelation_name\x18\r \x01(\t\"\x9f\x02\n\x0f\x41\x64\x61pterNodeInfo\x12\x11\n\tnode_path\x18\x01 \x01(\t\x12\x11\n\tnode_name\x18\x02 \x01(\t\x12\x11\n\tunique_id\x18\x03 \x01(\t\x12\x15\n\rresource_type\x18\x04 \x01(\t\x12\x14\n\x0cmaterialized\x18\x05 \x01(\t\x12\x13\n\x0bnode_status\x18\x06 \x01(\t\x12\x17\n\x0fnode_started_at\x18\x07 \x01(\t\x12\x18\n\x10node_finished_at\x18\x08 \x01(\t\x12%\n\x04meta\x18\t \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x37\n\rnode_relation\x18\n \x01(\x0b\x32 .proto_types.AdapterNodeRelation\"G\n\x0fReferenceKeyMsg\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12\x12\n\nidentifier\x18\x03 \x01(\t\"?\n\x19\x41\x64\x61pterDeprecationWarning\x12\x10\n\x08old_name\x18\x01 \x01(\t\x12\x10\n\x08new_name\x18\x02 \x01(\t\"\x87\x01\n\x1c\x41\x64\x61pterDeprecationWarningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x34\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32&.proto_types.AdapterDeprecationWarning\"!\n\x1f\x43ollectFreshnessReturnSignature\"\x93\x01\n\"CollectFreshnessReturnSignatureMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12:\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32,.proto_types.CollectFreshnessReturnSignature\"\x8e\x01\n\x11\x41\x64\x61pterEventDebug\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\"w\n\x14\x41\x64\x61pterEventDebugMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.AdapterEventDebug\"\x8d\x01\n\x10\x41\x64\x61pterEventInfo\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\"u\n\x13\x41\x64\x61pterEventInfoMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.AdapterEventInfo\"\x90\x01\n\x13\x41\x64\x61pterEventWarning\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\"{\n\x16\x41\x64\x61pterEventWarningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.AdapterEventWarning\"\xa0\x01\n\x11\x41\x64\x61pterEventError\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08\x62\x61se_msg\x18\x03 \x01(\t\x12(\n\x04\x61rgs\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.ListValue\x12\x10\n\x08\x65xc_info\x18\x05 \x01(\t\"w\n\x14\x41\x64\x61pterEventErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.AdapterEventError\"f\n\rNewConnection\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_type\x18\x02 \x01(\t\x12\x11\n\tconn_name\x18\x03 \x01(\t\"o\n\x10NewConnectionMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.NewConnection\"=\n\x10\x43onnectionReused\x12\x11\n\tconn_name\x18\x01 \x01(\t\x12\x16\n\x0eorig_conn_name\x18\x02 \x01(\t\"u\n\x13\x43onnectionReusedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.ConnectionReused\"0\n\x1b\x43onnectionLeftOpenInCleanup\x12\x11\n\tconn_name\x18\x01 \x01(\t\"\x8b\x01\n\x1e\x43onnectionLeftOpenInCleanupMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x36\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32(.proto_types.ConnectionLeftOpenInCleanup\".\n\x19\x43onnectionClosedInCleanup\x12\x11\n\tconn_name\x18\x01 \x01(\t\"\x87\x01\n\x1c\x43onnectionClosedInCleanupMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x34\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32&.proto_types.ConnectionClosedInCleanup\"f\n\x0eRollbackFailed\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\x12\x10\n\x08\x65xc_info\x18\x03 \x01(\t\"q\n\x11RollbackFailedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.RollbackFailed\"V\n\x10\x43onnectionClosed\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"u\n\x13\x43onnectionClosedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.ConnectionClosed\"X\n\x12\x43onnectionLeftOpen\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"y\n\x15\x43onnectionLeftOpenMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.ConnectionLeftOpen\"N\n\x08Rollback\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"e\n\x0bRollbackMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12#\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x15.proto_types.Rollback\"@\n\tCacheMiss\x12\x11\n\tconn_name\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\"g\n\x0c\x43\x61\x63heMissMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x16.proto_types.CacheMiss\"b\n\rListRelations\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\t\x12/\n\trelations\x18\x03 \x03(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"o\n\x10ListRelationsMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.ListRelations\"g\n\x0e\x43onnectionUsed\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_type\x18\x02 \x01(\t\x12\x11\n\tconn_name\x18\x03 \x01(\t\"q\n\x11\x43onnectionUsedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.ConnectionUsed\"[\n\x08SQLQuery\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\x12\x0b\n\x03sql\x18\x03 \x01(\t\"e\n\x0bSQLQueryMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12#\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x15.proto_types.SQLQuery\"b\n\x0eSQLQueryStatus\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x0e\n\x06status\x18\x02 \x01(\t\x12\x0f\n\x07\x65lapsed\x18\x03 \x01(\x02\"q\n\x11SQLQueryStatusMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.SQLQueryStatus\"O\n\tSQLCommit\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x11\n\tconn_name\x18\x02 \x01(\t\"g\n\x0cSQLCommitMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x16.proto_types.SQLCommit\"a\n\rColTypeChange\x12\x11\n\torig_type\x18\x01 \x01(\t\x12\x10\n\x08new_type\x18\x02 \x01(\t\x12+\n\x05table\x18\x03 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"o\n\x10\x43olTypeChangeMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.ColTypeChange\"@\n\x0eSchemaCreation\x12.\n\x08relation\x18\x01 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"q\n\x11SchemaCreationMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.SchemaCreation\"<\n\nSchemaDrop\x12.\n\x08relation\x18\x01 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"i\n\rSchemaDropMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12%\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x17.proto_types.SchemaDrop\"\xde\x01\n\x0b\x43\x61\x63heAction\x12\x0e\n\x06\x61\x63tion\x18\x01 \x01(\t\x12-\n\x07ref_key\x18\x02 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\x12/\n\tref_key_2\x18\x03 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\x12/\n\tref_key_3\x18\x04 \x01(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\x12.\n\x08ref_list\x18\x05 \x03(\x0b\x32\x1c.proto_types.ReferenceKeyMsg\"k\n\x0e\x43\x61\x63heActionMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12&\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.proto_types.CacheAction\"\x98\x01\n\x0e\x43\x61\x63heDumpGraph\x12\x33\n\x04\x64ump\x18\x01 \x03(\x0b\x32%.proto_types.CacheDumpGraph.DumpEntry\x12\x14\n\x0c\x62\x65\x66ore_after\x18\x02 \x01(\t\x12\x0e\n\x06\x61\x63tion\x18\x03 \x01(\t\x1a+\n\tDumpEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"q\n\x11\x43\x61\x63heDumpGraphMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.CacheDumpGraph\"B\n\x11\x41\x64\x61pterRegistered\x12\x14\n\x0c\x61\x64\x61pter_name\x18\x01 \x01(\t\x12\x17\n\x0f\x61\x64\x61pter_version\x18\x02 \x01(\t\"w\n\x14\x41\x64\x61pterRegisteredMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1e.proto_types.AdapterRegistered\"!\n\x12\x41\x64\x61pterImportError\x12\x0b\n\x03\x65xc\x18\x01 \x01(\t\"y\n\x15\x41\x64\x61pterImportErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.AdapterImportError\"#\n\x0fPluginLoadError\x12\x10\n\x08\x65xc_info\x18\x01 \x01(\t\"s\n\x12PluginLoadErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12*\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1c.proto_types.PluginLoadError\"a\n\x14NewConnectionOpening\x12/\n\tnode_info\x18\x01 \x01(\x0b\x32\x1c.proto_types.AdapterNodeInfo\x12\x18\n\x10\x63onnection_state\x18\x02 \x01(\t\"}\n\x17NewConnectionOpeningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12/\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32!.proto_types.NewConnectionOpening\"8\n\rCodeExecution\x12\x11\n\tconn_name\x18\x01 \x01(\t\x12\x14\n\x0c\x63ode_content\x18\x02 \x01(\t\"o\n\x10\x43odeExecutionMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12(\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1a.proto_types.CodeExecution\"6\n\x13\x43odeExecutionStatus\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x65lapsed\x18\x02 \x01(\x02\"{\n\x16\x43odeExecutionStatusMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.CodeExecutionStatus\"%\n\x16\x43\x61talogGenerationError\x12\x0b\n\x03\x65xc\x18\x01 \x01(\t\"\x81\x01\n\x19\x43\x61talogGenerationErrorMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x31\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32#.proto_types.CatalogGenerationError\"-\n\x13WriteCatalogFailure\x12\x16\n\x0enum_exceptions\x18\x01 \x01(\x05\"{\n\x16WriteCatalogFailureMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12.\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32 .proto_types.WriteCatalogFailure\"\x1e\n\x0e\x43\x61talogWritten\x12\x0c\n\x04path\x18\x01 \x01(\t\"q\n\x11\x43\x61talogWrittenMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12)\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1b.proto_types.CatalogWritten\"\x14\n\x12\x43\x61nnotGenerateDocs\"y\n\x15\x43\x61nnotGenerateDocsMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12-\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1f.proto_types.CannotGenerateDocs\"\x11\n\x0f\x42uildingCatalog\"s\n\x12\x42uildingCatalogMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12*\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1c.proto_types.BuildingCatalog\"-\n\x18\x44\x61tabaseErrorRunningHook\x12\x11\n\thook_type\x18\x01 \x01(\t\"\x85\x01\n\x1b\x44\x61tabaseErrorRunningHookMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x33\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32%.proto_types.DatabaseErrorRunningHook\"4\n\x0cHooksRunning\x12\x11\n\tnum_hooks\x18\x01 \x01(\x05\x12\x11\n\thook_type\x18\x02 \x01(\t\"m\n\x0fHooksRunningMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\'\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x19.proto_types.HooksRunning\"T\n\x14\x46inishedRunningStats\x12\x11\n\tstat_line\x18\x01 \x01(\t\x12\x11\n\texecution\x18\x02 \x01(\t\x12\x16\n\x0e\x65xecution_time\x18\x03 \x01(\x02\"}\n\x17\x46inishedRunningStatsMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12/\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32!.proto_types.FinishedRunningStats\"<\n\x15\x43onstraintNotEnforced\x12\x12\n\nconstraint\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x61pter\x18\x02 \x01(\t\"\x7f\n\x18\x43onstraintNotEnforcedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x30\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\".proto_types.ConstraintNotEnforced\"=\n\x16\x43onstraintNotSupported\x12\x12\n\nconstraint\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x61pter\x18\x02 \x01(\t\"\x81\x01\n\x19\x43onstraintNotSupportedMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12\x31\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32#.proto_types.ConstraintNotSupported\"%\n\x10TypeCodeNotFound\x12\x11\n\ttype_code\x18\x01 \x01(\x05\"u\n\x13TypeCodeNotFoundMsg\x12\x31\n\x04info\x18\x01 \x01(\x0b\x32#.proto_types.AdapterCommonEventInfo\x12+\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x1d.proto_types.TypeCodeNotFoundb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -203,4 +202,8 @@ _globals['_CONSTRAINTNOTSUPPORTED']._serialized_end=8961 _globals['_CONSTRAINTNOTSUPPORTEDMSG']._serialized_start=8964 _globals['_CONSTRAINTNOTSUPPORTEDMSG']._serialized_end=9093 + _globals['_TYPECODENOTFOUND']._serialized_start=9095 + _globals['_TYPECODENOTFOUND']._serialized_end=9132 + _globals['_TYPECODENOTFOUNDMSG']._serialized_start=9134 + _globals['_TYPECODENOTFOUNDMSG']._serialized_end=9251 # @@protoc_insertion_point(module_scope) diff --git a/dbt/adapters/events/types.py b/dbt/adapters/events/types.py index c49fef9b..7d98269d 100644 --- a/dbt/adapters/events/types.py +++ b/dbt/adapters/events/types.py @@ -422,3 +422,16 @@ def message(self) -> str: "be ignored. Set 'warn_unsupported: false' on this constraint to ignore this warning." ) return line_wrap_message(warning_tag(msg)) + + +class TypeCodeNotFound(DebugLevel): + def code(self) -> str: + return "E050" + + def message(self) -> str: + msg = ( + f"The `type_code` {self.type_code} was not recognized, which may affect error " + "messages for enforced contracts that fail as well as `Column.data_type` values " + "returned by `get_column_schema_from_query`" + ) + return line_wrap_message(warning_tag(msg)) diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index bda7c6f4..22e71896 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -153,6 +153,7 @@ def test_event_codes(self): types.FinishedRunningStats(stat_line="", execution="", execution_time=0), types.ConstraintNotEnforced(constraint="", adapter=""), types.ConstraintNotSupported(constraint="", adapter=""), + types.TypeCodeNotFound(type_code=0), ] From b65b761fcf156a233b41d397679755c3755c982d Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 12 Apr 2024 12:43:14 -0700 Subject: [PATCH 04/66] Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch (#127) --- .../unreleased/Features-20240325-180611.yaml | 6 + dbt/adapters/base/impl.py | 117 ++++++++++++++---- dbt/adapters/capability.py | 3 + 3 files changed, 103 insertions(+), 23 deletions(-) create mode 100644 .changes/unreleased/Features-20240325-180611.yaml diff --git a/.changes/unreleased/Features-20240325-180611.yaml b/.changes/unreleased/Features-20240325-180611.yaml new file mode 100644 index 00000000..2299d2ee --- /dev/null +++ b/.changes/unreleased/Features-20240325-180611.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +time: 2024-03-25T18:06:11.816163-04:00 +custom: + Author: michelleark + Issue: "138" diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 5b4b8080..3abe5e09 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1062,6 +1062,7 @@ def execute_macro( project: Optional[str] = None, context_override: Optional[Dict[str, Any]] = None, kwargs: Optional[Dict[str, Any]] = None, + needs_conn: bool = False, ) -> AttrDict: """Look macro_name up in the manifest and execute its results. @@ -1074,6 +1075,10 @@ def execute_macro( execution context. :param kwargs: An optional dict of keyword args used to pass to the macro. + : param needs_conn: A boolean that indicates whether the specified macro + requires an open connection to execute. If needs_conn is True, a + connection is expected and opened if necessary. Otherwise (and by default), + no connection is expected prior to executing the macro. """ if kwargs is None: @@ -1106,6 +1111,10 @@ def execute_macro( macro_function = CallableMacroGenerator(macro, macro_context) + if needs_conn: + connection = self.connections.get_thread_connection() + self.connections.open(connection) + with self.connections.exception_handler(f"macro {macro_name}"): result = macro_function(**kwargs) return result @@ -1297,48 +1306,110 @@ def calculate_freshness( } return adapter_response, freshness + def calculate_freshness_from_metadata_batch( + self, + sources: List[BaseRelation], + macro_resolver: Optional[MacroResolverProtocol] = None, + ) -> Tuple[List[Optional[AdapterResponse]], Dict[BaseRelation, FreshnessResponse]]: + """ + Given a list of sources (BaseRelations), calculate the metadata-based freshness in batch. + This method should _not_ execute a warehouse query per source, but rather batch up + the sources into as few requests as possible to minimize the number of roundtrips required + to compute metadata-based freshness for each input source. + + :param sources: The list of sources to calculate metadata-based freshness for + :param macro_resolver: An optional macro_resolver to use for get_relation_last_modified + :return: a tuple where: + * the first element is a list of optional AdapterResponses indicating the response + for each request the method made to compute the freshness for the provided sources. + * the second element is a dictionary mapping an input source BaseRelation to a FreshnessResponse, + if it was possible to calculate a FreshnessResponse for the source. + """ + # Track schema, identifiers of sources for lookup from batch query + schema_identifier_to_source = { + ( + source.path.get_lowered_part(ComponentName.Schema), + source.path.get_lowered_part(ComponentName.Identifier), + ): source + for source in sources + } + + # Group metadata sources by information schema -- one query per information schema will be necessary + sources_by_info_schema: Dict[InformationSchema, List[BaseRelation]] = self._get_catalog_relations_by_info_schema(sources) + + freshness_responses: Dict[BaseRelation, FreshnessResponse] = {} + adapter_responses: List[Optional[AdapterResponse]] = [] + for ( + information_schema, + sources_for_information_schema, + ) in sources_by_info_schema.items(): + result = self.execute_macro( + GET_RELATION_LAST_MODIFIED_MACRO_NAME, + kwargs={ + "information_schema": information_schema, + "relations": sources_for_information_schema, + }, + macro_resolver=macro_resolver, + needs_conn=True, + ) + adapter_response, table = result.response, result.table # type: ignore[attr-defined] + adapter_responses.append(adapter_response) + + for row in table: + raw_relation, freshness_response = self._parse_freshness_row(row, table) + source_relation_for_result = schema_identifier_to_source[raw_relation] + freshness_responses[source_relation_for_result] = freshness_response + + return adapter_responses, freshness_responses + def calculate_freshness_from_metadata( self, source: BaseRelation, macro_resolver: Optional[MacroResolverProtocol] = None, ) -> Tuple[Optional[AdapterResponse], FreshnessResponse]: - kwargs: Dict[str, Any] = { - "information_schema": source.information_schema_only(), - "relations": [source], - } - result = self.execute_macro( - GET_RELATION_LAST_MODIFIED_MACRO_NAME, - kwargs=kwargs, + adapter_responses, freshness_responses = self.calculate_freshness_from_metadata_batch( + sources=[source], macro_resolver=macro_resolver, ) - adapter_response, table = result.response, result.table # type: ignore[attr-defined] - - try: - from dbt_common.clients.agate_helper import get_column_value_uncased + adapter_response = adapter_responses[0] if adapter_responses else None + return adapter_response, freshness_responses[source] - row = table[0] - last_modified_val = get_column_value_uncased("last_modified", row) - snapshotted_at_val = get_column_value_uncased("snapshotted_at", row) - except Exception: - raise MacroResultError(GET_RELATION_LAST_MODIFIED_MACRO_NAME, table) - - if last_modified_val is None: + def _create_freshness_response( + self, last_modified: Optional[datetime], snapshotted_at: Optional[datetime] + ) -> FreshnessResponse: + if last_modified is None: # Interpret missing value as "infinitely long ago" max_loaded_at = datetime(1, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) else: - max_loaded_at = _utc(last_modified_val, None, "last_modified") - - snapshotted_at = _utc(snapshotted_at_val, None, "snapshotted_at") + max_loaded_at = _utc(last_modified, None, "last_modified") + snapshotted_at = _utc(snapshotted_at, None, "snapshotted_at") age = (snapshotted_at - max_loaded_at).total_seconds() - freshness: FreshnessResponse = { "max_loaded_at": max_loaded_at, "snapshotted_at": snapshotted_at, "age": age, } - return adapter_response, freshness + return freshness + + def _parse_freshness_row(self, row: "agate.Row", table: "agate.Table") -> Tuple[Any, FreshnessResponse]: + from dbt_common.clients.agate_helper import get_column_value_uncased + + try: + last_modified_val = get_column_value_uncased("last_modified", row) + snapshotted_at_val = get_column_value_uncased("snapshotted_at", row) + identifier = get_column_value_uncased("identifier", row) + schema = get_column_value_uncased("schema", row) + except Exception: + raise MacroResultError(GET_RELATION_LAST_MODIFIED_MACRO_NAME, table) + + freshness_response = self._create_freshness_response( + last_modified_val, + snapshotted_at_val + ) + raw_relation = schema.lower().strip(), identifier.lower().strip() + return raw_relation, freshness_response def pre_model_hook(self, config: Mapping[str, Any]) -> Any: """A hook for running some operation before the model materialization diff --git a/dbt/adapters/capability.py b/dbt/adapters/capability.py index 745cb27a..305604c7 100644 --- a/dbt/adapters/capability.py +++ b/dbt/adapters/capability.py @@ -13,6 +13,9 @@ class Capability(str, Enum): TableLastModifiedMetadata = "TableLastModifiedMetadata" """Indicates support for determining the time of the last table modification by querying database metadata.""" + TableLastModifiedMetadataBatch = "TableLastModifiedMetadataBatch" + """Indicates support for performantly determining the time of the last table modification by querying database metadata in batch.""" + class Support(str, Enum): Unknown = "Unknown" From 5836b0e379881ebffe124ebe646b389e011db415 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 16 Apr 2024 21:32:07 +0000 Subject: [PATCH 05/66] Bumping version to 1.0.8b3 and generate changelog --- .changes/1.0.8-b3.md | 12 ++++++++++++ .../Features-20240323-160251.yaml | 0 .../Features-20240325-180611.yaml | 0 .../Features-20240409-084844.yaml | 0 .../Under the Hood-20240329-093307.yaml | 0 .../Under the Hood-20240410-184109.yaml | 0 CHANGELOG.md | 13 +++++++++++++ dbt/adapters/__about__.py | 2 +- 8 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .changes/1.0.8-b3.md rename .changes/{unreleased => 1.0.8}/Features-20240323-160251.yaml (100%) rename .changes/{unreleased => 1.0.8}/Features-20240325-180611.yaml (100%) rename .changes/{unreleased => 1.0.8}/Features-20240409-084844.yaml (100%) rename .changes/{unreleased => 1.0.8}/Under the Hood-20240329-093307.yaml (100%) rename .changes/{unreleased => 1.0.8}/Under the Hood-20240410-184109.yaml (100%) diff --git a/.changes/1.0.8-b3.md b/.changes/1.0.8-b3.md new file mode 100644 index 00000000..4c613d15 --- /dev/null +++ b/.changes/1.0.8-b3.md @@ -0,0 +1,12 @@ +## dbt-adapter 1.0.8-b3 - April 16, 2024 + +### Features + +* Debug log when `type_code` fails to convert to a `data_type` +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +* Support for sql fixtures in unit testing + +### Under the Hood + +* Add the option to set the log level of the AdapterRegistered event +* Update dependabot config to cover GHA diff --git a/.changes/unreleased/Features-20240323-160251.yaml b/.changes/1.0.8/Features-20240323-160251.yaml similarity index 100% rename from .changes/unreleased/Features-20240323-160251.yaml rename to .changes/1.0.8/Features-20240323-160251.yaml diff --git a/.changes/unreleased/Features-20240325-180611.yaml b/.changes/1.0.8/Features-20240325-180611.yaml similarity index 100% rename from .changes/unreleased/Features-20240325-180611.yaml rename to .changes/1.0.8/Features-20240325-180611.yaml diff --git a/.changes/unreleased/Features-20240409-084844.yaml b/.changes/1.0.8/Features-20240409-084844.yaml similarity index 100% rename from .changes/unreleased/Features-20240409-084844.yaml rename to .changes/1.0.8/Features-20240409-084844.yaml diff --git a/.changes/unreleased/Under the Hood-20240329-093307.yaml b/.changes/1.0.8/Under the Hood-20240329-093307.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240329-093307.yaml rename to .changes/1.0.8/Under the Hood-20240329-093307.yaml diff --git a/.changes/unreleased/Under the Hood-20240410-184109.yaml b/.changes/1.0.8/Under the Hood-20240410-184109.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240410-184109.yaml rename to .changes/1.0.8/Under the Hood-20240410-184109.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a024c80..63dc2e36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapter 1.0.8-b3 - April 16, 2024 + +### Features + +* Debug log when `type_code` fails to convert to a `data_type` +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +* Support for sql fixtures in unit testing + +### Under the Hood + +* Add the option to set the log level of the AdapterRegistered event +* Update dependabot config to cover GHA + ## dbt-adapter 1.0.0 - April 01, 2024 ### Fixes diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 11a716ec..2bc96277 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.0.0" +version = "1.0.8b3" From a7bbffcd4b67dbaf49fb5982cc015f6b6ca15741 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:47:44 -0500 Subject: [PATCH 06/66] reverting release generated pr to fix 1.0.8 to correct 1.0.0b3 (#176) --- .changes/1.0.8-b3.md | 12 ------------ .../Features-20240323-160251.yaml | 0 .../Features-20240325-180611.yaml | 0 .../Features-20240409-084844.yaml | 0 .../Under the Hood-20240329-093307.yaml | 0 .../Under the Hood-20240410-184109.yaml | 0 CHANGELOG.md | 13 ------------- dbt/adapters/__about__.py | 2 +- 8 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 .changes/1.0.8-b3.md rename .changes/{1.0.8 => unreleased}/Features-20240323-160251.yaml (100%) rename .changes/{1.0.8 => unreleased}/Features-20240325-180611.yaml (100%) rename .changes/{1.0.8 => unreleased}/Features-20240409-084844.yaml (100%) rename .changes/{1.0.8 => unreleased}/Under the Hood-20240329-093307.yaml (100%) rename .changes/{1.0.8 => unreleased}/Under the Hood-20240410-184109.yaml (100%) diff --git a/.changes/1.0.8-b3.md b/.changes/1.0.8-b3.md deleted file mode 100644 index 4c613d15..00000000 --- a/.changes/1.0.8-b3.md +++ /dev/null @@ -1,12 +0,0 @@ -## dbt-adapter 1.0.8-b3 - April 16, 2024 - -### Features - -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing - -### Under the Hood - -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA diff --git a/.changes/1.0.8/Features-20240323-160251.yaml b/.changes/unreleased/Features-20240323-160251.yaml similarity index 100% rename from .changes/1.0.8/Features-20240323-160251.yaml rename to .changes/unreleased/Features-20240323-160251.yaml diff --git a/.changes/1.0.8/Features-20240325-180611.yaml b/.changes/unreleased/Features-20240325-180611.yaml similarity index 100% rename from .changes/1.0.8/Features-20240325-180611.yaml rename to .changes/unreleased/Features-20240325-180611.yaml diff --git a/.changes/1.0.8/Features-20240409-084844.yaml b/.changes/unreleased/Features-20240409-084844.yaml similarity index 100% rename from .changes/1.0.8/Features-20240409-084844.yaml rename to .changes/unreleased/Features-20240409-084844.yaml diff --git a/.changes/1.0.8/Under the Hood-20240329-093307.yaml b/.changes/unreleased/Under the Hood-20240329-093307.yaml similarity index 100% rename from .changes/1.0.8/Under the Hood-20240329-093307.yaml rename to .changes/unreleased/Under the Hood-20240329-093307.yaml diff --git a/.changes/1.0.8/Under the Hood-20240410-184109.yaml b/.changes/unreleased/Under the Hood-20240410-184109.yaml similarity index 100% rename from .changes/1.0.8/Under the Hood-20240410-184109.yaml rename to .changes/unreleased/Under the Hood-20240410-184109.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 63dc2e36..9a024c80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,19 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapter 1.0.8-b3 - April 16, 2024 - -### Features - -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing - -### Under the Hood - -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA - ## dbt-adapter 1.0.0 - April 01, 2024 ### Fixes diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 2bc96277..11a716ec 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.0.8b3" +version = "1.0.0" From ca1974e8e0b8f99fd5dd69a99d9c9689455d7510 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 17 Apr 2024 01:04:37 +0000 Subject: [PATCH 07/66] Bumping version to 1.1.0rc1 and generate changelog --- .changes/1.1.0-rc1.md | 12 ++++++++++++ .../Features-20240323-160251.yaml | 0 .../Features-20240325-180611.yaml | 0 .../Features-20240409-084844.yaml | 0 .../Under the Hood-20240329-093307.yaml | 0 .../Under the Hood-20240410-184109.yaml | 0 CHANGELOG.md | 13 +++++++++++++ dbt/adapters/__about__.py | 2 +- 8 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 .changes/1.1.0-rc1.md rename .changes/{unreleased => 1.1.0}/Features-20240323-160251.yaml (100%) rename .changes/{unreleased => 1.1.0}/Features-20240325-180611.yaml (100%) rename .changes/{unreleased => 1.1.0}/Features-20240409-084844.yaml (100%) rename .changes/{unreleased => 1.1.0}/Under the Hood-20240329-093307.yaml (100%) rename .changes/{unreleased => 1.1.0}/Under the Hood-20240410-184109.yaml (100%) diff --git a/.changes/1.1.0-rc1.md b/.changes/1.1.0-rc1.md new file mode 100644 index 00000000..8b47b109 --- /dev/null +++ b/.changes/1.1.0-rc1.md @@ -0,0 +1,12 @@ +## dbt-adapter 1.1.0-rc1 - April 17, 2024 + +### Features + +* Debug log when `type_code` fails to convert to a `data_type` +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +* Support for sql fixtures in unit testing + +### Under the Hood + +* Add the option to set the log level of the AdapterRegistered event +* Update dependabot config to cover GHA diff --git a/.changes/unreleased/Features-20240323-160251.yaml b/.changes/1.1.0/Features-20240323-160251.yaml similarity index 100% rename from .changes/unreleased/Features-20240323-160251.yaml rename to .changes/1.1.0/Features-20240323-160251.yaml diff --git a/.changes/unreleased/Features-20240325-180611.yaml b/.changes/1.1.0/Features-20240325-180611.yaml similarity index 100% rename from .changes/unreleased/Features-20240325-180611.yaml rename to .changes/1.1.0/Features-20240325-180611.yaml diff --git a/.changes/unreleased/Features-20240409-084844.yaml b/.changes/1.1.0/Features-20240409-084844.yaml similarity index 100% rename from .changes/unreleased/Features-20240409-084844.yaml rename to .changes/1.1.0/Features-20240409-084844.yaml diff --git a/.changes/unreleased/Under the Hood-20240329-093307.yaml b/.changes/1.1.0/Under the Hood-20240329-093307.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240329-093307.yaml rename to .changes/1.1.0/Under the Hood-20240329-093307.yaml diff --git a/.changes/unreleased/Under the Hood-20240410-184109.yaml b/.changes/1.1.0/Under the Hood-20240410-184109.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240410-184109.yaml rename to .changes/1.1.0/Under the Hood-20240410-184109.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a024c80..0af6f692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapter 1.1.0-rc1 - April 17, 2024 + +### Features + +* Debug log when `type_code` fails to convert to a `data_type` +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +* Support for sql fixtures in unit testing + +### Under the Hood + +* Add the option to set the log level of the AdapterRegistered event +* Update dependabot config to cover GHA + ## dbt-adapter 1.0.0 - April 01, 2024 ### Fixes diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 11a716ec..d37cdcc7 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.0.0" +version = "1.1.0rc1" From 4774638f334908815ec685bb21877ac10ce200df Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 17 Apr 2024 16:41:18 -0400 Subject: [PATCH 08/66] [fix] use cast macro in default__get_empty_schema_sql (#165) --- .changes/unreleased/Fixes-20240411-185203.yaml | 6 ++++++ dbt/include/global_project/macros/adapters/columns.sql | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20240411-185203.yaml diff --git a/.changes/unreleased/Fixes-20240411-185203.yaml b/.changes/unreleased/Fixes-20240411-185203.yaml new file mode 100644 index 00000000..5df91f93 --- /dev/null +++ b/.changes/unreleased/Fixes-20240411-185203.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix adapter-specific cast handling for constraint enforcement +time: 2024-04-11T18:52:03.960202-07:00 +custom: + Author: michelleark + Issue: "166" diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index e1099649..663a827b 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -53,7 +53,7 @@ {%- do col_naked_numeric.append(col['name']) -%} {%- endif -%} {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %} - cast(null as {{ col['data_type'] }}) as {{ col_name }}{{ ", " if not loop.last }} + {{ cast('null', col['data_type']) }} as {{ col_name }}{{ ", " if not loop.last }} {%- endfor -%} {%- if (col_err | length) > 0 -%} {{ exceptions.column_type_missing(column_names=col_err) }} From a9f1e14ade33a14ecb1d08c1c38d9c01d86f2f75 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:42:43 -0400 Subject: [PATCH 09/66] Add and configure `pre-commit` (#174) --- .../Under the Hood-20240417-192843.yaml | 6 + .github/ISSUE_TEMPLATE/internal-epic.yml | 6 +- .../internal-feature-implementation.yml | 6 +- .github/actions/setup-hatch/action.yml | 8 ++ .github/workflows/code-quality.yml | 14 +-- .github/workflows/github-release.yml | 2 +- .github/workflows/release_prep_hatch.yml | 2 +- .gitignore | 2 +- .pre-commit-config.yaml | 112 +++++++++--------- CONTRIBUTING.md | 2 +- .../tests/adapter/hooks/test_model_hooks.py | 1 + .../unit_testing/test_invalid_input.py | 14 ++- dbt/adapters/__init__.py | 1 + dbt/adapters/base/connections.py | 4 +- dbt/adapters/base/impl.py | 17 +-- dbt/adapters/contracts/relation.py | 6 +- dbt/adapters/events/README.md | 2 +- dbt/adapters/factory.py | 15 +-- dbt/adapters/protocol.py | 78 ++++-------- dbt/adapters/relation_configs/README.md | 2 +- .../relation_configs/config_change.py | 4 +- .../macros/materializations/tests/helpers.sql | 2 +- .../macros/unit_test_sql/get_fixture_sql.sql | 2 +- pyproject.toml | 60 +--------- 24 files changed, 152 insertions(+), 216 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240417-192843.yaml diff --git a/.changes/unreleased/Under the Hood-20240417-192843.yaml b/.changes/unreleased/Under the Hood-20240417-192843.yaml new file mode 100644 index 00000000..94ec9292 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240417-192843.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Validate that dbt-core and dbt-adapters remain de-coupled +time: 2024-04-17T19:28:43.400023-04:00 +custom: + Author: mikealfare + Issue: "144" diff --git a/.github/ISSUE_TEMPLATE/internal-epic.yml b/.github/ISSUE_TEMPLATE/internal-epic.yml index 2f3bfaa7..8cfb3aef 100644 --- a/.github/ISSUE_TEMPLATE/internal-epic.yml +++ b/.github/ISSUE_TEMPLATE/internal-epic.yml @@ -30,7 +30,7 @@ body: label: Objectives description: | What are the high level goals we are trying to achieve? Provide use cases if available. - + Example: - [ ] Allow adapter maintainers to support custom materializations - [ ] Reduce maintenance burden for incremental users by offering materialized views @@ -48,7 +48,7 @@ body: Provide a list of GH issues that will build out this functionality. This may start empty, or as a checklist of items. However, it should eventually become a list of Feature Implementation tickets. - + Example: - [ ] Create new macro to select warehouse - [ ] https://github.com/dbt-labs/dbt-adapters/issues/42 @@ -66,7 +66,7 @@ body: Provide a list of relevant documentation. Is there a proof of concept? Does this require and RFCs, ADRs, etc.? If the documentation exists, link it; if it does not exist yet, reference it descriptively. - + Example: - [ ] RFC for updating connection interface to accept new parameters - [ ] POC: https://github.com/dbt-labs/dbt-adapters/pull/42 diff --git a/.github/ISSUE_TEMPLATE/internal-feature-implementation.yml b/.github/ISSUE_TEMPLATE/internal-feature-implementation.yml index ab3c4ffc..7a99365b 100644 --- a/.github/ISSUE_TEMPLATE/internal-feature-implementation.yml +++ b/.github/ISSUE_TEMPLATE/internal-feature-implementation.yml @@ -41,7 +41,7 @@ body: label: Acceptance criteria description: | What is the definition of done for this feature? Include any relevant edge cases and/or test cases. - + Example: - [ ] If there are no config changes, don't alter the materialized view - [ ] If the materialized view is scheduled to refresh, a manual refresh should not be issued @@ -58,7 +58,7 @@ body: description: | Provide scenarios to test. Include both positive and negative tests if possible. Link to existing similar tests if appropriate. - + Example: - [ ] Test with no `materialized` field in the model config. Expect pass. - [ ] Test with a `materialized` field in the model config that is not valid. Expect ConfigError. @@ -68,7 +68,7 @@ body: ``` validations: required: true - + - type: textarea attributes: label: Security diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml index 7b7780ef..6b15cdbf 100644 --- a/.github/actions/setup-hatch/action.yml +++ b/.github/actions/setup-hatch/action.yml @@ -18,5 +18,13 @@ runs: python-version: ${{ inputs.python-version }} - name: Install dev dependencies + shell: bash run: ${{ inputs.setup-command }} + + - name: Add brew to the PATH + shell: bash + run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: Install pre-commit shell: bash + run: brew install pre-commit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 4f5b392e..9c203847 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -10,17 +10,13 @@ on: permissions: read-all -defaults: - run: - shell: bash - # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true jobs: - lint: + code-quality: name: Code Quality runs-on: ubuntu-latest @@ -33,8 +29,6 @@ jobs: - name: Setup `hatch` uses: ./.github/actions/setup-hatch - - name: Run linters - run: hatch run lint:all - - - name: Run typechecks - run: hatch run typecheck:all + - name: Run code quality + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index fd20d9ab..1c2f41b5 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -256,4 +256,4 @@ jobs: RELEASE_NOTES: ${{ inputs.changelog_path }} COMMIT: ${{ inputs.sha }} PRERELEASE: ${{ steps.release_type.outputs.prerelease }} - DRAFT: ${{ steps.draft.outputs.draft }} \ No newline at end of file + DRAFT: ${{ steps.draft.outputs.draft }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index 32a267e0..b043e19e 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -539,4 +539,4 @@ jobs: - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" if: ${{ inputs.deploy_to == 'prod' && inputs.nightly_release == 'false' && needs.create-temp-branch.outputs.branch_name != '' }} run: | - git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} \ No newline at end of file + git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} diff --git a/.gitignore b/.gitignore index cf98fcf8..a14d6d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -153,4 +153,4 @@ dmypy.json cython_debug/ # PyCharm -.idea/ \ No newline at end of file +.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d80b955..caf34209 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,63 +1,57 @@ -# For more on configuring pre-commit hooks (see https://pre-commit.com/) - -# Force all unspecified python hooks to run python 3.8 default_language_version: - python: python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - args: [--unsafe] - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict -- repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - id: black - alias: black-check - stages: [manual] - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - "--check" - - "--diff" -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - - id: flake8 - alias: flake8-check - stages: [manual] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 - hooks: - - id: mypy - # N.B.: Mypy is... a bit fragile. - # - # By using `language: system` we run this hook in the local - # environment instead of a pre-commit isolated one. This is needed - # to ensure mypy correctly parses the project. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + - --force-exclude=dbt/adapters/events/adapter_types_pb2.py + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E704,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 - # It may cause trouble in that it adds environmental variables out - # of our control to the mix. Unfortunately, there's nothing we can - # do about per pre-commit's author. - # See https://github.com/pre-commit/pre-commit/issues/730 for details. - args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters/.* - language: system - - id: mypy - alias: mypy-check - stages: [manual] - args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters - language: system +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py + args: + - --explicit-package-bases + - --ignore-missing-imports + - --pretty + - --show-error-codes + files: ^dbt/adapters/ + additional_dependencies: + - types-PyYAML + - types-protobuf + - types-pytz diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb0002fa..e1b87103 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -163,7 +163,7 @@ Remember to commit and push the file that's created. ### Signing the CLA -> **_NOTE:_** All contributors to `dbt-adapter` must sign the +> **_NOTE:_** All contributors to `dbt-adapter` must sign the > [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA). Maintainers will be unable to merge contributions until the contributor signs the CLA. diff --git a/dbt-tests-adapter/dbt/tests/adapter/hooks/test_model_hooks.py b/dbt-tests-adapter/dbt/tests/adapter/hooks/test_model_hooks.py index 6a544af0..8423c9ca 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/hooks/test_model_hooks.py +++ b/dbt-tests-adapter/dbt/tests/adapter/hooks/test_model_hooks.py @@ -1,6 +1,7 @@ from pathlib import Path from dbt_common.exceptions import CompilationError + # TODO: does this belong in dbt-tests-adapter? from dbt.exceptions import ParsingError import pytest diff --git a/dbt-tests-adapter/dbt/tests/adapter/unit_testing/test_invalid_input.py b/dbt-tests-adapter/dbt/tests/adapter/unit_testing/test_invalid_input.py index 6c41ceb9..c5bf2a09 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/unit_testing/test_invalid_input.py +++ b/dbt-tests-adapter/dbt/tests/adapter/unit_testing/test_invalid_input.py @@ -46,16 +46,22 @@ def models(self): def test_invalid_input(self, project): results = run_dbt(["run"]) assert len(results) == 2 - + _, out = run_dbt_and_capture( ["test", "--select", "test_name:test_invalid_input_column_name"], expect_pass=False ) - assert "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." in out - + assert ( + "Invalid column name: 'invalid_column_name' in unit test fixture for 'my_upstream_model'." + in out + ) + _, out = run_dbt_and_capture( ["test", "--select", "test_name:test_invalid_expect_column_name"], expect_pass=False ) - assert "Invalid column name: 'invalid_column_name' in unit test fixture for expected output." in out + assert ( + "Invalid column name: 'invalid_column_name' in unit test fixture for expected output." + in out + ) class TestPostgresUnitTestInvalidInput(BaseUnitTestInvalidInput): diff --git a/dbt/adapters/__init__.py b/dbt/adapters/__init__.py index 65e1d483..1713e032 100644 --- a/dbt/adapters/__init__.py +++ b/dbt/adapters/__init__.py @@ -2,6 +2,7 @@ This adds all subdirectories of directories on `sys.path` to this package’s `__path__` . It effectively combines all adapters into a single namespace (dbt.adapter). """ + from pkgutil import extend_path __path__ = extend_path(__path__, __name__) diff --git a/dbt/adapters/base/connections.py b/dbt/adapters/base/connections.py index a3a4d98d..6e038297 100644 --- a/dbt/adapters/base/connections.py +++ b/dbt/adapters/base/connections.py @@ -165,7 +165,9 @@ def set_connection_name(self, name: Optional[str] = None) -> Connection: conn.handle = LazyHandle(self.open) # Add the connection to thread_connections for this thread self.set_thread_connection(conn) - fire_event(NewConnection(conn_name=conn_name, conn_type=self.TYPE, node_info=get_node_info())) + fire_event( + NewConnection(conn_name=conn_name, conn_type=self.TYPE, node_info=get_node_info()) + ) else: # existing connection either wasn't open or didn't have the right name if conn.state != "open": conn.handle = LazyHandle(self.open) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 3abe5e09..f58f8aba 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1328,14 +1328,16 @@ def calculate_freshness_from_metadata_batch( # Track schema, identifiers of sources for lookup from batch query schema_identifier_to_source = { ( - source.path.get_lowered_part(ComponentName.Schema), - source.path.get_lowered_part(ComponentName.Identifier), + source.path.get_lowered_part(ComponentName.Schema), # type: ignore + source.path.get_lowered_part(ComponentName.Identifier), # type: ignore ): source for source in sources } # Group metadata sources by information schema -- one query per information schema will be necessary - sources_by_info_schema: Dict[InformationSchema, List[BaseRelation]] = self._get_catalog_relations_by_info_schema(sources) + sources_by_info_schema: Dict[InformationSchema, List[BaseRelation]] = ( + self._get_catalog_relations_by_info_schema(sources) + ) freshness_responses: Dict[BaseRelation, FreshnessResponse] = {} adapter_responses: List[Optional[AdapterResponse]] = [] @@ -1393,7 +1395,9 @@ def _create_freshness_response( return freshness - def _parse_freshness_row(self, row: "agate.Row", table: "agate.Table") -> Tuple[Any, FreshnessResponse]: + def _parse_freshness_row( + self, row: "agate.Row", table: "agate.Table" + ) -> Tuple[Any, FreshnessResponse]: from dbt_common.clients.agate_helper import get_column_value_uncased try: @@ -1404,10 +1408,7 @@ def _parse_freshness_row(self, row: "agate.Row", table: "agate.Table") -> Tuple[ except Exception: raise MacroResultError(GET_RELATION_LAST_MODIFIED_MACRO_NAME, table) - freshness_response = self._create_freshness_response( - last_modified_val, - snapshotted_at_val - ) + freshness_response = self._create_freshness_response(last_modified_val, snapshotted_at_val) raw_relation = schema.lower().strip(), identifier.lower().strip() return raw_relation, freshness_response diff --git a/dbt/adapters/contracts/relation.py b/dbt/adapters/contracts/relation.py index 6a88d074..3560c2b3 100644 --- a/dbt/adapters/contracts/relation.py +++ b/dbt/adapters/contracts/relation.py @@ -40,11 +40,9 @@ class MaterializationConfig(Mapping, ABC): contract: MaterializationContract extra: Dict[str, Any] - def __contains__(self, item): - ... + def __contains__(self, item): ... - def __delitem__(self, key): - ... + def __delitem__(self, key): ... class RelationConfig(Protocol): diff --git a/dbt/adapters/events/README.md b/dbt/adapters/events/README.md index fe39a18e..c98488db 100644 --- a/dbt/adapters/events/README.md +++ b/dbt/adapters/events/README.md @@ -14,7 +14,7 @@ When events are processed via `fire_event`, nearly everything is logged. Whether We have switched from using betterproto to using google protobuf, because of a lack of support for Struct fields in betterproto. -The google protobuf interface is janky and very much non-Pythonic. The "generated" classes in types_pb2.py do not resemble regular Python classes. They do not have normal constructors; they can only be constructed empty. They can be "filled" by setting fields individually or using a json_format method like ParseDict. We have wrapped the logging events with a class (in types.py) which allows using a constructor -- keywords only, no positional parameters. +The google protobuf interface is janky and very much non-Pythonic. The "generated" classes in types_pb2.py do not resemble regular Python classes. They do not have normal constructors; they can only be constructed empty. They can be "filled" by setting fields individually or using a json_format method like ParseDict. We have wrapped the logging events with a class (in types.py) which allows using a constructor -- keywords only, no positional parameters. ## Required for Every Event diff --git a/dbt/adapters/factory.py b/dbt/adapters/factory.py index e63d43e1..b1854f67 100644 --- a/dbt/adapters/factory.py +++ b/dbt/adapters/factory.py @@ -101,17 +101,14 @@ def register_adapter( self, config: AdapterRequiredConfig, mp_context: SpawnContext, - adapter_registered_log_level: Optional[EventLevel] = EventLevel.INFO + adapter_registered_log_level: Optional[EventLevel] = EventLevel.INFO, ) -> None: adapter_name = config.credentials.type adapter_type = self.get_adapter_class_by_name(adapter_name) adapter_version = self._adapter_version(adapter_name) fire_event( - AdapterRegistered( - adapter_name=adapter_name, - adapter_version=adapter_version - ), - level=adapter_registered_log_level + AdapterRegistered(adapter_name=adapter_name, adapter_version=adapter_version), + level=adapter_registered_log_level, ) with self.lock: if adapter_name in self.adapters: @@ -199,9 +196,9 @@ def get_adapter_constraint_support(self, name: Optional[str]) -> List[str]: def register_adapter( - config: AdapterRequiredConfig, - mp_context: SpawnContext, - adapter_registered_log_level: Optional[EventLevel] = EventLevel.INFO + config: AdapterRequiredConfig, + mp_context: SpawnContext, + adapter_registered_log_level: Optional[EventLevel] = EventLevel.INFO, ) -> None: FACTORY.register_adapter(config, mp_context, adapter_registered_log_level) diff --git a/dbt/adapters/protocol.py b/dbt/adapters/protocol.py index bbfdd330..35219866 100644 --- a/dbt/adapters/protocol.py +++ b/dbt/adapters/protocol.py @@ -47,8 +47,7 @@ class ColumnProtocol(Protocol): class RelationProtocol(Protocol): @classmethod - def get_default_quote_policy(cls) -> Policy: - ... + def get_default_quote_policy(cls) -> Policy: ... @classmethod def create_from( @@ -56,8 +55,7 @@ def create_from( quoting: HasQuoting, relation_config: RelationConfig, **kwargs: Any, - ) -> Self: - ... + ) -> Self: ... AdapterConfig_T = TypeVar("AdapterConfig_T", bound=AdapterConfig) @@ -73,8 +71,7 @@ def __call__( config: AdapterRequiredConfig, macro_resolver: MacroResolverProtocol, package_name: Optional[str], - ) -> Dict[str, Any]: - ... + ) -> Dict[str, Any]: ... # TODO CT-211 @@ -96,81 +93,58 @@ class AdapterProtocol( # type: ignore[misc] ConnectionManager: Type[ConnectionManager_T] connections: ConnectionManager_T - def __init__(self, config: AdapterRequiredConfig) -> None: - ... + def __init__(self, config: AdapterRequiredConfig) -> None: ... - def set_macro_resolver(self, macro_resolver: MacroResolverProtocol) -> None: - ... + def set_macro_resolver(self, macro_resolver: MacroResolverProtocol) -> None: ... - def get_macro_resolver(self) -> Optional[MacroResolverProtocol]: - ... + def get_macro_resolver(self) -> Optional[MacroResolverProtocol]: ... - def clear_macro_resolver(self) -> None: - ... + def clear_macro_resolver(self) -> None: ... def set_macro_context_generator( self, macro_context_generator: MacroContextGeneratorCallable, - ) -> None: - ... + ) -> None: ... @classmethod def type(cls) -> str: pass - def set_query_header(self, query_header_context: Dict[str, Any]) -> None: - ... + def set_query_header(self, query_header_context: Dict[str, Any]) -> None: ... @staticmethod - def get_thread_identifier() -> Hashable: - ... + def get_thread_identifier() -> Hashable: ... - def get_thread_connection(self) -> Connection: - ... + def get_thread_connection(self) -> Connection: ... - def set_thread_connection(self, conn: Connection) -> None: - ... + def set_thread_connection(self, conn: Connection) -> None: ... - def get_if_exists(self) -> Optional[Connection]: - ... + def get_if_exists(self) -> Optional[Connection]: ... - def clear_thread_connection(self) -> None: - ... + def clear_thread_connection(self) -> None: ... - def clear_transaction(self) -> None: - ... + def clear_transaction(self) -> None: ... - def exception_handler(self, sql: str) -> ContextManager: - ... + def exception_handler(self, sql: str) -> ContextManager: ... - def set_connection_name(self, name: Optional[str] = None) -> Connection: - ... + def set_connection_name(self, name: Optional[str] = None) -> Connection: ... - def cancel_open(self) -> Optional[List[str]]: - ... + def cancel_open(self) -> Optional[List[str]]: ... - def open(cls, connection: Connection) -> Connection: - ... + def open(cls, connection: Connection) -> Connection: ... - def release(self) -> None: - ... + def release(self) -> None: ... - def cleanup_all(self) -> None: - ... + def cleanup_all(self) -> None: ... - def begin(self) -> None: - ... + def begin(self) -> None: ... - def commit(self) -> None: - ... + def commit(self) -> None: ... - def close(cls, connection: Connection) -> Connection: - ... + def close(cls, connection: Connection) -> Connection: ... - def commit_if_has_connection(self) -> None: - ... + def commit_if_has_connection(self) -> None: ... def execute( self, sql: str, auto_begin: bool = False, fetch: bool = False - ) -> Tuple[AdapterResponse, "agate.Table"]: - ... + ) -> Tuple[AdapterResponse, "agate.Table"]: ... diff --git a/dbt/adapters/relation_configs/README.md b/dbt/adapters/relation_configs/README.md index 6be3bc59..22d6bf78 100644 --- a/dbt/adapters/relation_configs/README.md +++ b/dbt/adapters/relation_configs/README.md @@ -1,6 +1,6 @@ # RelationConfig This package serves as an initial abstraction for managing the inspection of existing relations and determining -changes on those relations. It arose from the materialized view work and is currently only supporting +changes on those relations. It arose from the materialized view work and is currently only supporting materialized views for Postgres and Redshift as well as dynamic tables for Snowflake. There are three main classes in this package. diff --git a/dbt/adapters/relation_configs/config_change.py b/dbt/adapters/relation_configs/config_change.py index 9d3c8e01..a776dc6b 100644 --- a/dbt/adapters/relation_configs/config_change.py +++ b/dbt/adapters/relation_configs/config_change.py @@ -16,7 +16,9 @@ class RelationConfigChangeAction(StrEnum): @dataclass(frozen=True, eq=True, unsafe_hash=True) class RelationConfigChange(RelationConfigBase, ABC): action: RelationConfigChangeAction - context: Hashable # this is usually a RelationConfig, e.g. IndexConfig, but shouldn't be limited + context: ( + Hashable # this is usually a RelationConfig, e.g. IndexConfig, but shouldn't be limited + ) @property @abstractmethod diff --git a/dbt/include/global_project/macros/materializations/tests/helpers.sql b/dbt/include/global_project/macros/materializations/tests/helpers.sql index ead727d9..a385d1ea 100644 --- a/dbt/include/global_project/macros/materializations/tests/helpers.sql +++ b/dbt/include/global_project/macros/materializations/tests/helpers.sql @@ -41,4 +41,4 @@ dbt_internal_unit_test_expected as ( select * from dbt_internal_unit_test_actual union all select * from dbt_internal_unit_test_expected -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index ca39a39c..53d7a93b 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -79,7 +79,7 @@ union all {%- endif -%} {%- set column_type = column_name_to_data_types[column_name] %} - + {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#} {%- set column_value_clean = column_value -%} {%- if column_value is string -%} diff --git a/pyproject.toml b/pyproject.toml index 74c2374e..b08a8159 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,35 +54,16 @@ include = ["dbt/adapters", "dbt/include", "dbt/__init__.py"] [tool.hatch.envs.default] dependencies = [ "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", + 'pre-commit==3.7.0;python_version>="3.9"', + 'pre-commit==3.5.0;python_version=="3.8"', ] - -[tool.hatch.envs.lint] -detached = true -dependencies = [ - "black>=24.3", - "flake8", - "Flake8-pyproject", -] -[tool.hatch.envs.lint.scripts] -all = [ - "- black-only", - "- flake8-only", -] -black-only = "python -m black ." -flake8-only = "python -m flake8 ." - -[tool.hatch.envs.typecheck] -dependencies = [ - "mypy", - "types-PyYAML", - "types-protobuf", - "types-pytz", -] -[tool.hatch.envs.typecheck.scripts] -all = "python -m mypy ." +[tool.hatch.envs.default.scripts] +dev = "pre-commit install" +code-quality = "pre-commit run --all-files" [tool.hatch.envs.unit-tests] dependencies = [ + "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", "pytest", "pytest-dotenv", "pytest-xdist", @@ -114,37 +95,8 @@ check-sdist = [ ] protobuf = "protoc -I=./dbt/adapters/events --python_out=./dbt/adapters/events ./dbt/adapters/events/adapter_types.proto" -[tool.black] -extend-exclude = "dbt/adapters/events/adapter_types_pb2.py" -line-length = 99 -target-version = ['py38'] - -[tool.flake8] -select = ["E", "W", "F"] -ignore = ["E203", "E501", "E741", "W503", "W504"] -exclude = [ - "dbt/adapters/events/adapter_types_pb2.py", - "tests/functional", - "venv", -] -per-file-ignores = ["*/__init__.py: F401"] - [tool.mypy] -namespace_packages = true -show_error_codes = true -explicit_package_bases = true -ignore_missing_imports = true -pretty = true mypy_path = "third-party-stubs/" -files = [ - "dbt", - "tests/unit", -] -exclude = [ - "dbt/adapters/events/adapter_types_pb2.py", - "dbt-tests-adapter/dbt/__init__.py", # overlaps with `dbt/__init__.py` as expected for namespaces - "venv", -] [[tool.mypy.overrides]] module = ["dbt.adapters.events.adapter_types_pb2"] follow_imports = "skip" From 9cfb21ef861f2da82b86880de6d3fd0f5842c1f5 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Fri, 19 Apr 2024 13:49:15 -0600 Subject: [PATCH 10/66] Add tests for cross-database `cast` macro (#173) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Features-20240412-192040.yaml | 6 ++++ .../dbt/tests/adapter/utils/fixture_cast.py | 33 +++++++++++++++++++ .../dbt/tests/adapter/utils/test_cast.py | 23 +++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .changes/unreleased/Features-20240412-192040.yaml create mode 100644 dbt-tests-adapter/dbt/tests/adapter/utils/fixture_cast.py create mode 100644 dbt-tests-adapter/dbt/tests/adapter/utils/test_cast.py diff --git a/.changes/unreleased/Features-20240412-192040.yaml b/.changes/unreleased/Features-20240412-192040.yaml new file mode 100644 index 00000000..dc3f6799 --- /dev/null +++ b/.changes/unreleased/Features-20240412-192040.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Cross-database `cast` macro +time: 2024-04-12T19:20:40.904842-06:00 +custom: + Author: MichelleArk dbeatty10 + Issue: "84" diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_cast.py b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_cast.py new file mode 100644 index 00000000..fe29c706 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_cast.py @@ -0,0 +1,33 @@ +# cast + +seeds__data_cast_csv = """field,output +abc,abc +123,123 +, +""" + + +models__test_cast_sql = """ +with data as ( + + select * from {{ ref('data_cast') }} + +) + +select + {{ cast('field', api.Column.translate_type('string')) }} as actual, + output as expected + +from data +""" + + +models__test_cast_yml = """ +version: 2 +models: + - name: test_cast + data_tests: + - assert_equal: + actual: actual + expected: expected +""" diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_cast.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_cast.py new file mode 100644 index 00000000..686b7697 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/test_cast.py @@ -0,0 +1,23 @@ +import pytest + +from dbt.tests.adapter.utils import base_utils, fixture_cast + + +class BaseCast(base_utils.BaseUtils): + @pytest.fixture(scope="class") + def seeds(self): + return {"data_cast.csv": fixture_cast.seeds__data_cast_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "test_cast.yml": fixture_cast.models__test_cast_yml, + "test_cast.sql": self.interpolate_macro_namespace( + self.interpolate_macro_namespace(fixture_cast.models__test_cast_sql, "cast"), + "type_string", + ), + } + + +class TestCast(BaseCast): + pass From b61d66d72bc6ef519b25d6d6d15e7636091a10bf Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Mon, 22 Apr 2024 09:57:04 -0700 Subject: [PATCH 11/66] Make aliasing subquery configurable for render_limited (#179) --- .../unreleased/Features-20240418-155123.yaml | 6 +++++ .../unreleased/Features-20240418-155531.yaml | 7 ++++++ .../dbt/tests/adapter/empty/test_empty.py | 19 +++++++++++++++ dbt/adapters/base/relation.py | 15 ++++++++++-- tests/unit/test_relation.py | 23 +++++++++++++++---- 5 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/Features-20240418-155123.yaml create mode 100644 .changes/unreleased/Features-20240418-155531.yaml diff --git a/.changes/unreleased/Features-20240418-155123.yaml b/.changes/unreleased/Features-20240418-155123.yaml new file mode 100644 index 00000000..a3d9a56d --- /dev/null +++ b/.changes/unreleased/Features-20240418-155123.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow adapters to opt out of aliasing the subquery generated by render_limited +time: 2024-04-18T15:51:23.584295-07:00 +custom: + Author: colin-rogers-dbt + Issue: "124" diff --git a/.changes/unreleased/Features-20240418-155531.yaml b/.changes/unreleased/Features-20240418-155531.yaml new file mode 100644 index 00000000..02614b08 --- /dev/null +++ b/.changes/unreleased/Features-20240418-155531.yaml @@ -0,0 +1,7 @@ +kind: Features +body: subquery alias generated by render_limited now includes the relation name to + mitigate duplicate aliasing +time: 2024-04-18T15:55:31.826729-07:00 +custom: + Author: colin-rogers-dbt + Issue: ' 124' diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py index 373a13ee..2249d98d 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py @@ -72,5 +72,24 @@ def test_run_with_empty(self, project): self.assert_row_count(project, "model", 0) +class BaseTestEmptyInlineSourceRef(BaseTestEmpty): + @pytest.fixture(scope="class") + def models(self): + model_sql = """ + select * from {{ source('seed_sources', 'raw_source') }} as raw_source + """ + + return { + "model.sql": model_sql, + "sources.yml": schema_sources_yml, + } + + def test_run_with_empty(self, project): + # create source from seed + run_dbt(["seed"]) + run_dbt(["run", "--empty", "--debug"]) + self.assert_row_count(project, "model", 0) + + class TestEmpty(BaseTestEmpty): pass diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index d74b29bd..210a2dcd 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -47,6 +47,9 @@ class BaseRelation(FakeAPIObject, Hashable): quote_policy: Policy = field(default_factory=lambda: Policy()) dbt_created: bool = False limit: Optional[int] = None + require_alias: bool = ( + True # used to govern whether to add an alias when render_limited is called + ) # register relation types that can be renamed for the purpose of replacing relations using stages and backups # adding a relation type here also requires defining the associated rename macro @@ -205,14 +208,22 @@ def render(self) -> str: # if there is nothing set, this will return the empty string. return ".".join(part for _, part in self._render_iterator() if part is not None) + def _render_limited_alias(self) -> str: + """Some databases require an alias for subqueries (postgres, mysql) for all others we want to avoid adding + an alias as it has the potential to introduce issues with the query if the user also defines an alias. + """ + if self.require_alias: + return f" _dbt_limit_subq_{self.table}" + return "" + def render_limited(self) -> str: rendered = self.render() if self.limit is None: return rendered elif self.limit == 0: - return f"(select * from {rendered} where false limit 0) _dbt_limit_subq" + return f"(select * from {rendered} where false limit 0){self._render_limited_alias()}" else: - return f"(select * from {rendered} limit {self.limit}) _dbt_limit_subq" + return f"(select * from {rendered} limit {self.limit}){self._render_limited_alias()}" def quoted(self, identifier): return "{quote_char}{identifier}{quote_char}".format( diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py index aa9cda25..a1c01c5c 100644 --- a/tests/unit/test_relation.py +++ b/tests/unit/test_relation.py @@ -43,25 +43,38 @@ def test_can_be_replaced_default(): @pytest.mark.parametrize( - "limit,expected_result", + "limit,require_alias,expected_result", [ - (None, '"test_database"."test_schema"."test_identifier"'), + (None, False, '"test_database"."test_schema"."test_identifier"'), ( 0, - '(select * from "test_database"."test_schema"."test_identifier" where false limit 0) _dbt_limit_subq', + True, + '(select * from "test_database"."test_schema"."test_identifier" where false limit 0) _dbt_limit_subq_test_identifier', ), ( 1, - '(select * from "test_database"."test_schema"."test_identifier" limit 1) _dbt_limit_subq', + True, + '(select * from "test_database"."test_schema"."test_identifier" limit 1) _dbt_limit_subq_test_identifier', + ), + ( + 0, + False, + '(select * from "test_database"."test_schema"."test_identifier" where false limit 0)', + ), + ( + 1, + False, + '(select * from "test_database"."test_schema"."test_identifier" limit 1)', ), ], ) -def test_render_limited(limit, expected_result): +def test_render_limited(limit, require_alias, expected_result): my_relation = BaseRelation.create( database="test_database", schema="test_schema", identifier="test_identifier", limit=limit, + require_alias=require_alias, ) actual_result = my_relation.render_limited() assert actual_result == expected_result From 596b44cc0ab8ad23982e82540db0b75f1c50f215 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:32:22 -0700 Subject: [PATCH 12/66] fix BaseMacroArgsQueryComments by removing ref to dbt_version in fixture (#184) --- .changes/unreleased/Under the Hood-20240423-094843.yaml | 6 ++++++ .../dbt/tests/adapter/query_comment/fixtures.py | 1 - .../dbt/tests/adapter/query_comment/test_query_comment.py | 7 +------ 3 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240423-094843.yaml diff --git a/.changes/unreleased/Under the Hood-20240423-094843.yaml b/.changes/unreleased/Under the Hood-20240423-094843.yaml new file mode 100644 index 00000000..664f39a6 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240423-094843.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: remove dbt_version from query comment test fixture +time: 2024-04-23T09:48:43.208035-07:00 +custom: + Author: colin-rogers-dbt + Issue: "184" diff --git a/dbt-tests-adapter/dbt/tests/adapter/query_comment/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/query_comment/fixtures.py index d8848dc0..ccaf3292 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/query_comment/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/query_comment/fixtures.py @@ -10,7 +10,6 @@ {%- set comment_dict = dict( app='dbt++', macro_version='0.1.0', - dbt_version=dbt_version, message='blah: '~ message) -%} {{ return(comment_dict) }} {%- endmacro -%} diff --git a/dbt-tests-adapter/dbt/tests/adapter/query_comment/test_query_comment.py b/dbt-tests-adapter/dbt/tests/adapter/query_comment/test_query_comment.py index 66251c1d..4453c273 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/query_comment/test_query_comment.py +++ b/dbt-tests-adapter/dbt/tests/adapter/query_comment/test_query_comment.py @@ -1,5 +1,4 @@ import json -from importlib import import_module import pytest from dbt_common.exceptions import DbtRuntimeError @@ -53,19 +52,15 @@ def test_matches_comment(self, project): class BaseMacroArgsQueryComments(BaseDefaultQueryComments): - @pytest.fixture(scope="class") - def get_package_version(self, project): - return import_module("." + project.adapter_type, "dbt.adapters").__version__.version @pytest.fixture(scope="class") def project_config_update(self): return {"query-comment": "{{ return(ordered_to_json(query_header_args(target.name))) }}"} - def test_matches_comment(self, project, get_package_version): + def test_matches_comment(self, project): logs = self.run_get_json() expected_dct = { "app": "dbt++", - "dbt_version": get_package_version, "macro_version": "0.1.0", "message": f"blah: {project.adapter.config.target_name}", } From 6d097eead454faa49601d4426c87ec3213068f92 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 24 Apr 2024 22:30:39 -0700 Subject: [PATCH 13/66] Support py3.12 (#185) --- .changes/unreleased/Dependencies-20240424-140142.yaml | 6 ++++++ .github/workflows/unit-tests.yml | 2 +- dbt-tests-adapter/pyproject.toml | 1 + pyproject.toml | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Dependencies-20240424-140142.yaml diff --git a/.changes/unreleased/Dependencies-20240424-140142.yaml b/.changes/unreleased/Dependencies-20240424-140142.yaml new file mode 100644 index 00000000..c263e3c2 --- /dev/null +++ b/.changes/unreleased/Dependencies-20240424-140142.yaml @@ -0,0 +1,6 @@ +kind: Dependencies +body: add support for py3.12 +time: 2024-04-24T14:01:42.576383-07:00 +custom: + Author: colin-rogers-dbt + Issue: "57" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 26ff4aaa..7d720655 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Check out repository diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index c5df5998..35722b7b 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ # TODO: remove `dbt-core` dependency diff --git a/pyproject.toml b/pyproject.toml index b08a8159..a4b011a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dependencies = [ "dbt-common<2.0", From 99317a39c6d76f22a7924f2161a6ddf02e2ae415 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Mon, 29 Apr 2024 23:07:16 +0000 Subject: [PATCH 14/66] Bumping version to 1.1.0rc2 and generate changelog --- .changes/1.1.0-rc2.md | 20 ++++++++++++++++++ .../Dependencies-20240424-140142.yaml | 0 .../Features-20240412-192040.yaml | 0 .../Features-20240418-155123.yaml | 0 .../Features-20240418-155531.yaml | 0 .../Fixes-20240411-185203.yaml | 0 .../Under the Hood-20240417-192843.yaml | 0 .../Under the Hood-20240423-094843.yaml | 0 CHANGELOG.md | 21 +++++++++++++++++++ dbt/adapters/__about__.py | 2 +- 10 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .changes/1.1.0-rc2.md rename .changes/{unreleased => 1.1.0}/Dependencies-20240424-140142.yaml (100%) rename .changes/{unreleased => 1.1.0}/Features-20240412-192040.yaml (100%) rename .changes/{unreleased => 1.1.0}/Features-20240418-155123.yaml (100%) rename .changes/{unreleased => 1.1.0}/Features-20240418-155531.yaml (100%) rename .changes/{unreleased => 1.1.0}/Fixes-20240411-185203.yaml (100%) rename .changes/{unreleased => 1.1.0}/Under the Hood-20240417-192843.yaml (100%) rename .changes/{unreleased => 1.1.0}/Under the Hood-20240423-094843.yaml (100%) diff --git a/.changes/1.1.0-rc2.md b/.changes/1.1.0-rc2.md new file mode 100644 index 00000000..d5f63e58 --- /dev/null +++ b/.changes/1.1.0-rc2.md @@ -0,0 +1,20 @@ +## dbt-adapter 1.1.0-rc2 - April 29, 2024 + +### Features + +* Cross-database `cast` macro +* Allow adapters to opt out of aliasing the subquery generated by render_limited +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing + +### Fixes + +* Fix adapter-specific cast handling for constraint enforcement + +### Under the Hood + +* Validate that dbt-core and dbt-adapters remain de-coupled +* remove dbt_version from query comment test fixture + +### Dependencies + +* add support for py3.12 diff --git a/.changes/unreleased/Dependencies-20240424-140142.yaml b/.changes/1.1.0/Dependencies-20240424-140142.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240424-140142.yaml rename to .changes/1.1.0/Dependencies-20240424-140142.yaml diff --git a/.changes/unreleased/Features-20240412-192040.yaml b/.changes/1.1.0/Features-20240412-192040.yaml similarity index 100% rename from .changes/unreleased/Features-20240412-192040.yaml rename to .changes/1.1.0/Features-20240412-192040.yaml diff --git a/.changes/unreleased/Features-20240418-155123.yaml b/.changes/1.1.0/Features-20240418-155123.yaml similarity index 100% rename from .changes/unreleased/Features-20240418-155123.yaml rename to .changes/1.1.0/Features-20240418-155123.yaml diff --git a/.changes/unreleased/Features-20240418-155531.yaml b/.changes/1.1.0/Features-20240418-155531.yaml similarity index 100% rename from .changes/unreleased/Features-20240418-155531.yaml rename to .changes/1.1.0/Features-20240418-155531.yaml diff --git a/.changes/unreleased/Fixes-20240411-185203.yaml b/.changes/1.1.0/Fixes-20240411-185203.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240411-185203.yaml rename to .changes/1.1.0/Fixes-20240411-185203.yaml diff --git a/.changes/unreleased/Under the Hood-20240417-192843.yaml b/.changes/1.1.0/Under the Hood-20240417-192843.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240417-192843.yaml rename to .changes/1.1.0/Under the Hood-20240417-192843.yaml diff --git a/.changes/unreleased/Under the Hood-20240423-094843.yaml b/.changes/1.1.0/Under the Hood-20240423-094843.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240423-094843.yaml rename to .changes/1.1.0/Under the Hood-20240423-094843.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0af6f692..c528dfdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapter 1.1.0-rc2 - April 29, 2024 + +### Features + +* Cross-database `cast` macro +* Allow adapters to opt out of aliasing the subquery generated by render_limited +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing + +### Fixes + +* Fix adapter-specific cast handling for constraint enforcement + +### Under the Hood + +* Validate that dbt-core and dbt-adapters remain de-coupled +* remove dbt_version from query comment test fixture + +### Dependencies + +* add support for py3.12 + ## dbt-adapter 1.1.0-rc1 - April 17, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index d37cdcc7..685a4b25 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.1.0rc1" +version = "1.1.0rc2" From fc74270b84976bc65641dca1bf40a07c217ee0a0 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Mon, 29 Apr 2024 23:32:10 +0000 Subject: [PATCH 15/66] Bumping version to 1.8.0b2 and generate changelog --- .changes/1.8.0-b2.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.8.0-b2.md diff --git a/.changes/1.8.0-b2.md b/.changes/1.8.0-b2.md new file mode 100644 index 00000000..3954e30c --- /dev/null +++ b/.changes/1.8.0-b2.md @@ -0,0 +1 @@ +## dbt-adapter 1.8.0-b2 - April 29, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index c528dfdb..06d9d09b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapter 1.8.0-b2 - April 29, 2024 + ## dbt-adapter 1.1.0-rc2 - April 29, 2024 ### Features diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 6496f3e2..7d16c28f 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0b1" +version = "1.8.0b2" From 5fa2b678d858f763c7ab08fbd8a28de3c571baf5 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Tue, 30 Apr 2024 12:14:12 -0600 Subject: [PATCH 16/66] Use `dbt-adapters` throughout the contributing guide (#137) Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> --- .changes/unreleased/Docs-20240323-182010.yaml | 6 +++ CONTRIBUTING.md | 48 +++++++++---------- 2 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 .changes/unreleased/Docs-20240323-182010.yaml diff --git a/.changes/unreleased/Docs-20240323-182010.yaml b/.changes/unreleased/Docs-20240323-182010.yaml new file mode 100644 index 00000000..4cca5a83 --- /dev/null +++ b/.changes/unreleased/Docs-20240323-182010.yaml @@ -0,0 +1,6 @@ +kind: Docs +body: Use `dbt-adapters` throughout the contributing guide +time: 2024-03-23T18:20:10.796497-06:00 +custom: + Author: dbeatty10 + Issue: "136" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1b87103..c87ea23d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to `dbt-adapter` +# Contributing to `dbt-adapters` - [About this document](#about-this-document) - [Getting the code](#getting-the-code) @@ -10,8 +10,8 @@ ## About this document -This document is a guide for anyone interested in contributing to `dbt-adapter`. -It outlines how to install `dbt-adapter` for development, +This document is a guide for anyone interested in contributing to `dbt-adapters`. +It outlines how to install `dbt-adapters` for development, run tests locally, update documentation, and submit pull requests. This guide assumes users are developing on a Linux or MacOS system. The following utilities are needed or will be installed in this guide: @@ -31,7 +31,7 @@ Almost all information there is applicable here. ## Getting the code -`git` is required to download, modify, and sync the `dbt-adapter` code. +`git` is required to download, modify, and sync the `dbt-adapters` code. There are several ways to install Git. For MacOS: - Install [Xcode](https://developer.apple.com/support/xcode/) @@ -39,27 +39,27 @@ There are several ways to install Git. For MacOS: ### External contributors -Contributors external to the `dbt-labs` GitHub organization can contribute to `dbt-adapter` -by forking the `dbt-adapter` repository. For more on forking, check out the +Contributors external to the `dbt-labs` GitHub organization can contribute to `dbt-adapters` +by forking the `dbt-adapters` repository. For more on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). To contribute: -1. Fork the `dbt-labs/dbt-adapter` repository (e.g. `{forked-org}/dbt-adapter`) -2. Clone `{forked-org}/dbt-adapter` locally +1. Fork the `dbt-labs/dbt-adapters` repository (e.g. `{forked-org}/dbt-adapters`) +2. Clone `{forked-org}/dbt-adapters` locally 3. Check out a new branch locally 4. Make changes in the new branch -5. Push the new branch to `{forked-org}/dbt-adapter` -6. Open a pull request in `dbt-labs/dbt-adapter` to merge `{forked-org}/dbt-adapter/{new-branch}` into `main` +5. Push the new branch to `{forked-org}/dbt-adapters` +6. Open a pull request in `dbt-labs/dbt-adapters` to merge `{forked-org}/dbt-adapters/{new-branch}` into `main` ### dbt Labs contributors -Contributors in the `dbt Labs` GitHub organization have push access to the `dbt-adapter` repo. -Rather than forking `dbt-labs/dbt-adapter`, use `dbt-labs/dbt-adapter` directly. To contribute: +Contributors in the `dbt Labs` GitHub organization have push access to the `dbt-adapters` repo. +Rather than forking `dbt-labs/dbt-adapters`, use `dbt-labs/dbt-adapters` directly. To contribute: -1. Clone `dbt-labs/dbt-adapter` locally +1. Clone `dbt-labs/dbt-adapters` locally 2. Check out a new branch locally 3. Make changes in the new branch -4. Push the new branch to `dbt-labs/dbt-adapter` -5. Open a pull request in `dbt-labs/dbt-adapter` to merge `{new-branch}` into `main` +4. Push the new branch to `dbt-labs/dbt-adapters` +5. Open a pull request in `dbt-labs/dbt-adapters` to merge `{new-branch}` into `main` ## Developing @@ -72,19 +72,19 @@ Rather than forking `dbt-labs/dbt-adapter`, use `dbt-labs/dbt-adapter` directly. ``` 2. Configure and activate a virtual environment using `virtualenv` as described in [Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) -3. Install `dbt-adapter` and development dependencies in the virtual environment +3. Install `dbt-adapters` and development dependencies in the virtual environment ```shell pip install -e .[dev] ``` -When `dbt-adapter` is installed this way, any changes made to the `dbt-adapter` source code +When `dbt-adapters` is installed this way, any changes made to the `dbt-adapters` source code will be reflected in the virtual environment immediately. ## Testing -`dbt-adapter` contains [unit](https://github.com/dbt-labs/dbt-adapter/tree/main/tests/unit) -and [functional](https://github.com/dbt-labs/dbt-adapter/tree/main/tests/functional) tests. +`dbt-adapters` contains [unit](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit) +and [functional](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/functional) tests. ### Unit tests @@ -134,7 +134,7 @@ python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name::$ ### User documentation -Many changes will require an update to `dbt-adapter` user documentation. +Many changes will require an update to `dbt-adapters` user documentation. All contributors, whether internal or external, are encouraged to open an issue or PR in the docs repo when submitting user-facing changes. Here are some relevant links: @@ -145,7 +145,7 @@ in the docs repo when submitting user-facing changes. Here are some relevant lin ### CHANGELOG entry -`dbt-adapter` uses [changie](https://changie.dev) to generate `CHANGELOG` entries. +`dbt-adapters` uses [changie](https://changie.dev) to generate `CHANGELOG` entries. Follow the steps to [install `changie`](https://changie.dev/guide/installation/). Once changie is installed and the PR is created, run: @@ -172,11 +172,11 @@ Even without a CLA, anyone is welcome to open issues and comment on existing iss ### Opening a pull request -A `dbt-adapter` maintainer will be assigned to review each PR based on priority and capacity. +A `dbt-adapters` maintainer will be assigned to review each PR based on priority and capacity. They may suggest code revisions for style and clarity or they may request additional tests. These are good things! dbt Labs believes that contributing high-quality code is a collaborative effort. -The same process is followed whether the contributor is external or another `dbt-adapter` maintainer. +The same process is followed whether the contributor is external or another `dbt-adapters` maintainer. Once all tests are passing and the PR has been approved by the appropriate code owners, -a `dbt-adapter` maintainer will merge the changes into `main`. +a `dbt-adapters` maintainer will merge the changes into `main`. And that's it! Happy developing :tada: From b43892bd2bf9ad9027f9ae0f971a1896bb79b5e6 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 1 May 2024 21:03:02 +0000 Subject: [PATCH 17/66] Bumping version to 1.1.0 and generate changelog --- .changes/1.1.0-rc1.md | 12 --------- .changes/{1.1.0-rc2.md => 1.1.0.md} | 11 +++++++- .../1.1.0/Dependencies-20240424-140142.yaml | 6 ----- .changes/1.1.0/Features-20240323-160251.yaml | 6 ----- .changes/1.1.0/Features-20240325-180611.yaml | 6 ----- .changes/1.1.0/Features-20240409-084844.yaml | 6 ----- .changes/1.1.0/Features-20240412-192040.yaml | 6 ----- .changes/1.1.0/Features-20240418-155123.yaml | 6 ----- .changes/1.1.0/Features-20240418-155531.yaml | 7 ----- .changes/1.1.0/Fixes-20240411-185203.yaml | 6 ----- .../1.1.0/Under the Hood-20240329-093307.yaml | 6 ----- .../1.1.0/Under the Hood-20240410-184109.yaml | 6 ----- .../1.1.0/Under the Hood-20240417-192843.yaml | 6 ----- .../1.1.0/Under the Hood-20240423-094843.yaml | 6 ----- .changes/1.8.0-b2.md | 1 - .changes/unreleased/Docs-20240323-182010.yaml | 6 ----- CHANGELOG.md | 26 +++++++------------ dbt/adapters/__about__.py | 2 +- 18 files changed, 21 insertions(+), 110 deletions(-) delete mode 100644 .changes/1.1.0-rc1.md rename .changes/{1.1.0-rc2.md => 1.1.0.md} (53%) delete mode 100644 .changes/1.1.0/Dependencies-20240424-140142.yaml delete mode 100644 .changes/1.1.0/Features-20240323-160251.yaml delete mode 100644 .changes/1.1.0/Features-20240325-180611.yaml delete mode 100644 .changes/1.1.0/Features-20240409-084844.yaml delete mode 100644 .changes/1.1.0/Features-20240412-192040.yaml delete mode 100644 .changes/1.1.0/Features-20240418-155123.yaml delete mode 100644 .changes/1.1.0/Features-20240418-155531.yaml delete mode 100644 .changes/1.1.0/Fixes-20240411-185203.yaml delete mode 100644 .changes/1.1.0/Under the Hood-20240329-093307.yaml delete mode 100644 .changes/1.1.0/Under the Hood-20240410-184109.yaml delete mode 100644 .changes/1.1.0/Under the Hood-20240417-192843.yaml delete mode 100644 .changes/1.1.0/Under the Hood-20240423-094843.yaml delete mode 100644 .changes/1.8.0-b2.md delete mode 100644 .changes/unreleased/Docs-20240323-182010.yaml diff --git a/.changes/1.1.0-rc1.md b/.changes/1.1.0-rc1.md deleted file mode 100644 index 8b47b109..00000000 --- a/.changes/1.1.0-rc1.md +++ /dev/null @@ -1,12 +0,0 @@ -## dbt-adapter 1.1.0-rc1 - April 17, 2024 - -### Features - -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing - -### Under the Hood - -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA diff --git a/.changes/1.1.0-rc2.md b/.changes/1.1.0.md similarity index 53% rename from .changes/1.1.0-rc2.md rename to .changes/1.1.0.md index d5f63e58..c43ef9aa 100644 --- a/.changes/1.1.0-rc2.md +++ b/.changes/1.1.0.md @@ -1,7 +1,10 @@ -## dbt-adapter 1.1.0-rc2 - April 29, 2024 +## dbt-adapter 1.1.0 - May 01, 2024 ### Features +* Debug log when `type_code` fails to convert to a `data_type` +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +* Support for sql fixtures in unit testing * Cross-database `cast` macro * Allow adapters to opt out of aliasing the subquery generated by render_limited * subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing @@ -10,8 +13,14 @@ * Fix adapter-specific cast handling for constraint enforcement +### Docs + +* Use `dbt-adapters` throughout the contributing guide + ### Under the Hood +* Add the option to set the log level of the AdapterRegistered event +* Update dependabot config to cover GHA * Validate that dbt-core and dbt-adapters remain de-coupled * remove dbt_version from query comment test fixture diff --git a/.changes/1.1.0/Dependencies-20240424-140142.yaml b/.changes/1.1.0/Dependencies-20240424-140142.yaml deleted file mode 100644 index c263e3c2..00000000 --- a/.changes/1.1.0/Dependencies-20240424-140142.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: add support for py3.12 -time: 2024-04-24T14:01:42.576383-07:00 -custom: - Author: colin-rogers-dbt - Issue: "57" diff --git a/.changes/1.1.0/Features-20240323-160251.yaml b/.changes/1.1.0/Features-20240323-160251.yaml deleted file mode 100644 index c4b59464..00000000 --- a/.changes/1.1.0/Features-20240323-160251.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Debug log when `type_code` fails to convert to a `data_type` -time: 2024-03-23T16:02:51.306658-06:00 -custom: - Author: dbeatty10 - Issue: "8912" diff --git a/.changes/1.1.0/Features-20240325-180611.yaml b/.changes/1.1.0/Features-20240325-180611.yaml deleted file mode 100644 index 2299d2ee..00000000 --- a/.changes/1.1.0/Features-20240325-180611.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -time: 2024-03-25T18:06:11.816163-04:00 -custom: - Author: michelleark - Issue: "138" diff --git a/.changes/1.1.0/Features-20240409-084844.yaml b/.changes/1.1.0/Features-20240409-084844.yaml deleted file mode 100644 index 4fceebda..00000000 --- a/.changes/1.1.0/Features-20240409-084844.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support for sql fixtures in unit testing -time: 2024-04-09T08:48:44.441106-04:00 -custom: - Author: gshank - Issue: "148" diff --git a/.changes/1.1.0/Features-20240412-192040.yaml b/.changes/1.1.0/Features-20240412-192040.yaml deleted file mode 100644 index dc3f6799..00000000 --- a/.changes/1.1.0/Features-20240412-192040.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Cross-database `cast` macro -time: 2024-04-12T19:20:40.904842-06:00 -custom: - Author: MichelleArk dbeatty10 - Issue: "84" diff --git a/.changes/1.1.0/Features-20240418-155123.yaml b/.changes/1.1.0/Features-20240418-155123.yaml deleted file mode 100644 index a3d9a56d..00000000 --- a/.changes/1.1.0/Features-20240418-155123.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Allow adapters to opt out of aliasing the subquery generated by render_limited -time: 2024-04-18T15:51:23.584295-07:00 -custom: - Author: colin-rogers-dbt - Issue: "124" diff --git a/.changes/1.1.0/Features-20240418-155531.yaml b/.changes/1.1.0/Features-20240418-155531.yaml deleted file mode 100644 index 02614b08..00000000 --- a/.changes/1.1.0/Features-20240418-155531.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: subquery alias generated by render_limited now includes the relation name to - mitigate duplicate aliasing -time: 2024-04-18T15:55:31.826729-07:00 -custom: - Author: colin-rogers-dbt - Issue: ' 124' diff --git a/.changes/1.1.0/Fixes-20240411-185203.yaml b/.changes/1.1.0/Fixes-20240411-185203.yaml deleted file mode 100644 index 5df91f93..00000000 --- a/.changes/1.1.0/Fixes-20240411-185203.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix adapter-specific cast handling for constraint enforcement -time: 2024-04-11T18:52:03.960202-07:00 -custom: - Author: michelleark - Issue: "166" diff --git a/.changes/1.1.0/Under the Hood-20240329-093307.yaml b/.changes/1.1.0/Under the Hood-20240329-093307.yaml deleted file mode 100644 index 85a00a09..00000000 --- a/.changes/1.1.0/Under the Hood-20240329-093307.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add the option to set the log level of the AdapterRegistered event -time: 2024-03-29T09:33:07.737464-05:00 -custom: - Author: emmyoop - Issue: "141" diff --git a/.changes/1.1.0/Under the Hood-20240410-184109.yaml b/.changes/1.1.0/Under the Hood-20240410-184109.yaml deleted file mode 100644 index c0fc5b92..00000000 --- a/.changes/1.1.0/Under the Hood-20240410-184109.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update dependabot config to cover GHA -time: 2024-04-10T18:41:09.719652-04:00 -custom: - Author: mikealfare - Issue: "161" diff --git a/.changes/1.1.0/Under the Hood-20240417-192843.yaml b/.changes/1.1.0/Under the Hood-20240417-192843.yaml deleted file mode 100644 index 94ec9292..00000000 --- a/.changes/1.1.0/Under the Hood-20240417-192843.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Validate that dbt-core and dbt-adapters remain de-coupled -time: 2024-04-17T19:28:43.400023-04:00 -custom: - Author: mikealfare - Issue: "144" diff --git a/.changes/1.1.0/Under the Hood-20240423-094843.yaml b/.changes/1.1.0/Under the Hood-20240423-094843.yaml deleted file mode 100644 index 664f39a6..00000000 --- a/.changes/1.1.0/Under the Hood-20240423-094843.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: remove dbt_version from query comment test fixture -time: 2024-04-23T09:48:43.208035-07:00 -custom: - Author: colin-rogers-dbt - Issue: "184" diff --git a/.changes/1.8.0-b2.md b/.changes/1.8.0-b2.md deleted file mode 100644 index 3954e30c..00000000 --- a/.changes/1.8.0-b2.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapter 1.8.0-b2 - April 29, 2024 diff --git a/.changes/unreleased/Docs-20240323-182010.yaml b/.changes/unreleased/Docs-20240323-182010.yaml deleted file mode 100644 index 4cca5a83..00000000 --- a/.changes/unreleased/Docs-20240323-182010.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Use `dbt-adapters` throughout the contributing guide -time: 2024-03-23T18:20:10.796497-06:00 -custom: - Author: dbeatty10 - Issue: "136" diff --git a/CHANGELOG.md b/CHANGELOG.md index 06d9d09b..d9c85ffb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapter 1.8.0-b2 - April 29, 2024 - -## dbt-adapter 1.1.0-rc2 - April 29, 2024 +## dbt-adapter 1.1.0 - May 01, 2024 ### Features +* Debug log when `type_code` fails to convert to a `data_type` +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch +* Support for sql fixtures in unit testing * Cross-database `cast` macro * Allow adapters to opt out of aliasing the subquery generated by render_limited * subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing @@ -19,8 +20,14 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Fix adapter-specific cast handling for constraint enforcement +### Docs + +* Use `dbt-adapters` throughout the contributing guide + ### Under the Hood +* Add the option to set the log level of the AdapterRegistered event +* Update dependabot config to cover GHA * Validate that dbt-core and dbt-adapters remain de-coupled * remove dbt_version from query comment test fixture @@ -28,19 +35,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * add support for py3.12 -## dbt-adapter 1.1.0-rc1 - April 17, 2024 - -### Features - -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing - -### Under the Hood - -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA - ## dbt-adapter 1.0.0 - April 01, 2024 ### Fixes diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 685a4b25..b2b60a55 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.1.0rc2" +version = "1.1.0" From 2bfc2fd507ab7316a3d8382144cc9e6dfa57443b Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Wed, 1 May 2024 21:17:50 +0000 Subject: [PATCH 18/66] Bumping version to 1.8.0rc1 and generate changelog --- .changes/1.8.0-rc1.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.8.0-rc1.md diff --git a/.changes/1.8.0-rc1.md b/.changes/1.8.0-rc1.md new file mode 100644 index 00000000..7467c510 --- /dev/null +++ b/.changes/1.8.0-rc1.md @@ -0,0 +1 @@ +## dbt-adapter 1.8.0-rc1 - May 01, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c85ffb..168dff4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapter 1.8.0-rc1 - May 01, 2024 + ## dbt-adapter 1.1.0 - May 01, 2024 ### Features diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 7d16c28f..037ac1fa 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0b2" +version = "1.8.0rc1" From c47c8b5cfc97a351dd5fb804a4c6fb02d8208df0 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Wed, 1 May 2024 17:38:07 -0600 Subject: [PATCH 19/66] Cross-database `date` macro (#191) Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Features-20240501-091604.yaml | 6 +++ .../dbt/tests/adapter/utils/fixture_date.py | 45 +++++++++++++++++++ .../dbt/tests/adapter/utils/test_date.py | 18 ++++++++ .../global_project/macros/utils/date.sql | 10 +++++ 4 files changed, 79 insertions(+) create mode 100644 .changes/unreleased/Features-20240501-091604.yaml create mode 100644 dbt-tests-adapter/dbt/tests/adapter/utils/fixture_date.py create mode 100644 dbt-tests-adapter/dbt/tests/adapter/utils/test_date.py create mode 100644 dbt/include/global_project/macros/utils/date.sql diff --git a/.changes/unreleased/Features-20240501-091604.yaml b/.changes/unreleased/Features-20240501-091604.yaml new file mode 100644 index 00000000..fb934881 --- /dev/null +++ b/.changes/unreleased/Features-20240501-091604.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Cross-database `date` macro +time: 2024-05-01T09:16:04.909686-06:00 +custom: + Author: dbeatty10 + Issue: https://github.com/dbt-labs/dbt-core/issues/8831 diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_date.py b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_date.py new file mode 100644 index 00000000..7c9b2084 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_date.py @@ -0,0 +1,45 @@ +# date + +models__test_date_sql = """ +with generated_dates as ( + + {{ + dbt.date_spine( + "day", + date(2023, 9, 7), + date(2023, 9, 10), + ) + }} + +), + +expected_dates as ( + + select cast('2023-09-07' as date) as expected + union all + select cast('2023-09-08' as date) as expected + union all + select cast('2023-09-09' as date) as expected + +), + +joined as ( + select + generated_dates.date_day, + expected_dates.expected + from generated_dates + full outer join expected_dates on generated_dates.date_day = expected_dates.expected +) + +select * from joined +""" + +models__test_date_yml = """ +version: 2 +models: + - name: test_date + data_tests: + - assert_equal: + actual: date_day + expected: expected +""" diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_date.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_date.py new file mode 100644 index 00000000..a096b5af --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/test_date.py @@ -0,0 +1,18 @@ +import pytest + +from dbt.tests.adapter.utils import base_utils, fixture_date + + +class BaseDate(base_utils.BaseUtils): + @pytest.fixture(scope="class") + def models(self): + return { + "test_date.yml": fixture_date.models__test_date_yml, + "test_date.sql": self.interpolate_macro_namespace( + fixture_date.models__test_date_sql, "date" + ), + } + + +class TestDate(BaseDate): + pass diff --git a/dbt/include/global_project/macros/utils/date.sql b/dbt/include/global_project/macros/utils/date.sql new file mode 100644 index 00000000..d41b4438 --- /dev/null +++ b/dbt/include/global_project/macros/utils/date.sql @@ -0,0 +1,10 @@ +{% macro date(year, month, day) %} + {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }} +{% endmacro %} + + +{% macro default__date(year, month, day) -%} + {%- set dt = modules.datetime.date(year, month, day) -%} + {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%} + to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD') +{%- endmacro %} From 0a4c9f0d9c13fdbb0ea4561c9a3f50616e1347b0 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 2 May 2024 12:54:03 -0700 Subject: [PATCH 20/66] remove upper bound on dbt-core for dbt-tests-adapters (#193) --- dbt-tests-adapter/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index 35722b7b..20342078 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ # TODO: remove `dbt-core` dependency - "dbt-core>=1.8.0a1,<1.9.0", + "dbt-core>=1.8.0a1", # `dbt-tests-adapter` will ultimately depend on the packages below # `dbt-tests-adapter` temporarily uses `dbt-core` for a dbt runner # `dbt-core` takes the packages below as dependencies, so they are unpinned to avoid conflicts From df393ae6ae62e459c7b08dc2ce6e64a4c9d315cd Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 2 May 2024 15:51:21 -0700 Subject: [PATCH 21/66] remove reference to node.deffered in seed assertion for clone test (#194) --- .changes/unreleased/Fixes-20240502-150038.yaml | 7 +++++++ .../dbt/tests/adapter/dbt_clone/test_dbt_clone.py | 4 +--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240502-150038.yaml diff --git a/.changes/unreleased/Fixes-20240502-150038.yaml b/.changes/unreleased/Fixes-20240502-150038.yaml new file mode 100644 index 00000000..ed80bfc5 --- /dev/null +++ b/.changes/unreleased/Fixes-20240502-150038.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Update Clone test to reflect core change removing `deferred` attribute from + nodes +time: 2024-05-02T15:00:38.882727-07:00 +custom: + Author: colin-rogers-dbt + Issue: "194" diff --git a/dbt-tests-adapter/dbt/tests/adapter/dbt_clone/test_dbt_clone.py b/dbt-tests-adapter/dbt/tests/adapter/dbt_clone/test_dbt_clone.py index 61472d1a..f59956c1 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/dbt_clone/test_dbt_clone.py +++ b/dbt-tests-adapter/dbt/tests/adapter/dbt_clone/test_dbt_clone.py @@ -73,17 +73,14 @@ def copy_state(self, project_root): def run_and_save_state(self, project_root, with_snapshot=False): results = run_dbt(["seed"]) assert len(results) == 1 - assert not any(r.node.deferred for r in results) results = run_dbt(["run"]) assert len(results) == 2 - assert not any(r.node.deferred for r in results) results = run_dbt(["test"]) assert len(results) == 2 if with_snapshot: results = run_dbt(["snapshot"]) assert len(results) == 1 - assert not any(r.node.deferred for r in results) # copy files self.copy_state(project_root) @@ -214,6 +211,7 @@ def test_clone_same_target_and_state(self, project, unique_schema, other_schema) clone_args = [ "clone", + "--defer", "--state", "target", ] From b469cd510e0df7f3ce09323ae8f692948450433c Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Mon, 6 May 2024 20:27:57 +0000 Subject: [PATCH 22/66] Bumping version to 1.8.0rc2 and generate changelog --- .changes/1.8.0-rc2.md | 9 +++++++++ .../Features-20240501-091604.yaml | 0 .../{unreleased => 1.8.0}/Fixes-20240502-150038.yaml | 0 CHANGELOG.md | 10 ++++++++++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 5 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 .changes/1.8.0-rc2.md rename .changes/{unreleased => 1.8.0}/Features-20240501-091604.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20240502-150038.yaml (100%) diff --git a/.changes/1.8.0-rc2.md b/.changes/1.8.0-rc2.md new file mode 100644 index 00000000..caa75d3a --- /dev/null +++ b/.changes/1.8.0-rc2.md @@ -0,0 +1,9 @@ +## dbt-adapter 1.8.0-rc2 - May 06, 2024 + +### Features + +* Cross-database `date` macro + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes diff --git a/.changes/unreleased/Features-20240501-091604.yaml b/.changes/1.8.0/Features-20240501-091604.yaml similarity index 100% rename from .changes/unreleased/Features-20240501-091604.yaml rename to .changes/1.8.0/Features-20240501-091604.yaml diff --git a/.changes/unreleased/Fixes-20240502-150038.yaml b/.changes/1.8.0/Fixes-20240502-150038.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240502-150038.yaml rename to .changes/1.8.0/Fixes-20240502-150038.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 168dff4f..05640af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapter 1.8.0-rc2 - May 06, 2024 + +### Features + +* Cross-database `date` macro + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes + ## dbt-adapter 1.8.0-rc1 - May 01, 2024 ## dbt-adapter 1.1.0 - May 01, 2024 diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 037ac1fa..76a9c93f 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0rc1" +version = "1.8.0rc2" From 73e122dfe585a7b53563737759eb2d30563ae474 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 7 May 2024 16:34:12 -0400 Subject: [PATCH 23/66] Adjust __post_serialize__ signature (#197) --- .changes/unreleased/Features-20240506-170313.yaml | 6 ++++++ dbt/adapters/contracts/connection.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20240506-170313.yaml diff --git a/.changes/unreleased/Features-20240506-170313.yaml b/.changes/unreleased/Features-20240506-170313.yaml new file mode 100644 index 00000000..a794f657 --- /dev/null +++ b/.changes/unreleased/Features-20240506-170313.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable serialization contexts +time: 2024-05-06T17:03:13.128814-04:00 +custom: + Author: gshank + Issue: "198" diff --git a/dbt/adapters/contracts/connection.py b/dbt/adapters/contracts/connection.py index 2ff4e5af..e3baf284 100644 --- a/dbt/adapters/contracts/connection.py +++ b/dbt/adapters/contracts/connection.py @@ -170,7 +170,7 @@ def __pre_deserialize__(cls, data): def translate_aliases(cls, kwargs: Dict[str, Any], recurse: bool = False) -> Dict[str, Any]: return translate_aliases(kwargs, cls._ALIASES, recurse) - def __post_serialize__(self, dct): + def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None): # no super() -- do we need it? if self._ALIASES: dct.update( From 994c62d57b195b65f6a76554cc55950d29383d62 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 7 May 2024 15:01:54 -0700 Subject: [PATCH 24/66] Create CODEOWNERS (#200) --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..f6283d12 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# This codeowners file is used to ensure all PRs require reviews from the adapters team + +* @dbt-labs/core-adapters From ca83f204d98245c28b66c239d790e1c177b16cfb Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 7 May 2024 22:43:21 +0000 Subject: [PATCH 25/66] Bumping version to 1.1.1 and generate changelog --- .changes/1.1.1.md | 5 +++++ .changes/unreleased/Features-20240506-170313.yaml | 6 ------ CHANGELOG.md | 6 ++++++ dbt/adapters/__about__.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 .changes/1.1.1.md delete mode 100644 .changes/unreleased/Features-20240506-170313.yaml diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md new file mode 100644 index 00000000..0a28c3ad --- /dev/null +++ b/.changes/1.1.1.md @@ -0,0 +1,5 @@ +## dbt-adapter 1.1.1 - May 07, 2024 + +### Features + +* Enable serialization contexts diff --git a/.changes/unreleased/Features-20240506-170313.yaml b/.changes/unreleased/Features-20240506-170313.yaml deleted file mode 100644 index a794f657..00000000 --- a/.changes/unreleased/Features-20240506-170313.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Enable serialization contexts -time: 2024-05-06T17:03:13.128814-04:00 -custom: - Author: gshank - Issue: "198" diff --git a/CHANGELOG.md b/CHANGELOG.md index 05640af3..d4e3d116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ## dbt-adapter 1.8.0-rc1 - May 01, 2024 +## dbt-adapter 1.1.1 - May 07, 2024 + +### Features + +* Enable serialization contexts + ## dbt-adapter 1.1.0 - May 01, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index b2b60a55..64c8c21b 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.1.0" +version = "1.1.1" From 2a20e81c391e3033b98d8586e23e567dafc19b8a Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Wed, 8 May 2024 07:38:44 -0700 Subject: [PATCH 26/66] Update CODEOWNERS to add dbt-labs/adapters (#201) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f6283d12..02ed72d4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ # This codeowners file is used to ensure all PRs require reviews from the adapters team -* @dbt-labs/core-adapters +* @dbt-labs/adapters From e9ae363ef627103379d5ef79d49cba38b825ab43 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Thu, 9 May 2024 16:38:57 +0000 Subject: [PATCH 27/66] Bumping version to 1.8.0 and generate changelog --- .changes/1.8.0-rc1.md | 1 - .changes/{1.8.0-rc2.md => 1.8.0.md} | 2 +- .changes/1.8.0/Features-20240501-091604.yaml | 6 ------ .changes/1.8.0/Fixes-20240502-150038.yaml | 7 ------- CHANGELOG.md | 4 +--- dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 6 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 .changes/1.8.0-rc1.md rename .changes/{1.8.0-rc2.md => 1.8.0.md} (77%) delete mode 100644 .changes/1.8.0/Features-20240501-091604.yaml delete mode 100644 .changes/1.8.0/Fixes-20240502-150038.yaml diff --git a/.changes/1.8.0-rc1.md b/.changes/1.8.0-rc1.md deleted file mode 100644 index 7467c510..00000000 --- a/.changes/1.8.0-rc1.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapter 1.8.0-rc1 - May 01, 2024 diff --git a/.changes/1.8.0-rc2.md b/.changes/1.8.0.md similarity index 77% rename from .changes/1.8.0-rc2.md rename to .changes/1.8.0.md index caa75d3a..bbe54825 100644 --- a/.changes/1.8.0-rc2.md +++ b/.changes/1.8.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.8.0-rc2 - May 06, 2024 +## dbt-adapter 1.8.0 - May 09, 2024 ### Features diff --git a/.changes/1.8.0/Features-20240501-091604.yaml b/.changes/1.8.0/Features-20240501-091604.yaml deleted file mode 100644 index fb934881..00000000 --- a/.changes/1.8.0/Features-20240501-091604.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Cross-database `date` macro -time: 2024-05-01T09:16:04.909686-06:00 -custom: - Author: dbeatty10 - Issue: https://github.com/dbt-labs/dbt-core/issues/8831 diff --git a/.changes/1.8.0/Fixes-20240502-150038.yaml b/.changes/1.8.0/Fixes-20240502-150038.yaml deleted file mode 100644 index ed80bfc5..00000000 --- a/.changes/1.8.0/Fixes-20240502-150038.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: Update Clone test to reflect core change removing `deferred` attribute from - nodes -time: 2024-05-02T15:00:38.882727-07:00 -custom: - Author: colin-rogers-dbt - Issue: "194" diff --git a/CHANGELOG.md b/CHANGELOG.md index d4e3d116..76b1928b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapter 1.8.0-rc2 - May 06, 2024 +## dbt-adapter 1.8.0 - May 09, 2024 ### Features @@ -15,8 +15,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes -## dbt-adapter 1.8.0-rc1 - May 01, 2024 - ## dbt-adapter 1.1.1 - May 07, 2024 ### Features diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 76a9c93f..6aaa73b8 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0rc2" +version = "1.8.0" From b0b7d29ee8d3a868ba5e9c489cdc03b6d254c3f1 Mon Sep 17 00:00:00 2001 From: Peter Webb Date: Tue, 14 May 2024 17:50:17 -0400 Subject: [PATCH 28/66] Add query recording for adapters which use SQLConnectionManager (#195) Co-authored-by: Emily Rockman --- .../Under the Hood-20240503-162655.yaml | 6 ++ dbt/adapters/record.py | 67 +++++++++++++++++++ dbt/adapters/sql/connections.py | 3 + docs/README.md | 5 ++ docs/guides/record_replay.md | 27 ++++++++ 5 files changed, 108 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240503-162655.yaml create mode 100644 dbt/adapters/record.py create mode 100644 docs/README.md create mode 100644 docs/guides/record_replay.md diff --git a/.changes/unreleased/Under the Hood-20240503-162655.yaml b/.changes/unreleased/Under the Hood-20240503-162655.yaml new file mode 100644 index 00000000..1d2946f0 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240503-162655.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add query recording for adapters which use SQLConnectionManager +time: 2024-05-03T16:26:55.350916-04:00 +custom: + Author: peterallenwebb + Issue: "195" diff --git a/dbt/adapters/record.py b/dbt/adapters/record.py new file mode 100644 index 00000000..5204f59c --- /dev/null +++ b/dbt/adapters/record.py @@ -0,0 +1,67 @@ +import dataclasses +from io import StringIO +import json +import re +from typing import Any, Optional, Mapping + +from agate import Table + +from dbt_common.events.contextvars import get_node_info +from dbt_common.record import Record, Recorder + +from dbt.adapters.contracts.connection import AdapterResponse + + +@dataclasses.dataclass +class QueryRecordParams: + sql: str + auto_begin: bool = False + fetch: bool = False + limit: Optional[int] = None + node_unique_id: Optional[str] = None + + def __post_init__(self) -> None: + if self.node_unique_id is None: + node_info = get_node_info() + self.node_unique_id = node_info["unique_id"] if node_info else "" + + @staticmethod + def _clean_up_sql(sql: str) -> str: + sql = re.sub(r"--.*?\n", "", sql) # Remove single-line comments (--) + sql = re.sub(r"/\*.*?\*/", "", sql, flags=re.DOTALL) # Remove multi-line comments (/* */) + return sql.replace(" ", "").replace("\n", "") + + def _matches(self, other: "QueryRecordParams") -> bool: + return self.node_unique_id == other.node_unique_id and self._clean_up_sql( + self.sql + ) == self._clean_up_sql(other.sql) + + +@dataclasses.dataclass +class QueryRecordResult: + adapter_response: Optional["AdapterResponse"] + table: Optional[Table] + + def _to_dict(self) -> Any: + buf = StringIO() + self.table.to_json(buf) # type: ignore + + return { + "adapter_response": self.adapter_response.to_dict(), # type: ignore + "table": buf.getvalue(), + } + + @classmethod + def _from_dict(cls, dct: Mapping) -> "QueryRecordResult": + return QueryRecordResult( + adapter_response=AdapterResponse.from_dict(dct["adapter_response"]), + table=Table.from_object(json.loads(dct["table"])), + ) + + +class QueryRecord(Record): + params_cls = QueryRecordParams + result_cls = QueryRecordResult + + +Recorder.register_record_type(QueryRecord) diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index 78cd3c9b..9adaafce 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -5,6 +5,7 @@ from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError, NotImplementedError +from dbt_common.record import record_function from dbt_common.utils import cast_to_str from dbt.adapters.base import BaseConnectionManager @@ -19,6 +20,7 @@ SQLQuery, SQLQueryStatus, ) +from dbt.adapters.record import QueryRecord if TYPE_CHECKING: import agate @@ -143,6 +145,7 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Tab return table_from_data_flat(data, column_names) + @record_function(QueryRecord, method=True, tuple_result=True) def execute( self, sql: str, diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..a1096d54 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,5 @@ +## Documentation + +The documentation is divided into the following sub-folders: +* arch: Architecture Decision Records (ADRs) which explain and justify major architectural decisions +* guides: Informal documents which describe the code or our development practices at a high level diff --git a/docs/guides/record_replay.md b/docs/guides/record_replay.md new file mode 100644 index 00000000..670bb843 --- /dev/null +++ b/docs/guides/record_replay.md @@ -0,0 +1,27 @@ +# Supporting Record/Replay in Adapters + +This document describes how to implement support for dbt's Record/Replay Subsystem for adapters. Before reading it, make sure you understand the fundamental ideas behind Record/Replay, which are [documented in the dbt-common repo](https://github.com/dbt-labs/dbt-common/blob/docs/guides/record_replay.md). + +## Recording and Replaying Warehouse Interaction + +The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most obvious. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. (It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, and so forth.) + +In practice, this means that any request sent to the warehouse must be recorded, along with the corresponding response. If this is done correctly, as described in the document linked in the intro, the Record portion of the Record/Replay subsystem should work as expected. + +At the time of this writing, there is only an incomplete implementation of this goal, which can be found in `dbt-adapters/dbt/adapters/record.py`. + +There are some important things to notice about this implementation. First, the QueryRecordResult class provides custom serialization methods `to_dict()` and `from_dict()`. This is necessary because the `AdapterResponse` and `Agate.Table` types cannot be automatically converted to and from JSON by the dataclass library, and JSON is the format used to persist recordings to disk and reload them for replay. + +Another important feature is that `QueryRecordParams` implements the `_matches()` method. This method allows `dbt-adapters` to customize the way that the Record/Replay determines whether a query issued by dbt matches a previously recorded query. In this case, the method performs a comparison which attempts to ignore comments and whitespace which would not affect query behavior. + +## Misc. Notes and Suggestions + +Currently, support for recording data warehouse interaction is very rudimentary, however, even rudimentary support is valuable and we should be concentrating on extending it in a way that adds the most value with the least work. Usefulness, rather than perfection, is the initial goal. + +Picking the right functions to record, at the right level of abstraction, will probably be the most important part of carrying this work forward. + +Not every interaction with an external system has to be recorded in full detail, and authentication might prove to be a place where we exclude sensitive secrets from the recording. For example, since replay will not actually be communicating with the warehouse, it may be possible to exclude passwords and auth keys from the parameters recorded, and to exclude auth tokens from the results. + +In addition to adding an appropriate decorator to functions which communicate with external systems, you should check those functions for side-effects. Since the function's calls will be mocked out in replay mode, those side-effects will not be carried out during replay. At present, we are focusing on support for recording and comparing recordings, but this is worth keeping in mind. + +The current implementation records which dbt node issues a query, and uses that information to ensure a match during replay. The same node should issue the same query. A better model might be to monitor which connection issued which query, and associate the same connection with open/close operations, transaction starts/stops and so forth. From 2e7d04c266f2d4f2f2e73b61244ea78221c5668c Mon Sep 17 00:00:00 2001 From: Isaac Blanco <40709873+AlejandroBlanco2001@users.noreply.github.com> Date: Tue, 14 May 2024 22:28:52 -0500 Subject: [PATCH 29/66] =?UTF-8?q?Improving=20message=20description=20for?= =?UTF-8?q?=20compile=20error=20in=20get=5Ffixtu=E2=80=A6=20(#203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changes/unreleased/Features-20240511-032253.yaml | 7 +++++++ .../macros/unit_test_sql/get_fixture_sql.sql | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20240511-032253.yaml diff --git a/.changes/unreleased/Features-20240511-032253.yaml b/.changes/unreleased/Features-20240511-032253.yaml new file mode 100644 index 00000000..b81ff9c0 --- /dev/null +++ b/.changes/unreleased/Features-20240511-032253.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Improvement of the compile error message in the get_fixture-sql.sql when the + relation or the model not exist +time: 2024-05-11T03:22:53.40817477-05:00 +custom: + Author: AlejandroBlanco2001 + Issue: "10014" diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index 53d7a93b..c25a87f7 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -15,7 +15,7 @@ {%- endif -%} {%- if not column_name_to_data_types -%} - {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this) }} + {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this ~ " because the relation doesn't exist") }} {%- endif -%} {%- for column_name, column_type in column_name_to_data_types.items() -%} From eda109542efd1703fd08e242006ae4d3fc476b38 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 15 May 2024 16:15:42 -0400 Subject: [PATCH 30/66] Add docstring for "catch all" relation type of `External` (#215) --- dbt/adapters/contracts/relation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/contracts/relation.py b/dbt/adapters/contracts/relation.py index 3560c2b3..42beb579 100644 --- a/dbt/adapters/contracts/relation.py +++ b/dbt/adapters/contracts/relation.py @@ -18,8 +18,9 @@ class RelationType(StrEnum): View = "view" CTE = "cte" MaterializedView = "materialized_view" - External = "external" Ephemeral = "ephemeral" + # this is a "catch all" that is better than `None` == external to anything dbt is aware of + External = "external" class MaterializationContract(Protocol): From fd33aafe8276051b313a3c89557b50d224bc50ed Mon Sep 17 00:00:00 2001 From: Peter Webb Date: Thu, 16 May 2024 17:27:08 -0400 Subject: [PATCH 31/66] Improve memory efficiency of process_results by iterating. (#217) --- .../Under the Hood-20240516-105757.yaml | 6 ++++++ dbt/adapters/sql/connections.py | 15 ++++++--------- tests/unit/test_sql_result.py | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240516-105757.yaml diff --git a/.changes/unreleased/Under the Hood-20240516-105757.yaml b/.changes/unreleased/Under the Hood-20240516-105757.yaml new file mode 100644 index 00000000..a5a47c8c --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240516-105757.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Improve memory efficiency of process_results() +time: 2024-05-16T10:57:57.480672-04:00 +custom: + Author: peterallenwebb + Issue: "217" diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index 9adaafce..d8699fd3 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -1,6 +1,6 @@ import abc import time -from typing import Any, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, TYPE_CHECKING from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event @@ -112,27 +112,24 @@ def get_response(cls, cursor: Any) -> AdapterResponse: @classmethod def process_results( cls, column_names: Iterable[str], rows: Iterable[Any] - ) -> List[Dict[str, Any]]: - # TODO CT-211 + ) -> Iterator[Dict[str, Any]]: unique_col_names = dict() # type: ignore[var-annotated] - # TODO CT-211 for idx in range(len(column_names)): # type: ignore[arg-type] - # TODO CT-211 col_name = column_names[idx] # type: ignore[index] if col_name in unique_col_names: unique_col_names[col_name] += 1 - # TODO CT-211 column_names[idx] = f"{col_name}_{unique_col_names[col_name]}" # type: ignore[index] # noqa else: - # TODO CT-211 unique_col_names[column_names[idx]] = 1 # type: ignore[index] - return [dict(zip(column_names, row)) for row in rows] + + for row in rows: + yield dict(zip(column_names, row)) @classmethod def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Table": from dbt_common.clients.agate_helper import table_from_data_flat - data: List[Any] = [] + data: Iterable[Any] = [] column_names: List[str] = [] if cursor.description is not None: diff --git a/tests/unit/test_sql_result.py b/tests/unit/test_sql_result.py index 12c173cb..454e6572 100644 --- a/tests/unit/test_sql_result.py +++ b/tests/unit/test_sql_result.py @@ -8,13 +8,13 @@ def test_duplicated_columns(self): cols_with_one_dupe = ["a", "b", "a", "d"] rows = [(1, 2, 3, 4)] self.assertEqual( - SQLConnectionManager.process_results(cols_with_one_dupe, rows), + list(SQLConnectionManager.process_results(cols_with_one_dupe, rows)), [{"a": 1, "b": 2, "a_2": 3, "d": 4}], ) cols_with_more_dupes = ["a", "a", "a", "b"] rows = [(1, 2, 3, 4)] self.assertEqual( - SQLConnectionManager.process_results(cols_with_more_dupes, rows), + list(SQLConnectionManager.process_results(cols_with_more_dupes, rows)), [{"a": 1, "a_2": 2, "a_3": 3, "b": 4}], ) From 95e5c77e8a27ac964599cbeeddd0c58b13fbf390 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 21 May 2024 22:42:11 +0000 Subject: [PATCH 32/66] Bumping version to 1.2.1 and generate changelog --- .changes/1.2.1.md | 10 ++++++++++ .changes/unreleased/Features-20240511-032253.yaml | 7 ------- .../unreleased/Under the Hood-20240503-162655.yaml | 6 ------ .../unreleased/Under the Hood-20240516-105757.yaml | 6 ------ CHANGELOG.md | 11 +++++++++++ dbt/adapters/__about__.py | 2 +- 6 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 .changes/1.2.1.md delete mode 100644 .changes/unreleased/Features-20240511-032253.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240503-162655.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240516-105757.yaml diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md new file mode 100644 index 00000000..f838b769 --- /dev/null +++ b/.changes/1.2.1.md @@ -0,0 +1,10 @@ +## dbt-adapter 1.2.1 - May 21, 2024 + +### Features + +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist + +### Under the Hood + +* Add query recording for adapters which use SQLConnectionManager +* Improve memory efficiency of process_results() diff --git a/.changes/unreleased/Features-20240511-032253.yaml b/.changes/unreleased/Features-20240511-032253.yaml deleted file mode 100644 index b81ff9c0..00000000 --- a/.changes/unreleased/Features-20240511-032253.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: Improvement of the compile error message in the get_fixture-sql.sql when the - relation or the model not exist -time: 2024-05-11T03:22:53.40817477-05:00 -custom: - Author: AlejandroBlanco2001 - Issue: "10014" diff --git a/.changes/unreleased/Under the Hood-20240503-162655.yaml b/.changes/unreleased/Under the Hood-20240503-162655.yaml deleted file mode 100644 index 1d2946f0..00000000 --- a/.changes/unreleased/Under the Hood-20240503-162655.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add query recording for adapters which use SQLConnectionManager -time: 2024-05-03T16:26:55.350916-04:00 -custom: - Author: peterallenwebb - Issue: "195" diff --git a/.changes/unreleased/Under the Hood-20240516-105757.yaml b/.changes/unreleased/Under the Hood-20240516-105757.yaml deleted file mode 100644 index a5a47c8c..00000000 --- a/.changes/unreleased/Under the Hood-20240516-105757.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Improve memory efficiency of process_results() -time: 2024-05-16T10:57:57.480672-04:00 -custom: - Author: peterallenwebb - Issue: "217" diff --git a/CHANGELOG.md b/CHANGELOG.md index 76b1928b..aaa477cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,17 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapter 1.2.1 - May 21, 2024 + +### Features + +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist + +### Under the Hood + +* Add query recording for adapters which use SQLConnectionManager +* Improve memory efficiency of process_results() + ## dbt-adapter 1.1.1 - May 07, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 64c8c21b..eb1d9a0f 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.1.1" +version = "1.2.1" From 8ff69435a753f0a94dd1274d0e8dad199805cd13 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:50:44 -0600 Subject: [PATCH 33/66] Update `dbt-adapter` to `dbt-adapters` (#235) --- .changes/1.0.0.md | 2 +- .changes/1.1.0.md | 2 +- .changes/1.1.1.md | 2 +- .changes/1.2.1.md | 2 +- .changes/1.8.0.md | 2 +- .changie.yaml | 2 +- .github/ISSUE_TEMPLATE/bug-report.yml | 6 +++--- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- .github/ISSUE_TEMPLATE/feature-request.yml | 10 +++++----- .github/ISSUE_TEMPLATE/regression-report.yml | 10 +++++----- .github/pull_request_template.md | 2 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 10 +++++----- CONTRIBUTING.md | 2 +- 14 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.changes/1.0.0.md b/.changes/1.0.0.md index b6cc44a9..d381d635 100644 --- a/.changes/1.0.0.md +++ b/.changes/1.0.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.0.0 - April 01, 2024 +## dbt-adapters 1.0.0 - April 01, 2024 ### Fixes diff --git a/.changes/1.1.0.md b/.changes/1.1.0.md index c43ef9aa..9e7db78e 100644 --- a/.changes/1.1.0.md +++ b/.changes/1.1.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.1.0 - May 01, 2024 +## dbt-adapters 1.1.0 - May 01, 2024 ### Features diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md index 0a28c3ad..a1c38c88 100644 --- a/.changes/1.1.1.md +++ b/.changes/1.1.1.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.1.1 - May 07, 2024 +## dbt-adapters 1.1.1 - May 07, 2024 ### Features diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md index f838b769..2aac5674 100644 --- a/.changes/1.2.1.md +++ b/.changes/1.2.1.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.2.1 - May 21, 2024 +## dbt-adapters 1.2.1 - May 21, 2024 ### Features diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md index bbe54825..e781a985 100644 --- a/.changes/1.8.0.md +++ b/.changes/1.8.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.8.0 - May 09, 2024 +## dbt-adapters 1.8.0 - May 09, 2024 ### Features diff --git a/.changie.yaml b/.changie.yaml index 9f78b81e..afbafb22 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -4,7 +4,7 @@ headerPath: header.tpl.md changelogPath: CHANGELOG.md versionExt: md envPrefix: CHANGIE_ -versionFormat: '## dbt-adapter {{.Version}} - {{.Time.Format "January 02, 2006"}}' +versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' changeFormat: '* {{.Body}}' kinds: diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 3bd90bf7..27cb521e 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,5 +1,5 @@ name: 🐞 Bug -description: Report a bug or an issue you've found with dbt-adapter +description: Report a bug or an issue you've found with dbt-adapters title: "[Bug] " labels: ["bug", "triage"] body: @@ -62,11 +62,11 @@ body: examples: - **OS**: Ubuntu 20.04 - **Python**: 3.11.6 (`python3 --version`) - - **dbt-adapter**: 1.0.0 + - **dbt-adapters**: 1.0.0 value: | - OS: - Python: - - dbt-adapter: + - dbt-adapters: render: markdown validations: required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 2e23e0fd..a89889af 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -7,5 +7,5 @@ contact_links: url: mailto:support@getdbt.com about: Are you using dbt Cloud? Contact our support team for help! - name: Participate in Discussions - url: https://github.com/dbt-labs/dbt-adapter/discussions - about: Do you have a Big Idea for dbt-adapter? Read open discussions, or start a new one + url: https://github.com/dbt-labs/dbt-adapters/discussions + about: Do you have a Big Idea for dbt-adapters? Read open discussions, or start a new one diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 25b28aae..22960c2d 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -1,5 +1,5 @@ name: ✨ Feature -description: Propose a straightforward extension of dbt-adapter functionality +description: Propose a straightforward extension of dbt-adapters functionality title: "[Feature] <title>" labels: ["enhancement", "triage"] body: @@ -14,15 +14,15 @@ body: We want to make sure that features are distinct and discoverable, so that other members of the community can find them and offer their thoughts. - Issues are the right place to request straightforward extensions of existing dbt-adapter functionality. - For "big ideas" about future capabilities of dbt-adapter, we ask that you open a - [discussion](https://github.com/dbt-labs/dbt-adapter/discussions/new?category=ideas) in the "Ideas" category instead. + Issues are the right place to request straightforward extensions of existing dbt-adapters functionality. + For "big ideas" about future capabilities of dbt-adapters, we ask that you open a + [discussion](https://github.com/dbt-labs/dbt-adapters/discussions/new?category=ideas) in the "Ideas" category instead. options: - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) required: true - label: I have searched the existing issues, and I could not find an existing issue for this feature required: true - - label: I am requesting a straightforward extension of existing dbt-adapter functionality, rather than a Big Idea better suited to a discussion + - label: I am requesting a straightforward extension of existing dbt-adapters functionality, rather than a Big Idea better suited to a discussion required: true - type: textarea attributes: diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml index 01775507..6831ede2 100644 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ b/.github/ISSUE_TEMPLATE/regression-report.yml @@ -1,5 +1,5 @@ name: ☣️ Regression -description: Report a regression you've observed in a newer version of dbt-adapter +description: Report a regression you've observed in a newer version of dbt-adapters title: "[Regression] <title>" labels: ["regression", "triage"] body: @@ -57,13 +57,13 @@ body: examples: - **OS**: Ubuntu 20.04 - **Python**: 3.11.6 (`python3 --version`) - - **dbt-adapter (working version)**: 1.1.0 - - **dbt-adapter (regression version)**: 1.2.0 + - **dbt-adapters (working version)**: 1.1.0 + - **dbt-adapters (regression version)**: 1.2.0 value: | - OS: - Python: - - dbt-adapter (working version): - - dbt-adapter (regression version): + - dbt-adapters (working version): + - dbt-adapters (regression version): render: markdown validations: required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4fc2fcf8..3879b653 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -29,7 +29,7 @@ resolves # ### Checklist -- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-adapter/blob/main/CONTRIBUTING.md) and understand what's expected of me +- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md) and understand what's expected of me - [ ] I have run this code in development, and it appears to resolve the stated issue - [ ] This PR includes tests, or tests are not required/relevant for this PR - [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1135adb8..828350dd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -151,7 +151,7 @@ jobs: github-release: name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapters assets to the same release if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} needs: [release-inputs, build-and-test, bump-version-generate-changelog] uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa477cf..43c3937e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapter 1.8.0 - May 09, 2024 +## dbt-adapters 1.8.0 - May 09, 2024 ### Features @@ -15,7 +15,7 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes -## dbt-adapter 1.2.1 - May 21, 2024 +## dbt-adapters 1.2.1 - May 21, 2024 ### Features @@ -26,13 +26,13 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Add query recording for adapters which use SQLConnectionManager * Improve memory efficiency of process_results() -## dbt-adapter 1.1.1 - May 07, 2024 +## dbt-adapters 1.1.1 - May 07, 2024 ### Features * Enable serialization contexts -## dbt-adapter 1.1.0 - May 01, 2024 +## dbt-adapters 1.1.0 - May 01, 2024 ### Features @@ -62,7 +62,7 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * add support for py3.12 -## dbt-adapter 1.0.0 - April 01, 2024 +## dbt-adapters 1.0.0 - April 01, 2024 ### Fixes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c87ea23d..28f13f6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -163,7 +163,7 @@ Remember to commit and push the file that's created. ### Signing the CLA -> **_NOTE:_** All contributors to `dbt-adapter` must sign the +> **_NOTE:_** All contributors to `dbt-adapters` must sign the > [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA). Maintainers will be unable to merge contributions until the contributor signs the CLA. From 6687ccc7cdf7b16ab5dbaab57d8aae5f9905b21a Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Tue, 18 Jun 2024 21:27:35 +0100 Subject: [PATCH 34/66] Add `get_catalog_for_single_relation` macro and capability (#231) --- .../unreleased/Features-20240528-013623.yaml | 6 ++ .../test_get_catalog_for_single_relation.py | 87 +++++++++++++++++++ dbt/adapters/base/impl.py | 15 +++- dbt/adapters/capability.py | 7 +- dbt/adapters/sql/impl.py | 9 +- .../macros/adapters/metadata.sql | 9 ++ pyproject.toml | 5 +- 7 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Features-20240528-013623.yaml create mode 100644 dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py diff --git a/.changes/unreleased/Features-20240528-013623.yaml b/.changes/unreleased/Features-20240528-013623.yaml new file mode 100644 index 00000000..2c01e972 --- /dev/null +++ b/.changes/unreleased/Features-20240528-013623.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation +time: 2024-05-28T01:36:23.588295+01:00 +custom: + Author: aranke + Issue: "231" diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py b/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py new file mode 100644 index 00000000..78055cc5 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py @@ -0,0 +1,87 @@ +import pytest + +from dbt.tests.util import run_dbt, get_connection + +models__my_table_model_sql = """ +select * from {{ ref('my_seed') }} +""" + + +models__my_view_model_sql = """ +{{ + config( + materialized='view', + ) +}} + +select * from {{ ref('my_seed') }} +""" + +seed__my_seed_csv = """id,first_name,email,ip_address,updated_at +1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 +""" + + +class BaseGetCatalogForSingleRelation: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "get_catalog_for_single_relation"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "my_seed.csv": seed__my_seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_view_model.sql": models__my_view_model_sql, + "my_table_model.sql": models__my_table_model_sql, + } + + @pytest.fixture(scope="class") + def expected_catalog_my_seed(self, project): + raise NotImplementedError( + "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`." + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_model(self, project): + raise NotImplementedError( + "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`." + ) + + def get_relation_for_identifier(self, project, identifier): + return project.adapter.get_relation( + database=project.database, + schema=project.test_schema, + identifier=identifier, + ) + + def test_get_catalog_for_single_relation( + self, project, expected_catalog_my_seed, expected_catalog_my_view_model + ): + results = run_dbt(["seed"]) + assert len(results) == 1 + + my_seed_relation = self.get_relation_for_identifier(project, "my_seed") + + with get_connection(project.adapter): + actual_catalog_my_seed = project.adapter.get_catalog_for_single_relation( + my_seed_relation + ) + + assert actual_catalog_my_seed == expected_catalog_my_seed + + results = run_dbt(["run"]) + assert len(results) == 2 + + my_view_model_relation = self.get_relation_for_identifier(project, "my_view_model") + + with get_connection(project.adapter): + actual_catalog_my_view_model = project.adapter.get_catalog_for_single_relation( + my_view_model_relation + ) + + assert actual_catalog_my_view_model == expected_catalog_my_view_model diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index f58f8aba..1fdbce48 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1,10 +1,10 @@ import abc +import time from concurrent.futures import as_completed, Future from contextlib import contextmanager from datetime import datetime from enum import Enum from multiprocessing.context import SpawnContext -import time from typing import ( Any, Callable, @@ -23,12 +23,15 @@ TYPE_CHECKING, ) +import pytz from dbt_common.clients.jinja import CallableMacroGenerator from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, ) +from dbt_common.contracts.metadata import CatalogTable +from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.exceptions import ( DbtInternalError, DbtRuntimeError, @@ -38,14 +41,12 @@ NotImplementedError, UnexpectedNullError, ) -from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.utils import ( AttrDict, cast_to_str, executor, filter_null_values, ) -import pytz from dbt.adapters.base.column import Column as BaseColumn from dbt.adapters.base.connections import ( @@ -222,6 +223,7 @@ class BaseAdapter(metaclass=AdapterMeta): - truncate_relation - rename_relation - get_columns_in_relation + - get_catalog_for_single_relation - get_column_schema_from_query - expand_column_types - list_relations_without_caching @@ -627,6 +629,13 @@ def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" raise NotImplementedError("`get_columns_in_relation` is not implemented for this adapter!") + @abc.abstractmethod + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + """Get catalog information including table-level and column-level metadata for a single relation.""" + raise NotImplementedError( + "`get_catalog_for_single_relation` is not implemented for this adapter!" + ) + @available.deprecated("get_columns_in_relation", lambda *a, **k: []) def get_columns_in_table(self, schema: str, identifier: str) -> List[BaseColumn]: """DEPRECATED: Get a list of the columns in the given table.""" diff --git a/dbt/adapters/capability.py b/dbt/adapters/capability.py index 305604c7..2bd49112 100644 --- a/dbt/adapters/capability.py +++ b/dbt/adapters/capability.py @@ -14,7 +14,12 @@ class Capability(str, Enum): """Indicates support for determining the time of the last table modification by querying database metadata.""" TableLastModifiedMetadataBatch = "TableLastModifiedMetadataBatch" - """Indicates support for performantly determining the time of the last table modification by querying database metadata in batch.""" + """Indicates support for performantly determining the time of the last table modification by querying database + metadata in batch.""" + + GetCatalogForSingleRelation = "GetCatalogForSingleRelation" + """Indicates support for getting catalog information including table-level and column-level metadata for a single + relation.""" class Support(str, Enum): diff --git a/dbt/adapters/sql/impl.py b/dbt/adapters/sql/impl.py index 8c6e0e8e..91ca3ebb 100644 --- a/dbt/adapters/sql/impl.py +++ b/dbt/adapters/sql/impl.py @@ -1,5 +1,6 @@ from typing import Any, List, Optional, Tuple, Type, TYPE_CHECKING +from dbt_common.contracts.metadata import CatalogTable from dbt_common.events.functions import fire_event from dbt.adapters.base import BaseAdapter, BaseRelation, available @@ -9,9 +10,9 @@ from dbt.adapters.exceptions import RelationTypeNullError from dbt.adapters.sql.connections import SQLConnectionManager - LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" +GET_CATALOG_FOR_SINGLE_RELATION_NAME = "get_catalog_for_single_relation" LIST_SCHEMAS_MACRO_NAME = "list_schemas" CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists" CREATE_SCHEMA_MACRO_NAME = "create_schema" @@ -41,6 +42,7 @@ class SQLAdapter(BaseAdapter): - get_catalog - list_relations_without_caching - get_columns_in_relation + - get_catalog_for_single_relation """ ConnectionManager: Type[SQLConnectionManager] @@ -158,6 +160,11 @@ def get_columns_in_relation(self, relation): GET_COLUMNS_IN_RELATION_MACRO_NAME, kwargs={"relation": relation} ) + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + return self.execute_macro( + GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs={"relation": relation} + ) + def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) diff --git a/dbt/include/global_project/macros/adapters/metadata.sql b/dbt/include/global_project/macros/adapters/metadata.sql index c8e8a414..0aa7aabb 100644 --- a/dbt/include/global_project/macros/adapters/metadata.sql +++ b/dbt/include/global_project/macros/adapters/metadata.sql @@ -77,6 +77,15 @@ 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} {% endmacro %} +{% macro get_catalog_for_single_relation(relation) %} + {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__get_catalog_for_single_relation(relation) %} + {{ exceptions.raise_not_implemented( + 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + {% macro get_relations() %} {{ return(adapter.dispatch('get_relations', 'dbt')()) }} {% endmacro %} diff --git a/pyproject.toml b/pyproject.toml index a4b011a8..e50aa63a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common<2.0", + "dbt-common>=1.3,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", @@ -43,6 +43,9 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] path = "dbt/adapters/__about__.py" From 9f504d3fc7a822ebeab7da1000ad78b9a7efc8b3 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 18 Jun 2024 22:01:45 +0000 Subject: [PATCH 35/66] Bumping version to 1.3.0 and generate changelog --- .changes/1.3.0.md | 5 +++++ .changes/unreleased/Features-20240528-013623.yaml | 6 ------ CHANGELOG.md | 6 ++++++ dbt/adapters/__about__.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 .changes/1.3.0.md delete mode 100644 .changes/unreleased/Features-20240528-013623.yaml diff --git a/.changes/1.3.0.md b/.changes/1.3.0.md new file mode 100644 index 00000000..dae1f819 --- /dev/null +++ b/.changes/1.3.0.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.3.0 - June 18, 2024 + +### Features + +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation diff --git a/.changes/unreleased/Features-20240528-013623.yaml b/.changes/unreleased/Features-20240528-013623.yaml deleted file mode 100644 index 2c01e972..00000000 --- a/.changes/unreleased/Features-20240528-013623.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation -time: 2024-05-28T01:36:23.588295+01:00 -custom: - Author: aranke - Issue: "231" diff --git a/CHANGELOG.md b/CHANGELOG.md index 43c3937e..5f7560f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.0 - June 18, 2024 + +### Features + +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation + ## dbt-adapters 1.2.1 - May 21, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index eb1d9a0f..d28b3ddc 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.2.1" +version = "1.3.0" From 3460543a155eeb4ccb749fe185d65c8f579dbd04 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 18 Jun 2024 22:58:01 +0000 Subject: [PATCH 36/66] Bumping version to 1.9.0 and generate changelog --- .changes/1.9.0.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.9.0.md diff --git a/.changes/1.9.0.md b/.changes/1.9.0.md new file mode 100644 index 00000000..2f016563 --- /dev/null +++ b/.changes/1.9.0.md @@ -0,0 +1 @@ +## dbt-adapters 1.9.0 - June 18, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f7560f7..016c13ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.9.0 - June 18, 2024 + ## dbt-adapters 1.8.0 - May 09, 2024 ### Features diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 6aaa73b8..7aba6409 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0" +version = "1.9.0" From 46e470cbbe8cefc98b00a2f0cd7223c43c2be4d1 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Wed, 19 Jun 2024 16:50:08 +0100 Subject: [PATCH 37/66] Make `get_catalog_for_single_relation` a concrete method in `BaseAdapter` (#240) --- dbt/adapters/base/impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 1fdbce48..3b36022f 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -629,7 +629,6 @@ def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" raise NotImplementedError("`get_columns_in_relation` is not implemented for this adapter!") - @abc.abstractmethod def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: """Get catalog information including table-level and column-level metadata for a single relation.""" raise NotImplementedError( From ecf3e1d52bc31a4814f25fd67bc186bbb4ed132b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Thu, 20 Jun 2024 17:41:48 +0100 Subject: [PATCH 38/66] Remove get_catalog_for_single_relation from sql/impl.py (#241) --- dbt/adapters/sql/impl.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dbt/adapters/sql/impl.py b/dbt/adapters/sql/impl.py index 91ca3ebb..8a8473f2 100644 --- a/dbt/adapters/sql/impl.py +++ b/dbt/adapters/sql/impl.py @@ -1,6 +1,5 @@ from typing import Any, List, Optional, Tuple, Type, TYPE_CHECKING -from dbt_common.contracts.metadata import CatalogTable from dbt_common.events.functions import fire_event from dbt.adapters.base import BaseAdapter, BaseRelation, available @@ -12,7 +11,6 @@ LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" -GET_CATALOG_FOR_SINGLE_RELATION_NAME = "get_catalog_for_single_relation" LIST_SCHEMAS_MACRO_NAME = "list_schemas" CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists" CREATE_SCHEMA_MACRO_NAME = "create_schema" @@ -160,11 +158,6 @@ def get_columns_in_relation(self, relation): GET_COLUMNS_IN_RELATION_MACRO_NAME, kwargs={"relation": relation} ) - def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: - return self.execute_macro( - GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs={"relation": relation} - ) - def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) From 10f197a09ebca9404e79710b49ec04ac71d23b99 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Thu, 20 Jun 2024 19:17:15 +0000 Subject: [PATCH 39/66] Bumping version to 1.3.1 and generate changelog --- .changes/1.3.1.md | 1 + CHANGELOG.md | 2 ++ dbt/adapters/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.3.1.md diff --git a/.changes/1.3.1.md b/.changes/1.3.1.md new file mode 100644 index 00000000..b8ec7374 --- /dev/null +++ b/.changes/1.3.1.md @@ -0,0 +1 @@ +## dbt-adapters 1.3.1 - June 20, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 016c13ee..b06ec9f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.1 - June 20, 2024 + ## dbt-adapters 1.3.0 - June 18, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index d28b3ddc..35ec9a37 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.0" +version = "1.3.1" From f4880e57ae59215d6f4beedcaee290b5ffe22ae3 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Thu, 20 Jun 2024 19:27:26 +0000 Subject: [PATCH 40/66] Bumping version to 1.9.1 and generate changelog --- .changes/1.9.1.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.9.1.md diff --git a/.changes/1.9.1.md b/.changes/1.9.1.md new file mode 100644 index 00000000..900e6b75 --- /dev/null +++ b/.changes/1.9.1.md @@ -0,0 +1 @@ +## dbt-adapters 1.9.1 - June 20, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index b06ec9f2..9114bcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.9.1 - June 20, 2024 + ## dbt-adapters 1.9.0 - June 18, 2024 ## dbt-adapters 1.8.0 - May 09, 2024 diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 7aba6409..70227976 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.9.0" +version = "1.9.1" From 267cf5e31a132be6bf994c95ccce1d628043be64 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:44:33 -0500 Subject: [PATCH 41/66] update user docs-issue workflow (#238) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/docs-issue.yml | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/docs-issue.yml diff --git a/.github/workflows/docs-issue.yml b/.github/workflows/docs-issue.yml new file mode 100644 index 00000000..f49cf517 --- /dev/null +++ b/.github/workflows/docs-issue.yml @@ -0,0 +1,41 @@ +# **what?** +# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed + +# **why?** +# To reduce barriers for keeping docs up to date + +# **when?** +# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. + + +name: Open issues in docs.getdbt.com repo when an issue is labeled +run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" + +on: + issues: + types: [labeled, closed] + +defaults: + run: + shell: bash + +permissions: + issues: write # comments on issues + +jobs: + open_issues: + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs')) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit From e964302121f70bacc8f827683c329e06696216cf Mon Sep 17 00:00:00 2001 From: Peter Webb <peterallenwebb@gmail.com> Date: Fri, 21 Jun 2024 15:21:15 -0400 Subject: [PATCH 42/66] Fix Query Timer Resolution (#246) --- .changes/unreleased/Under the Hood-20240621-150837.yaml | 6 ++++++ dbt/adapters/events/types.py | 2 +- dbt/adapters/sql/connections.py | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240621-150837.yaml diff --git a/.changes/unreleased/Under the Hood-20240621-150837.yaml b/.changes/unreleased/Under the Hood-20240621-150837.yaml new file mode 100644 index 00000000..fab070c7 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240621-150837.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Fix query timer resolution +time: 2024-06-21T15:08:37.43527-04:00 +custom: + Author: peterallenwebb + Issue: "246" diff --git a/dbt/adapters/events/types.py b/dbt/adapters/events/types.py index 7d98269d..47c48da6 100644 --- a/dbt/adapters/events/types.py +++ b/dbt/adapters/events/types.py @@ -190,7 +190,7 @@ def code(self) -> str: return "E017" def message(self) -> str: - return f"SQL status: {self.status} in {self.elapsed} seconds" + return f"SQL status: {self.status} in {self.elapsed:.3f} seconds" class SQLCommit(DebugLevel): diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index d8699fd3..13111a6e 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -88,7 +88,8 @@ def add_query( node_info=get_node_info(), ) ) - pre = time.time() + + pre = time.perf_counter() cursor = connection.handle.cursor() cursor.execute(sql, bindings) @@ -96,7 +97,7 @@ def add_query( fire_event( SQLQueryStatus( status=str(self.get_response(cursor)), - elapsed=round((time.time() - pre)), + elapsed=time.perf_counter() - pre, node_info=get_node_info(), ) ) From 0739850873de0af14bc5506accb597256f0cdad1 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Wed, 26 Jun 2024 23:47:14 +0100 Subject: [PATCH 43/66] Add optional `should_release_connection` parameter to `connection_named` method (#247) --- .changes/unreleased/Under the Hood-20240624-161108.yaml | 6 ++++++ dbt/adapters/base/impl.py | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240624-161108.yaml diff --git a/.changes/unreleased/Under the Hood-20240624-161108.yaml b/.changes/unreleased/Under the Hood-20240624-161108.yaml new file mode 100644 index 00000000..9042547f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240624-161108.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add optional release_connection parameter to connection_named method +time: 2024-06-24T16:11:08.773419+01:00 +custom: + Author: aranke + Issue: "247" diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 3b36022f..ec590137 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -319,14 +319,18 @@ def nice_connection_name(self) -> str: return conn.name @contextmanager - def connection_named(self, name: str, query_header_context: Any = None) -> Iterator[None]: + def connection_named( + self, name: str, query_header_context: Any = None, should_release_connection=True + ) -> Iterator[None]: try: if self.connections.query_header is not None: self.connections.query_header.set(name, query_header_context) self.acquire_connection(name) yield finally: - self.release_connection() + if should_release_connection: + self.release_connection() + if self.connections.query_header is not None: self.connections.query_header.reset() From d6f736956ef0948cb3fa61d2f5a00d4aa9a6a2aa Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 2 Jul 2024 15:45:03 +0000 Subject: [PATCH 44/66] Bumping version to 1.3.2 and generate changelog --- .changes/1.3.2.md | 6 ++++++ .changes/unreleased/Under the Hood-20240621-150837.yaml | 6 ------ .changes/unreleased/Under the Hood-20240624-161108.yaml | 6 ------ CHANGELOG.md | 7 +++++++ dbt/adapters/__about__.py | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 .changes/1.3.2.md delete mode 100644 .changes/unreleased/Under the Hood-20240621-150837.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240624-161108.yaml diff --git a/.changes/1.3.2.md b/.changes/1.3.2.md new file mode 100644 index 00000000..bdc7f593 --- /dev/null +++ b/.changes/1.3.2.md @@ -0,0 +1,6 @@ +## dbt-adapters 1.3.2 - July 02, 2024 + +### Under the Hood + +* Fix query timer resolution +* Add optional release_connection parameter to connection_named method diff --git a/.changes/unreleased/Under the Hood-20240621-150837.yaml b/.changes/unreleased/Under the Hood-20240621-150837.yaml deleted file mode 100644 index fab070c7..00000000 --- a/.changes/unreleased/Under the Hood-20240621-150837.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Fix query timer resolution -time: 2024-06-21T15:08:37.43527-04:00 -custom: - Author: peterallenwebb - Issue: "246" diff --git a/.changes/unreleased/Under the Hood-20240624-161108.yaml b/.changes/unreleased/Under the Hood-20240624-161108.yaml deleted file mode 100644 index 9042547f..00000000 --- a/.changes/unreleased/Under the Hood-20240624-161108.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add optional release_connection parameter to connection_named method -time: 2024-06-24T16:11:08.773419+01:00 -custom: - Author: aranke - Issue: "247" diff --git a/CHANGELOG.md b/CHANGELOG.md index 9114bcd1..ce8d7837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.2 - July 02, 2024 + +### Under the Hood + +* Fix query timer resolution +* Add optional release_connection parameter to connection_named method + ## dbt-adapters 1.3.1 - June 20, 2024 ## dbt-adapters 1.3.0 - June 18, 2024 diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 35ec9a37..2238c8f6 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.1" +version = "1.3.2" From d2842a73d11b2e49d72d87429314e96eae0574fe Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:42:58 +0000 Subject: [PATCH 45/66] [Adap-207] remove subshell around get show sql with limit predicate (#249) Co-authored-by: Mila Page <versusfacit@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../Under the Hood-20240624-231955.yaml | 7 ++++ .gitignore | 3 ++ .../tests/adapter/dbt_show/test_dbt_show.py | 17 ++++++++++ .../global_project/macros/adapters/show.sql | 34 +++++++++++-------- 4 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240624-231955.yaml diff --git a/.changes/unreleased/Under the Hood-20240624-231955.yaml b/.changes/unreleased/Under the Hood-20240624-231955.yaml new file mode 100644 index 00000000..e3eaccb7 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240624-231955.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: --limit flag no longer subshells the query. This resolves the dbt Cloud experience + issue where limit prevents ordering elements.. +time: 2024-06-24T23:19:55.611142-07:00 +custom: + Author: versusfacit + Issue: "207" diff --git a/.gitignore b/.gitignore index a14d6d0d..807d3d41 100644 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,6 @@ cython_debug/ # PyCharm .idea/ + +# Vim +*.swp diff --git a/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py b/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py index c00491aa..3f3d36c5 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py +++ b/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py @@ -1,5 +1,6 @@ import pytest +from dbt_common.exceptions import DbtRuntimeError from dbt.tests.adapter.dbt_show import fixtures from dbt.tests.util import run_dbt @@ -47,9 +48,25 @@ def test_sql_header(self, project): run_dbt(["show", "--select", "sql_header", "--vars", "timezone: Asia/Kolkata"]) +class BaseShowDoesNotHandleDoubleLimit: + """see issue: https://github.com/dbt-labs/dbt-adapters/issues/207""" + + DATABASE_ERROR_MESSAGE = 'syntax error at or near "limit"' + + def test_double_limit_throws_syntax_error(self, project): + with pytest.raises(DbtRuntimeError) as e: + run_dbt(["show", "--limit", "1", "--inline", "select 1 limit 1"]) + + assert self.DATABASE_ERROR_MESSAGE in str(e) + + class TestPostgresShowSqlHeader(BaseShowSqlHeader): pass class TestPostgresShowLimit(BaseShowLimit): pass + + +class TestShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass diff --git a/dbt/include/global_project/macros/adapters/show.sql b/dbt/include/global_project/macros/adapters/show.sql index 33a93f3d..3a5faa98 100644 --- a/dbt/include/global_project/macros/adapters/show.sql +++ b/dbt/include/global_project/macros/adapters/show.sql @@ -1,22 +1,26 @@ +{# + We expect a syntax error if dbt show is invoked both with a --limit flag to show + and with a limit predicate embedded in its inline query. No special handling is + provided out-of-box. +#} {% macro get_show_sql(compiled_code, sql_header, limit) -%} - {%- if sql_header -%} + {%- if sql_header is not none -%} {{ sql_header }} - {%- endif -%} - {%- if limit is not none -%} + {%- endif %} {{ get_limit_subquery_sql(compiled_code, limit) }} - {%- else -%} - {{ compiled_code }} - {%- endif -%} {% endmacro %} -{% macro get_limit_subquery_sql(sql, limit) %} - {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }} -{% endmacro %} +{# + Not necessarily a true subquery anymore. Now, merely a query subordinate + to the calling macro. +#} +{%- macro get_limit_subquery_sql(sql, limit) -%} + {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }} +{%- endmacro -%} -{% macro default__get_limit_subquery_sql(sql, limit) %} - select * - from ( - {{ sql }} - ) as model_limit_subq - limit {{ limit }} +{% macro default__get_limit_sql(sql, limit) %} + {{ compiled_code }} + {% if limit is not none %} + limit {{ limit }} + {%- endif -%} {% endmacro %} From 99e81943d39baf74e9cbd9c797227a071949a66e Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:59:45 -0400 Subject: [PATCH 46/66] [Bug] Fix scenarios where `--empty` flag pushes limit 0 into metadata queries (#257) --- .../unreleased/Fixes-20240702-161935.yaml | 6 + .gitignore | 7 ++ .../dbt/tests/adapter/empty/_models.py | 111 ++++++++++++++++++ .../dbt/tests/adapter/empty/test_empty.py | 99 ++++++++-------- .../macros/adapters/apply_grants.sql | 8 +- .../macros/adapters/columns.sql | 10 +- .../macros/adapters/relation.sql | 2 +- .../materializations/models/clone/clone.sql | 4 +- .../models/clone/create_or_replace_clone.sql | 2 +- .../models/materialized_view.sql | 4 +- .../macros/materializations/seeds/helpers.sql | 2 +- .../macros/materializations/seeds/seed.sql | 2 +- .../materializations/snapshots/helpers.sql | 2 +- .../snapshots/snapshot_merge.sql | 2 +- .../global_project/macros/relations/drop.sql | 2 +- .../relations/materialized_view/drop.sql | 2 +- .../macros/relations/rename.sql | 2 +- .../macros/relations/table/drop.sql | 2 +- .../macros/relations/view/create.sql | 2 +- .../macros/relations/view/drop.sql | 2 +- .../macros/relations/view/replace.sql | 2 +- 21 files changed, 202 insertions(+), 73 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240702-161935.yaml create mode 100644 dbt-tests-adapter/dbt/tests/adapter/empty/_models.py diff --git a/.changes/unreleased/Fixes-20240702-161935.yaml b/.changes/unreleased/Fixes-20240702-161935.yaml new file mode 100644 index 00000000..0b0f21ee --- /dev/null +++ b/.changes/unreleased/Fixes-20240702-161935.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses +time: 2024-07-02T16:19:35.457997-04:00 +custom: + Author: mikealfare + Issue: "213" diff --git a/.gitignore b/.gitignore index 807d3d41..31258db2 100644 --- a/.gitignore +++ b/.gitignore @@ -155,5 +155,12 @@ cython_debug/ # PyCharm .idea/ +# MacOS +.DS_Store + +# vscode +.vscode/ +.venv/ + # Vim *.swp diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py b/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py new file mode 100644 index 00000000..f5e684f7 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py @@ -0,0 +1,111 @@ +model_input_sql = """ +select 1 as id +""" + +ephemeral_model_input_sql = """ +{{ config(materialized='ephemeral') }} +select 2 as id +""" + +raw_source_csv = """id +3 +""" + + +model_sql = """ +select * +from {{ ref('model_input') }} +union all +select * +from {{ ref('ephemeral_model_input') }} +union all +select * +from {{ source('seed_sources', 'raw_source') }} +""" + + +model_inline_sql = """ +select * from {{ source('seed_sources', 'raw_source') }} as raw_source +""" + +schema_sources_yml = """ +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_source +""" + + +SEED = """ +my_id,my_value +1,a +2,b +3,c +""".strip() + + +SCHEMA = """ +version: 2 + +seeds: + - name: my_seed + description: "This is my_seed" + columns: + - name: id + description: "This is my_seed.my_id" +""" + +CONTROL = """ +select * from {{ ref("my_seed") }} +""" + + +GET_COLUMNS_IN_RELATION = """ +{{ config(materialized="table") }} +{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_TYPE = """ +{{ config(materialized="table") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"relations": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"columns": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_ADD_REMOVE_COLUMNS = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{% set my_column = api.Column("my_column", "varchar") %} +{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %} +{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %} +select * from {{ ref("my_seed") }} +""" + + +TRUNCATE_RELATION = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{{ truncate_relation(my_seed) }} +select * from {{ ref("my_seed") }} +""" diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py index 2249d98d..a0a44edc 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py @@ -1,57 +1,23 @@ -import pytest - from dbt.tests.util import relation_from_name, run_dbt +import pytest - -model_input_sql = """ -select 1 as id -""" - -ephemeral_model_input_sql = """ -{{ config(materialized='ephemeral') }} -select 2 as id -""" - -raw_source_csv = """id -3 -""" - - -model_sql = """ -select * -from {{ ref('model_input') }} -union all -select * -from {{ ref('ephemeral_model_input') }} -union all -select * -from {{ source('seed_sources', 'raw_source') }} -""" - - -schema_sources_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_source -""" +from tests.functional.adapter.empty import _models class BaseTestEmpty: @pytest.fixture(scope="class") def seeds(self): return { - "raw_source.csv": raw_source_csv, + "raw_source.csv": _models.raw_source_csv, } @pytest.fixture(scope="class") def models(self): return { - "model_input.sql": model_input_sql, - "ephemeral_model_input.sql": ephemeral_model_input_sql, - "model.sql": model_sql, - "sources.yml": schema_sources_yml, + "model_input.sql": _models.model_input_sql, + "ephemeral_model_input.sql": _models.ephemeral_model_input_sql, + "model.sql": _models.model_sql, + "sources.yml": _models.schema_sources_yml, } def assert_row_count(self, project, relation_name: str, expected_row_count: int): @@ -75,13 +41,9 @@ def test_run_with_empty(self, project): class BaseTestEmptyInlineSourceRef(BaseTestEmpty): @pytest.fixture(scope="class") def models(self): - model_sql = """ - select * from {{ source('seed_sources', 'raw_source') }} as raw_source - """ - return { - "model.sql": model_sql, - "sources.yml": schema_sources_yml, + "model.sql": _models.model_inline_sql, + "sources.yml": _models.schema_sources_yml, } def test_run_with_empty(self, project): @@ -92,4 +54,47 @@ def test_run_with_empty(self, project): class TestEmpty(BaseTestEmpty): + """ + Though we don't create these classes anymore, we need to keep this one in case an adapter wanted to import the test as-is to automatically run it. + We should consider adding a deprecation warning that suggests moving this into the concrete adapter and importing `BaseTestEmpty` instead. + """ + pass + + +class MetadataWithEmptyFlag: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": _models.SEED} + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": _models.SCHEMA, + "control.sql": _models.CONTROL, + "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, + "alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT, + "alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT, + "alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS, + "truncate_relation.sql": _models.TRUNCATE_RELATION, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + + @pytest.mark.parametrize( + "model", + [ + "control", + "get_columns_in_relation", + "alter_column_type", + "alter_relation_comment", + "alter_column_comment", + "alter_relation_add_remove_columns", + "truncate_relation", + ], + ) + def test_run(self, project, model): + run_dbt(["run", "--empty", "--select", model]) diff --git a/dbt/include/global_project/macros/adapters/apply_grants.sql b/dbt/include/global_project/macros/adapters/apply_grants.sql index 10906e7f..c75eef89 100644 --- a/dbt/include/global_project/macros/adapters/apply_grants.sql +++ b/dbt/include/global_project/macros/adapters/apply_grants.sql @@ -61,7 +61,7 @@ {% endmacro %} {% macro default__get_show_grant_sql(relation) %} - show grants on {{ relation }} + show grants on {{ relation.render() }} {% endmacro %} @@ -70,7 +70,7 @@ {% endmacro %} {%- macro default__get_grant_sql(relation, privilege, grantees) -%} - grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }} + grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }} {%- endmacro -%} @@ -79,7 +79,7 @@ {% endmacro %} {%- macro default__get_revoke_sql(relation, privilege, grantees) -%} - revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }} + revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }} {%- endmacro -%} @@ -147,7 +147,7 @@ {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %} {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %} {% if not (needs_granting or needs_revoking) %} - {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}} + {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}} {% endif %} {% else %} {#-- We don't think there's any chance of previous grants having carried over. --#} diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index 663a827b..96e6f3f2 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -96,10 +96,10 @@ {%- set tmp_column = column_name + "__dbt_alter" -%} {% call statement('alter_column_type') %} - alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; - update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; - alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade; - alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} + alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; + update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; + alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade; + alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} {% endcall %} {% endmacro %} @@ -120,7 +120,7 @@ {% set sql -%} - alter {{ relation.type }} {{ relation }} + alter {{ relation.type }} {{ relation.render() }} {% for column in add_columns %} add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} diff --git a/dbt/include/global_project/macros/adapters/relation.sql b/dbt/include/global_project/macros/adapters/relation.sql index 1c2bd880..b9af4969 100644 --- a/dbt/include/global_project/macros/adapters/relation.sql +++ b/dbt/include/global_project/macros/adapters/relation.sql @@ -38,7 +38,7 @@ {% macro default__truncate_relation(relation) -%} {% call statement('truncate_relation') -%} - truncate table {{ relation }} + truncate table {{ relation.render() }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/models/clone/clone.sql b/dbt/include/global_project/macros/materializations/models/clone/clone.sql index 01c8c393..56d80082 100644 --- a/dbt/include/global_project/macros/materializations/models/clone/clone.sql +++ b/dbt/include/global_project/macros/materializations/models/clone/clone.sql @@ -27,14 +27,14 @@ {%- set target_relation = this.incorporate(type='table') -%} {% if existing_relation is not none and not existing_relation.is_table %} - {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }} + {{ log("Dropping relation " ~ existing_relation.render() ~ " because it is of type " ~ existing_relation.type) }} {{ drop_relation_if_exists(existing_relation) }} {% endif %} -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace' {% call statement('main') %} {% if target_relation and defer_relation and target_relation == defer_relation %} - {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation) }} + {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation.render()) }} {% else %} {{ create_or_replace_clone(target_relation, defer_relation) }} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql b/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql index 204e9e87..cdb2559c 100644 --- a/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +++ b/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql @@ -3,5 +3,5 @@ {% endmacro %} {% macro default__create_or_replace_clone(this_relation, defer_relation) %} - create or replace table {{ this_relation }} clone {{ defer_relation }} + create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/models/materialized_view.sql b/dbt/include/global_project/macros/materializations/models/materialized_view.sql index 6dc30bf9..a39f8aa2 100644 --- a/dbt/include/global_project/macros/materializations/models/materialized_view.sql +++ b/dbt/include/global_project/macros/materializations/models/materialized_view.sql @@ -71,9 +71,9 @@ {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %} {% elif on_configuration_change == 'continue' %} {% set build_sql = '' %} - {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }} + {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation.render() ~ "`") }} {% elif on_configuration_change == 'fail' %} - {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }} + {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation.render() ~ "`") }} {% else %} -- this only happens if the user provides a value other than `apply`, 'skip', 'fail' diff --git a/dbt/include/global_project/macros/materializations/seeds/helpers.sql b/dbt/include/global_project/macros/materializations/seeds/helpers.sql index 44dbf370..d87c258b 100644 --- a/dbt/include/global_project/macros/materializations/seeds/helpers.sql +++ b/dbt/include/global_project/macros/materializations/seeds/helpers.sql @@ -37,7 +37,7 @@ {% set sql = create_csv_table(model, agate_table) %} {% else %} {{ adapter.truncate_relation(old_relation) }} - {% set sql = "truncate table " ~ old_relation %} + {% set sql = "truncate table " ~ old_relation.render() %} {% endif %} {{ return(sql) }} diff --git a/dbt/include/global_project/macros/materializations/seeds/seed.sql b/dbt/include/global_project/macros/materializations/seeds/seed.sql index 3b66252d..4ee4fb80 100644 --- a/dbt/include/global_project/macros/materializations/seeds/seed.sql +++ b/dbt/include/global_project/macros/materializations/seeds/seed.sql @@ -22,7 +22,7 @@ -- build model {% set create_table_sql = "" %} {% if exists_as_view %} - {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation.render())) }} {% elif exists_as_table %} {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %} {% else %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 7fd4bfd5..bb71974c 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -8,7 +8,7 @@ {% macro default__create_columns(relation, columns) %} {% for column in columns %} {% call statement() %} - alter table {{ relation }} add column "{{ column.name }}" {{ column.data_type }}; + alter table {{ relation.render() }} add column "{{ column.name }}" {{ column.data_type }}; {% endcall %} {% endfor %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 6bc50fd3..56798811 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,7 +7,7 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - merge into {{ target }} as DBT_INTERNAL_DEST + merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id diff --git a/dbt/include/global_project/macros/relations/drop.sql b/dbt/include/global_project/macros/relations/drop.sql index 58abd14d..e66511da 100644 --- a/dbt/include/global_project/macros/relations/drop.sql +++ b/dbt/include/global_project/macros/relations/drop.sql @@ -16,7 +16,7 @@ {{ drop_materialized_view(relation) }} {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade + drop {{ relation.type }} if exists {{ relation.render() }} cascade {%- endif -%} diff --git a/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/dbt/include/global_project/macros/relations/materialized_view/drop.sql index b218d0f3..8235b1c6 100644 --- a/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} cascade + drop materialized view if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/rename.sql b/dbt/include/global_project/macros/relations/rename.sql index d7f3a72e..4b913df3 100644 --- a/dbt/include/global_project/macros/relations/rename.sql +++ b/dbt/include/global_project/macros/relations/rename.sql @@ -30,6 +30,6 @@ {% macro default__rename_relation(from_relation, to_relation) -%} {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ target_name }} + alter table {{ from_relation.render() }} rename to {{ target_name }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/global_project/macros/relations/table/drop.sql b/dbt/include/global_project/macros/relations/table/drop.sql index d7d5941c..038ded9e 100644 --- a/dbt/include/global_project/macros/relations/table/drop.sql +++ b/dbt/include/global_project/macros/relations/table/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_table(relation) -%} - drop table if exists {{ relation }} cascade + drop table if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/view/create.sql b/dbt/include/global_project/macros/relations/view/create.sql index 41cd196c..ee83befa 100644 --- a/dbt/include/global_project/macros/relations/view/create.sql +++ b/dbt/include/global_project/macros/relations/view/create.sql @@ -16,7 +16,7 @@ {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create view {{ relation }} + create view {{ relation.render() }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} {{ get_assert_columns_equivalent(sql) }} diff --git a/dbt/include/global_project/macros/relations/view/drop.sql b/dbt/include/global_project/macros/relations/view/drop.sql index 7e1924fa..84c91a36 100644 --- a/dbt/include/global_project/macros/relations/view/drop.sql +++ b/dbt/include/global_project/macros/relations/view/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_view(relation) -%} - drop view if exists {{ relation }} cascade + drop view if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/view/replace.sql b/dbt/include/global_project/macros/relations/view/replace.sql index 1da06134..a0f0dc76 100644 --- a/dbt/include/global_project/macros/relations/view/replace.sql +++ b/dbt/include/global_project/macros/relations/view/replace.sql @@ -61,6 +61,6 @@ {% endmacro %} {% macro default__handle_existing_table(full_refresh, old_relation) %} - {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} + {{ log("Dropping relation " ~ old_relation.render() ~ " because it is of type " ~ old_relation.type) }} {{ adapter.drop_relation(old_relation) }} {% endmacro %} From 7e9731cd819d521ffd638f0e1b96a96e58504eb5 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:25:11 -0400 Subject: [PATCH 47/66] Fix import statement for `--empty` tests (#258) --- dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py index a0a44edc..de15bd5b 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py @@ -1,7 +1,7 @@ from dbt.tests.util import relation_from_name, run_dbt import pytest -from tests.functional.adapter.empty import _models +from dbt.tests.adapter.empty import _models class BaseTestEmpty: From 78b2b4c5867e13d07f5c5d9bb4ffcd6081aedd22 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 9 Jul 2024 20:07:46 +0000 Subject: [PATCH 48/66] Bumping version to 1.3.3 and generate changelog --- .changes/1.3.3.md | 9 +++++++++ .changes/unreleased/Fixes-20240702-161935.yaml | 6 ------ .../unreleased/Under the Hood-20240624-231955.yaml | 7 ------- CHANGELOG.md | 10 ++++++++++ dbt/adapters/__about__.py | 2 +- 5 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 .changes/1.3.3.md delete mode 100644 .changes/unreleased/Fixes-20240702-161935.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240624-231955.yaml diff --git a/.changes/1.3.3.md b/.changes/1.3.3.md new file mode 100644 index 00000000..22ab18e3 --- /dev/null +++ b/.changes/1.3.3.md @@ -0,0 +1,9 @@ +## dbt-adapters 1.3.3 - July 09, 2024 + +### Fixes + +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses + +### Under the Hood + +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. diff --git a/.changes/unreleased/Fixes-20240702-161935.yaml b/.changes/unreleased/Fixes-20240702-161935.yaml deleted file mode 100644 index 0b0f21ee..00000000 --- a/.changes/unreleased/Fixes-20240702-161935.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses -time: 2024-07-02T16:19:35.457997-04:00 -custom: - Author: mikealfare - Issue: "213" diff --git a/.changes/unreleased/Under the Hood-20240624-231955.yaml b/.changes/unreleased/Under the Hood-20240624-231955.yaml deleted file mode 100644 index e3eaccb7..00000000 --- a/.changes/unreleased/Under the Hood-20240624-231955.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Under the Hood -body: --limit flag no longer subshells the query. This resolves the dbt Cloud experience - issue where limit prevents ordering elements.. -time: 2024-06-24T23:19:55.611142-07:00 -custom: - Author: versusfacit - Issue: "207" diff --git a/CHANGELOG.md b/CHANGELOG.md index ce8d7837..8834049e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,16 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.3 - July 09, 2024 + +### Fixes + +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses + +### Under the Hood + +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. + ## dbt-adapters 1.3.2 - July 02, 2024 ### Under the Hood diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 2238c8f6..cfa2db68 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.2" +version = "1.3.3" From 594ad067ea1e85f4cef4fb2b2b420695c6ba936f Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 9 Jul 2024 20:22:10 +0000 Subject: [PATCH 49/66] Bumping version to 1.9.2 and generate changelog --- .changes/1.9.2.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.9.2.md diff --git a/.changes/1.9.2.md b/.changes/1.9.2.md new file mode 100644 index 00000000..26505f20 --- /dev/null +++ b/.changes/1.9.2.md @@ -0,0 +1 @@ +## dbt-adapters 1.9.2 - July 09, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8834049e..1a2f1833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.9.2 - July 09, 2024 + ## dbt-adapters 1.9.1 - June 20, 2024 ## dbt-adapters 1.9.0 - June 18, 2024 diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 70227976..1b022739 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.9.1" +version = "1.9.2" From 6e50885b94af636bd8c39d92a201ff11c75b05af Mon Sep 17 00:00:00 2001 From: Peter Webb <peterallenwebb@gmail.com> Date: Tue, 16 Jul 2024 17:10:28 -0400 Subject: [PATCH 50/66] Rework record/replay to record at the database connection level. (#244) Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../Under the Hood-20240625-110833.yaml | 6 ++ dbt/adapters/record.py | 67 ------------------- dbt/adapters/record/__init__.py | 2 + dbt/adapters/record/cursor/cursor.py | 54 +++++++++++++++ dbt/adapters/record/cursor/description.py | 37 ++++++++++ dbt/adapters/record/cursor/execute.py | 20 ++++++ dbt/adapters/record/cursor/fetchall.py | 66 ++++++++++++++++++ dbt/adapters/record/cursor/fetchmany.py | 23 +++++++ dbt/adapters/record/cursor/fetchone.py | 23 +++++++ dbt/adapters/record/cursor/rowcount.py | 23 +++++++ dbt/adapters/record/handle.py | 24 +++++++ dbt/adapters/sql/connections.py | 3 - docs/guides/record_replay.md | 16 +---- 13 files changed, 280 insertions(+), 84 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240625-110833.yaml delete mode 100644 dbt/adapters/record.py create mode 100644 dbt/adapters/record/__init__.py create mode 100644 dbt/adapters/record/cursor/cursor.py create mode 100644 dbt/adapters/record/cursor/description.py create mode 100644 dbt/adapters/record/cursor/execute.py create mode 100644 dbt/adapters/record/cursor/fetchall.py create mode 100644 dbt/adapters/record/cursor/fetchmany.py create mode 100644 dbt/adapters/record/cursor/fetchone.py create mode 100644 dbt/adapters/record/cursor/rowcount.py create mode 100644 dbt/adapters/record/handle.py diff --git a/.changes/unreleased/Under the Hood-20240625-110833.yaml b/.changes/unreleased/Under the Hood-20240625-110833.yaml new file mode 100644 index 00000000..c4486a8f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240625-110833.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Rework record/replay to record at the database connection level. +time: 2024-06-25T11:08:33.264457-04:00 +custom: + Author: peteralllenwebb + Issue: "244" diff --git a/dbt/adapters/record.py b/dbt/adapters/record.py deleted file mode 100644 index 5204f59c..00000000 --- a/dbt/adapters/record.py +++ /dev/null @@ -1,67 +0,0 @@ -import dataclasses -from io import StringIO -import json -import re -from typing import Any, Optional, Mapping - -from agate import Table - -from dbt_common.events.contextvars import get_node_info -from dbt_common.record import Record, Recorder - -from dbt.adapters.contracts.connection import AdapterResponse - - -@dataclasses.dataclass -class QueryRecordParams: - sql: str - auto_begin: bool = False - fetch: bool = False - limit: Optional[int] = None - node_unique_id: Optional[str] = None - - def __post_init__(self) -> None: - if self.node_unique_id is None: - node_info = get_node_info() - self.node_unique_id = node_info["unique_id"] if node_info else "" - - @staticmethod - def _clean_up_sql(sql: str) -> str: - sql = re.sub(r"--.*?\n", "", sql) # Remove single-line comments (--) - sql = re.sub(r"/\*.*?\*/", "", sql, flags=re.DOTALL) # Remove multi-line comments (/* */) - return sql.replace(" ", "").replace("\n", "") - - def _matches(self, other: "QueryRecordParams") -> bool: - return self.node_unique_id == other.node_unique_id and self._clean_up_sql( - self.sql - ) == self._clean_up_sql(other.sql) - - -@dataclasses.dataclass -class QueryRecordResult: - adapter_response: Optional["AdapterResponse"] - table: Optional[Table] - - def _to_dict(self) -> Any: - buf = StringIO() - self.table.to_json(buf) # type: ignore - - return { - "adapter_response": self.adapter_response.to_dict(), # type: ignore - "table": buf.getvalue(), - } - - @classmethod - def _from_dict(cls, dct: Mapping) -> "QueryRecordResult": - return QueryRecordResult( - adapter_response=AdapterResponse.from_dict(dct["adapter_response"]), - table=Table.from_object(json.loads(dct["table"])), - ) - - -class QueryRecord(Record): - params_cls = QueryRecordParams - result_cls = QueryRecordResult - - -Recorder.register_record_type(QueryRecord) diff --git a/dbt/adapters/record/__init__.py b/dbt/adapters/record/__init__.py new file mode 100644 index 00000000..afde4a01 --- /dev/null +++ b/dbt/adapters/record/__init__.py @@ -0,0 +1,2 @@ +from dbt.adapters.record.handle import RecordReplayHandle +from dbt.adapters.record.cursor.cursor import RecordReplayCursor diff --git a/dbt/adapters/record/cursor/cursor.py b/dbt/adapters/record/cursor/cursor.py new file mode 100644 index 00000000..577178db --- /dev/null +++ b/dbt/adapters/record/cursor/cursor.py @@ -0,0 +1,54 @@ +from typing import Any, Optional + +from dbt_common.record import record_function + +from dbt.adapters.contracts.connection import Connection +from dbt.adapters.record.cursor.description import CursorGetDescriptionRecord +from dbt.adapters.record.cursor.execute import CursorExecuteRecord +from dbt.adapters.record.cursor.fetchone import CursorFetchOneRecord +from dbt.adapters.record.cursor.fetchmany import CursorFetchManyRecord +from dbt.adapters.record.cursor.fetchall import CursorFetchAllRecord +from dbt.adapters.record.cursor.rowcount import CursorGetRowCountRecord + + +class RecordReplayCursor: + """A proxy object used to wrap native database cursors under record/replay + modes. In record mode, this proxy notes the parameters and return values + of the methods and properties it implements, which closely match the Python + DB API 2.0 cursor methods used by many dbt adapters to interact with the + database or DWH. In replay mode, it mocks out those calls using previously + recorded calls, so that no interaction with a database actually occurs.""" + + def __init__(self, native_cursor: Any, connection: Connection) -> None: + self.native_cursor = native_cursor + self.connection = connection + + @record_function(CursorExecuteRecord, method=True, id_field_name="connection_name") + def execute(self, operation, parameters=None) -> None: + self.native_cursor.execute(operation, parameters) + + @record_function(CursorFetchOneRecord, method=True, id_field_name="connection_name") + def fetchone(self) -> Any: + return self.native_cursor.fetchone() + + @record_function(CursorFetchManyRecord, method=True, id_field_name="connection_name") + def fetchmany(self, size: int) -> Any: + return self.native_cursor.fetchmany(size) + + @record_function(CursorFetchAllRecord, method=True, id_field_name="connection_name") + def fetchall(self) -> Any: + return self.native_cursor.fetchall() + + @property + def connection_name(self) -> Optional[str]: + return self.connection.name + + @property + @record_function(CursorGetRowCountRecord, method=True, id_field_name="connection_name") + def rowcount(self) -> int: + return self.native_cursor.rowcount + + @property + @record_function(CursorGetDescriptionRecord, method=True, id_field_name="connection_name") + def description(self) -> str: + return self.native_cursor.description diff --git a/dbt/adapters/record/cursor/description.py b/dbt/adapters/record/cursor/description.py new file mode 100644 index 00000000..d6ba15d9 --- /dev/null +++ b/dbt/adapters/record/cursor/description.py @@ -0,0 +1,37 @@ +import dataclasses +from typing import Any, Iterable, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetDescriptionParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetDescriptionResult: + columns: Iterable[Any] + + def _to_dict(self) -> Any: + column_dicts = [] + for c in self.columns: + # This captures the mandatory column information, but we might need + # more for some adapters. + # See https://peps.python.org/pep-0249/#description + column_dicts.append((c[0], c[1])) + + return {"columns": column_dicts} + + @classmethod + def _from_dict(cls, dct: Mapping) -> "CursorGetDescriptionResult": + return CursorGetDescriptionResult(columns=dct["columns"]) + + +@Recorder.register_record_type +class CursorGetDescriptionRecord(Record): + """Implements record/replay support for the cursor.description property.""" + + params_cls = CursorGetDescriptionParams + result_cls = CursorGetDescriptionResult + group = "Database" diff --git a/dbt/adapters/record/cursor/execute.py b/dbt/adapters/record/cursor/execute.py new file mode 100644 index 00000000..e7e69859 --- /dev/null +++ b/dbt/adapters/record/cursor/execute.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import Any, Iterable, Union, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorExecuteParams: + connection_name: str + operation: str + parameters: Union[Iterable[Any], Mapping[str, Any]] + + +@Recorder.register_record_type +class CursorExecuteRecord(Record): + """Implements record/replay support for the cursor.execute() method.""" + + params_cls = CursorExecuteParams + result_cls = None + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchall.py b/dbt/adapters/record/cursor/fetchall.py new file mode 100644 index 00000000..090cc160 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchall.py @@ -0,0 +1,66 @@ +import dataclasses +import datetime +from typing import Any, Dict, List, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchAllParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchAllResult: + results: List[Any] + + def _to_dict(self) -> Dict[str, Any]: + processed_results = [] + for result in self.results: + result = tuple(map(self._process_value, result)) + processed_results.append(result) + + return {"results": processed_results} + + @classmethod + def _from_dict(cls, dct: Mapping) -> "CursorFetchAllResult": + unprocessed_results = [] + for result in dct["results"]: + result = tuple(map(cls._unprocess_value, result)) + unprocessed_results.append(result) + + return CursorFetchAllResult(unprocessed_results) + + @classmethod + def _process_value(cls, value: Any) -> Any: + if type(value) is datetime.date: + return {"type": "date", "value": value.isoformat()} + elif type(value) is datetime.datetime: + return {"type": "datetime", "value": value.isoformat()} + else: + return value + + @classmethod + def _unprocess_value(cls, value: Any) -> Any: + if type(value) is dict: + value_type = value.get("type") + if value_type == "date": + date_string = value.get("value") + assert isinstance(date_string, str) + return datetime.date.fromisoformat(date_string) + elif value_type == "datetime": + date_string = value.get("value") + assert isinstance(date_string, str) + return datetime.datetime.fromisoformat(date_string) + return value + else: + return value + + +@Recorder.register_record_type +class CursorFetchAllRecord(Record): + """Implements record/replay support for the cursor.fetchall() method.""" + + params_cls = CursorFetchAllParams + result_cls = CursorFetchAllResult + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchmany.py b/dbt/adapters/record/cursor/fetchmany.py new file mode 100644 index 00000000..86f15440 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchmany.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Any, List + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchManyParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchManyResult: + results: List[Any] + + +@Recorder.register_record_type +class CursorFetchManyRecord(Record): + """Implements record/replay support for the cursor.fetchmany() method.""" + + params_cls = CursorFetchManyParams + result_cls = CursorFetchManyResult + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchone.py b/dbt/adapters/record/cursor/fetchone.py new file mode 100644 index 00000000..42ffe210 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchone.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Any + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchOneParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchOneResult: + result: Any + + +@Recorder.register_record_type +class CursorFetchOneRecord(Record): + """Implements record/replay support for the cursor.fetchone() method.""" + + params_cls = CursorFetchOneParams + result_cls = CursorFetchOneResult + group = "Database" diff --git a/dbt/adapters/record/cursor/rowcount.py b/dbt/adapters/record/cursor/rowcount.py new file mode 100644 index 00000000..c024817e --- /dev/null +++ b/dbt/adapters/record/cursor/rowcount.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Optional + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetRowCountParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetRowCountResult: + rowcount: Optional[int] + + +@Recorder.register_record_type +class CursorGetRowCountRecord(Record): + """Implements record/replay support for the cursor.rowcount property.""" + + params_cls = CursorGetRowCountParams + result_cls = CursorGetRowCountResult + group = "Database" diff --git a/dbt/adapters/record/handle.py b/dbt/adapters/record/handle.py new file mode 100644 index 00000000..31817c37 --- /dev/null +++ b/dbt/adapters/record/handle.py @@ -0,0 +1,24 @@ +from typing import Any + +from dbt.adapters.contracts.connection import Connection + +from dbt.adapters.record.cursor.cursor import RecordReplayCursor + + +class RecordReplayHandle: + """A proxy object used for record/replay modes. What adapters call a + 'handle' is typically a native database connection, but should not be + confused with the Connection protocol, which is a dbt-adapters concept. + + Currently, the only function of the handle proxy is to provide a record/replay + aware cursor object when cursor() is called.""" + + def __init__(self, native_handle: Any, connection: Connection) -> None: + self.native_handle = native_handle + self.connection = connection + + def cursor(self) -> Any: + # The native handle could be None if we are in replay mode, because no + # actual database access should be performed in that mode. + cursor = None if self.native_handle is None else self.native_handle.cursor() + return RecordReplayCursor(cursor, self.connection) diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index 13111a6e..4d450c88 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -5,7 +5,6 @@ from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError, NotImplementedError -from dbt_common.record import record_function from dbt_common.utils import cast_to_str from dbt.adapters.base import BaseConnectionManager @@ -20,7 +19,6 @@ SQLQuery, SQLQueryStatus, ) -from dbt.adapters.record import QueryRecord if TYPE_CHECKING: import agate @@ -143,7 +141,6 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Tab return table_from_data_flat(data, column_names) - @record_function(QueryRecord, method=True, tuple_result=True) def execute( self, sql: str, diff --git a/docs/guides/record_replay.md b/docs/guides/record_replay.md index 670bb843..5bcbec06 100644 --- a/docs/guides/record_replay.md +++ b/docs/guides/record_replay.md @@ -4,24 +4,12 @@ This document describes how to implement support for dbt's Record/Replay Subsyst ## Recording and Replaying Warehouse Interaction -The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most obvious. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. (It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, and so forth.) +The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most important. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, etc. -In practice, this means that any request sent to the warehouse must be recorded, along with the corresponding response. If this is done correctly, as described in the document linked in the intro, the Record portion of the Record/Replay subsystem should work as expected. - -At the time of this writing, there is only an incomplete implementation of this goal, which can be found in `dbt-adapters/dbt/adapters/record.py`. - -There are some important things to notice about this implementation. First, the QueryRecordResult class provides custom serialization methods `to_dict()` and `from_dict()`. This is necessary because the `AdapterResponse` and `Agate.Table` types cannot be automatically converted to and from JSON by the dataclass library, and JSON is the format used to persist recordings to disk and reload them for replay. - -Another important feature is that `QueryRecordParams` implements the `_matches()` method. This method allows `dbt-adapters` to customize the way that the Record/Replay determines whether a query issued by dbt matches a previously recorded query. In this case, the method performs a comparison which attempts to ignore comments and whitespace which would not affect query behavior. +A basic implementation of Record/Replay functionality, suitable for most adapters which extend the `SQLAdapter` class, can be found in `dbt-adapters/dbt/adapters/record`. The `RecordReplayHandle` and `RecordReplayCursor` classes defined there are used to intercept and record or replay all DWH interactions. They are an excellent starting point for adapters which extend `SQLAdapter` and use a database library which substantially conforms to Python's DB API v2.0 (PEP 249). Examples of how library-specific deviations from that API can be found in the dbt-postgress and dbt-snowflake repositories. ## Misc. Notes and Suggestions -Currently, support for recording data warehouse interaction is very rudimentary, however, even rudimentary support is valuable and we should be concentrating on extending it in a way that adds the most value with the least work. Usefulness, rather than perfection, is the initial goal. - -Picking the right functions to record, at the right level of abstraction, will probably be the most important part of carrying this work forward. - Not every interaction with an external system has to be recorded in full detail, and authentication might prove to be a place where we exclude sensitive secrets from the recording. For example, since replay will not actually be communicating with the warehouse, it may be possible to exclude passwords and auth keys from the parameters recorded, and to exclude auth tokens from the results. In addition to adding an appropriate decorator to functions which communicate with external systems, you should check those functions for side-effects. Since the function's calls will be mocked out in replay mode, those side-effects will not be carried out during replay. At present, we are focusing on support for recording and comparing recordings, but this is worth keeping in mind. - -The current implementation records which dbt node issues a query, and uses that information to ensure a match during replay. The same node should issue the same query. A better model might be to monitor which connection issued which query, and associate the same connection with open/close operations, transaction starts/stops and so forth. From 43ea32d821ed041cdc03175308e1f411211847a3 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 16 Jul 2024 18:20:20 -0400 Subject: [PATCH 51/66] Update contributor guide for `hatch` (#251) --- .github/actions/build-hatch/action.yml | 3 +- .github/workflows/build.yml | 31 --------- .github/workflows/release_prep_hatch.yml | 6 +- .github/workflows/unit-tests.yml | 2 +- .gitignore | 2 +- CONTRIBUTING.md | 88 +++++++++++++----------- dbt-tests-adapter/pyproject.toml | 14 ++-- pyproject.toml | 19 ++--- 8 files changed, 64 insertions(+), 101 deletions(-) diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml index fe9825d4..f2da5935 100644 --- a/.github/actions/build-hatch/action.yml +++ b/.github/actions/build-hatch/action.yml @@ -13,7 +13,7 @@ inputs: default: "./" archive-name: description: Where to upload the artifacts - required: true + default: "" runs: using: composite @@ -30,6 +30,7 @@ runs: working-directory: ${{ inputs.working-dir }} - name: Upload artifacts + if: ${{ inputs.archive-name != '' }} uses: actions/upload-artifact@v3 with: name: ${{ inputs.archive-name }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33d94ff4..00afd704 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,6 @@ on: types: [checks_requested] workflow_dispatch: workflow_call: - inputs: - changelog_path: - description: "Path to changelog file" - required: true - type: string permissions: read-all @@ -51,35 +46,9 @@ jobs: uses: ./.github/actions/setup-hatch - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} uses: ./.github/actions/build-hatch - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} uses: ./.github/actions/build-hatch with: working-dir: "./dbt-tests-adapter/" - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" - - # this step is only needed for the release process - - name: "Upload Build Artifact" - if: ${{ github.event_name == 'workflow_call' }} - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.version.outputs.version_number }} - path: | - ${{ inputs.changelog_path }} - ./dist/ - retention-days: 3 diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index b043e19e..ec029ad5 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -34,7 +34,7 @@ # name: Version Bump and Changelog Generation -run-name: Bump ${{ inputs.package }}==${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog +run-name: Bump to ${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog on: workflow_call: inputs: @@ -398,7 +398,7 @@ jobs: - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - name: "Run Unit Tests" - run: hatch run unit-tests:all + run: hatch run unit-tests run-integration-tests: runs-on: ubuntu-20.04 @@ -447,7 +447,7 @@ jobs: python-version: ${{ env.PYTHON_TARGET_VERSION }} - name: Run tests - run: hatch run integration-tests:all + run: hatch run integration-tests merge-changes-into-target-branch: runs-on: ubuntu-latest diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7d720655..b61c83d7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -37,7 +37,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run unit tests - run: hatch run unit-tests:all + run: hatch run unit-tests shell: bash - name: Publish results diff --git a/.gitignore b/.gitignore index 31258db2..29c470c5 100644 --- a/.gitignore +++ b/.gitignore @@ -158,7 +158,7 @@ cython_debug/ # MacOS .DS_Store -# vscode +# VSCode .vscode/ .venv/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28f13f6c..1a6e92a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,13 +17,10 @@ This guide assumes users are developing on a Linux or MacOS system. The following utilities are needed or will be installed in this guide: - `pip` -- `virturalenv` +- `hatch` - `git` - `changie` -If local functional testing is required, then a database instance -and appropriate credentials are also required. - In addition to this guide, users are highly encouraged to read the `dbt-core` [CONTRIBUTING.md](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md). Almost all information there is applicable here. @@ -66,25 +63,39 @@ Rather than forking `dbt-labs/dbt-adapters`, use `dbt-labs/dbt-adapters` directl ### Installation -1. Ensure the latest version of `pip` is installed: +1. Ensure the latest versions of `pip` and `hatch` are installed: ```shell - pip install --upgrade pip + pip install --user --upgrade pip hatch + ``` +2. This step is optional, but it's recommended. Configure `hatch` to create its virtual environments in the project. Add this block to your `hatch` `config.toml` file: + ```toml + # MacOS: ~/Library/Application Support/hatch/config.toml + [dirs.env] + virtual = ".hatch" ``` -2. Configure and activate a virtual environment using `virtualenv` as described in -[Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) -3. Install `dbt-adapters` and development dependencies in the virtual environment + This makes `hatch` create all virtual environments in the project root inside of the directory `/.hatch`, similar to `/.tox` for `tox`. + It also makes it easier to add this environment as a runner in common IDEs like VSCode and PyCharm. +3. Create a `hatch` environment with all of the development dependencies and activate it: ```shell - pip install -e .[dev] + hatch run setup + hatch shell + ``` +4. Run any commands within the virtual environment by prefixing the command with `hatch run`: + ```shell + hatch run <command> ``` -When `dbt-adapters` is installed this way, any changes made to the `dbt-adapters` source code -will be reflected in the virtual environment immediately. +## Testing +`dbt-adapters` contains [code quality checks](https://github.com/dbt-labs/dbt-adapters/tree/main/.pre-commit-config.yaml) and [unit tests](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit). +While `dbt-tests-adapter` is also hosted in this repo, it requires a concrete adapter to run. -## Testing +### Code quality -`dbt-adapters` contains [unit](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit) -and [functional](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/functional) tests. +Code quality checks can run with a single command: +```shell +hatch run code-quality +``` ### Unit tests @@ -93,43 +104,38 @@ Unit tests can be run locally without setting up a database connection: ```shell # Note: replace $strings with valid names +# run all unit tests +hatch run unit-test + # run all unit tests in a module -python -m pytest tests/unit/$test_file_name.py +hatch run unit-test tests/unit/$test_file_name.py + # run a specific unit test -python -m pytest tests/unit/$test_file_name.py::$test_class_name::$test_method_name +hatch run unit-test tests/unit/$test_file_name.py::$test_class_name::$test_method_name ``` -### Functional tests - -Functional tests require a database to test against. There are two primary ways to run functional tests: +### Testing against a development branch -- Tests will run automatically against a dbt Labs owned database during PR checks -- Tests can be run locally by configuring a `test.env` file with appropriate `ENV` variables: - ```shell - cp test.env.example test.env - $EDITOR test.env - ``` +Some changes require a change in `dbt-common` and `dbt-adapters`. +In that case, the dependency on `dbt-common` must be updated to point to the development branch. For example: -> **_WARNING:_** The parameters in `test.env` must link to a valid database. -> `test.env` is git-ignored, but be _extra_ careful to never check in credentials -> or other sensitive information when developing. +```toml +[tool.hatch.envs.default] +dependencies = [ + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@my-dev-branch", + ..., +] +``` -Functional tests can be run locally with a valid database connection configured in `test.env`: +This will install `dbt-common` as a snapshot. In other words, if `my-dev-branch` is updated on GitHub, those updates will not be reflected locally. +In order to pick up those updates, the `hatch` environment(s) will need to be rebuilt: ```shell -# Note: replace $strings with valid names - -# run all functional tests in a directory -python -m pytest tests/functional/$test_directory -# run all functional tests in a module -python -m pytest tests/functional/$test_dir_and_filename.py -# run all functional tests in a class -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name -# run a specific functional test -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name::$test__method_name +exit +hatch env prune +hatch shell ``` - ## Documentation ### User documentation diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index 20342078..73bce49e 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -32,14 +32,6 @@ dependencies = [ "dbt-adapters", "pyyaml", ] - -[project.optional-dependencies] -build = [ - "wheel", - "twine", - "check-wheel-contents", -] - [project.urls] Homepage = "https://github.com/dbt-labs/dbt-adapters" Documentation = "https://docs.getdbt.com" @@ -62,7 +54,11 @@ include = ["dbt/tests", "dbt/__init__.py"] [tool.hatch.envs.build] detached = true -features = ["build"] +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] [tool.hatch.envs.build.scripts] check-all = [ "- check-wheel", diff --git a/pyproject.toml b/pyproject.toml index e50aa63a..55529f34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ maintainers = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] classifiers = [ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", @@ -43,9 +43,6 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.version] path = "dbt/adapters/__about__.py" @@ -60,20 +57,14 @@ dependencies = [ "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', -] -[tool.hatch.envs.default.scripts] -dev = "pre-commit install" -code-quality = "pre-commit run --all-files" - -[tool.hatch.envs.unit-tests] -dependencies = [ - "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", "pytest", "pytest-dotenv", "pytest-xdist", ] -[tool.hatch.envs.unit-tests.scripts] -all = "python -m pytest {args:tests/unit}" +[tool.hatch.envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" [tool.hatch.envs.build] detached = true From ebf8f37705823c8210e6b23b84946075d0e29a7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:20:41 -0400 Subject: [PATCH 52/66] Bump actions/checkout from 3 to 4 (#170) --- .github/workflows/changelog-existence.yml | 3 --- .github/workflows/release_prep_hatch.yml | 10 +++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml index d778f565..8732177f 100644 --- a/.github/workflows/changelog-existence.yml +++ b/.github/workflows/changelog-existence.yml @@ -19,9 +19,6 @@ name: Check Changelog Entry on: pull_request_target: types: [opened, reopened, labeled, unlabeled, synchronize] - paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - - workflow_dispatch: defaults: run: diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index ec029ad5..a13d45ce 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -288,7 +288,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} - name: Setup `hatch` @@ -392,7 +392,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} - name: "Setup `hatch`" @@ -407,7 +407,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} @@ -467,7 +467,7 @@ jobs: echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: "Merge Changes Into ${{ inputs.target_branch }}" uses: everlytic/branch-merge@1.1.5 @@ -524,7 +524,7 @@ jobs: message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.resolve_branch.outputs.target_branch }} From f76375b333615e05b2ee5f1bb2909e9b16da1dcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:22:17 -0400 Subject: [PATCH 53/66] Bump actions/download-artifact from 3 to 4 (#168) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/github-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index 1c2f41b5..ad0cc2d8 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -208,7 +208,7 @@ jobs: ref: ${{ inputs.sha }} - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive_name }} path: dist/ From 4b3806818f3831bbd0206a6b22fdcb4fe2a505c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 21:25:32 +0000 Subject: [PATCH 54/66] Bump dbt-labs/actions from 1.1.0 to 1.1.1 (#169) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/release_prep_hatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index a13d45ce..a6105786 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -131,7 +131,7 @@ jobs: - name: "Audit Version And Parse Into Parts" id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ inputs.version_number }} From e3465c2a7baaea66f71b1da94647a193c281a992 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:12:07 -0700 Subject: [PATCH 55/66] Correct CHANGELOG.md (#264) Co-authored-by: Mike Alfare <mike.alfare@dbtlabs.com> Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/0.1.0/Docs-20240109-131629.yaml | 6 -- .changes/0.1.0/Docs-20240109-131736.yaml | 6 -- .changes/0.1.0/Docs-20240109-131858.yaml | 6 -- .changes/0.1.0/Docs-20240109-131917.yaml | 6 -- .changes/0.1.0/Features-20240212-123544.yaml | 6 -- .changes/0.1.0/Fixes-20240215-141545.yaml | 6 -- .changes/0.1.0/Fixes-20240216-135420.yaml | 6 -- .../0.1.0/Under the Hood-20240109-131958.yaml | 6 -- .../0.1.0/Under the Hood-20240112-230236.yaml | 6 -- .../0.1.0/Under the Hood-20240123-121220.yaml | 6 -- .../0.1.0/Under the Hood-20240220-164223.yaml | 6 -- .changes/1.0.0.md | 27 +++++- .changes/1.1.0.md | 26 +++--- .changes/1.1.1.md | 2 +- .changes/1.2.1.md | 11 ++- .changes/1.3.0.md | 2 +- .changes/1.3.2.md | 4 +- .changes/1.3.3.md | 4 +- .changes/1.8.0.md | 9 -- .changes/1.9.0.md | 1 - .changes/1.9.1.md | 1 - .changes/1.9.2.md | 1 - .changie.yaml | 61 ++++++++++-- CHANGELOG.md | 92 ++++++++++--------- 24 files changed, 150 insertions(+), 157 deletions(-) delete mode 100644 .changes/0.1.0/Docs-20240109-131629.yaml delete mode 100644 .changes/0.1.0/Docs-20240109-131736.yaml delete mode 100644 .changes/0.1.0/Docs-20240109-131858.yaml delete mode 100644 .changes/0.1.0/Docs-20240109-131917.yaml delete mode 100644 .changes/0.1.0/Features-20240212-123544.yaml delete mode 100644 .changes/0.1.0/Fixes-20240215-141545.yaml delete mode 100644 .changes/0.1.0/Fixes-20240216-135420.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240109-131958.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240112-230236.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240123-121220.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240220-164223.yaml delete mode 100644 .changes/1.8.0.md delete mode 100644 .changes/1.9.0.md delete mode 100644 .changes/1.9.1.md delete mode 100644 .changes/1.9.2.md diff --git a/.changes/0.1.0/Docs-20240109-131629.yaml b/.changes/0.1.0/Docs-20240109-131629.yaml deleted file mode 100644 index 22b2ad3f..00000000 --- a/.changes/0.1.0/Docs-20240109-131629.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Configure `changie` -time: 2024-01-09T13:16:29.763021-05:00 -custom: - Author: mikealfare - Issue: 16 diff --git a/.changes/0.1.0/Docs-20240109-131736.yaml b/.changes/0.1.0/Docs-20240109-131736.yaml deleted file mode 100644 index 43186903..00000000 --- a/.changes/0.1.0/Docs-20240109-131736.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Setup ADR tracking framework -time: 2024-01-09T13:17:36.094147-05:00 -custom: - Author: mikealfare - Issue: "11" diff --git a/.changes/0.1.0/Docs-20240109-131858.yaml b/.changes/0.1.0/Docs-20240109-131858.yaml deleted file mode 100644 index decef9a7..00000000 --- a/.changes/0.1.0/Docs-20240109-131858.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Create issue templates -time: 2024-01-09T13:18:58.11819-05:00 -custom: - Author: mikealfare - Issue: "12" diff --git a/.changes/0.1.0/Docs-20240109-131917.yaml b/.changes/0.1.0/Docs-20240109-131917.yaml deleted file mode 100644 index 3c531060..00000000 --- a/.changes/0.1.0/Docs-20240109-131917.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Create PR template -time: 2024-01-09T13:19:17.749914-05:00 -custom: - Author: mikealfare - Issue: "13" diff --git a/.changes/0.1.0/Features-20240212-123544.yaml b/.changes/0.1.0/Features-20240212-123544.yaml deleted file mode 100644 index 239ad59f..00000000 --- a/.changes/0.1.0/Features-20240212-123544.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Update RelationConfig to capture all fields used by adapters -time: 2024-02-12T12:35:44.653555-08:00 -custom: - Author: colin-rogers-dbt - Issue: "30" diff --git a/.changes/0.1.0/Fixes-20240215-141545.yaml b/.changes/0.1.0/Fixes-20240215-141545.yaml deleted file mode 100644 index ced62f25..00000000 --- a/.changes/0.1.0/Fixes-20240215-141545.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Ignore adapter-level support warnings for 'custom' constraints -time: 2024-02-15T14:15:45.764145+01:00 -custom: - Author: jtcohen6 - Issue: "90" diff --git a/.changes/0.1.0/Fixes-20240216-135420.yaml b/.changes/0.1.0/Fixes-20240216-135420.yaml deleted file mode 100644 index a04cd26b..00000000 --- a/.changes/0.1.0/Fixes-20240216-135420.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Make all adapter zone tests importable by removing "Test" prefix -time: 2024-02-16T13:54:20.411864-05:00 -custom: - Author: mikealfare - Issue: "93" diff --git a/.changes/0.1.0/Under the Hood-20240109-131958.yaml b/.changes/0.1.0/Under the Hood-20240109-131958.yaml deleted file mode 100644 index a062a299..00000000 --- a/.changes/0.1.0/Under the Hood-20240109-131958.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Configure `dependabot` -time: 2024-01-09T13:19:58.060742-05:00 -custom: - Author: mikealfare - Issue: "14" diff --git a/.changes/0.1.0/Under the Hood-20240112-230236.yaml b/.changes/0.1.0/Under the Hood-20240112-230236.yaml deleted file mode 100644 index 1470ac6e..00000000 --- a/.changes/0.1.0/Under the Hood-20240112-230236.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Implement unit testing in CI -time: 2024-01-12T23:02:36.630106-05:00 -custom: - Author: mikealfare - Issue: "10" diff --git a/.changes/0.1.0/Under the Hood-20240123-121220.yaml b/.changes/0.1.0/Under the Hood-20240123-121220.yaml deleted file mode 100644 index 8d01f256..00000000 --- a/.changes/0.1.0/Under the Hood-20240123-121220.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Allow version to be specified in either __version__.py or __about__.py -time: 2024-01-23T12:12:20.529147-05:00 -custom: - Author: mikealfare - Issue: "44" diff --git a/.changes/0.1.0/Under the Hood-20240220-164223.yaml b/.changes/0.1.0/Under the Hood-20240220-164223.yaml deleted file mode 100644 index eefa441e..00000000 --- a/.changes/0.1.0/Under the Hood-20240220-164223.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove __init__.py file from dbt.tests -time: 2024-02-20T16:42:23.706-05:00 -custom: - Author: gshank - Issue: "96" diff --git a/.changes/1.0.0.md b/.changes/1.0.0.md index d381d635..c46c8148 100644 --- a/.changes/1.0.0.md +++ b/.changes/1.0.0.md @@ -1,15 +1,32 @@ ## dbt-adapters 1.0.0 - April 01, 2024 +### Features + +* Update RelationConfig to capture all fields used by adapters ([#30](https://github.com/dbt-labs/dbt-adapters/issues/30)) + ### Fixes -* Add field wrapper to BaseRelation members that were missing it. -* Add "description" and "meta" fields to RelationConfig protocol +* Add field wrapper to BaseRelation members that were missing it. ([#108](https://github.com/dbt-labs/dbt-adapters/issues/108)) +* Add "description" and "meta" fields to RelationConfig protocol ([#119](https://github.com/dbt-labs/dbt-adapters/issues/119)) +* Ignore adapter-level support warnings for 'custom' constraints ([#90](https://github.com/dbt-labs/dbt-adapters/issues/90)) +* Make all adapter zone tests importable by removing "Test" prefix ([#93](https://github.com/dbt-labs/dbt-adapters/issues/93)) + +### Docs + +* Configure `changie` ([#16](https://github.com/dbt-labs/dbt-adapters/issues/16)) +* Setup ADR tracking framework ([#11](https://github.com/dbt-labs/dbt-adapters/issues/11)) +* Create issue templates ([#12](https://github.com/dbt-labs/dbt-adapters/issues/12)) +* Create PR template ([#13](https://github.com/dbt-labs/dbt-adapters/issues/13)) ### Under the Hood -* Lazy load agate to improve dbt-core performance -* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS +* Lazy load agate to improve dbt-core performance ([#125](https://github.com/dbt-labs/dbt-adapters/issues/125)) +* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS ([#131](https://github.com/dbt-labs/dbt-adapters/issues/131)) +* Configure `dependabot` ([#14](https://github.com/dbt-labs/dbt-adapters/issues/14)) +* Implement unit testing in CI ([#22](https://github.com/dbt-labs/dbt-adapters/issues/22)) +* Allow version to be specified in either __version__.py or __about__.py ([#44](https://github.com/dbt-labs/dbt-adapters/issues/44)) +* Remove __init__.py file from dbt.tests ([#96](https://github.com/dbt-labs/dbt-adapters/issues/96)) ### Security -* Pin `black>=24.3` in `pyproject.toml` +* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) diff --git a/.changes/1.1.0.md b/.changes/1.1.0.md index 9e7db78e..224d8e85 100644 --- a/.changes/1.1.0.md +++ b/.changes/1.1.0.md @@ -2,28 +2,28 @@ ### Features -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing -* Cross-database `cast` macro -* Allow adapters to opt out of aliasing the subquery generated by render_limited -* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing +* Debug log when `type_code` fails to convert to a `data_type` ([#135](https://github.com/dbt-labs/dbt-adapters/issues/135)) +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch ([#127](https://github.com/dbt-labs/dbt-adapters/issues/127)) +* Support for sql fixtures in unit testing ([#146](https://github.com/dbt-labs/dbt-adapters/issues/146)) +* Cross-database `cast` macro ([#173](https://github.com/dbt-labs/dbt-adapters/issues/173)) +* Allow adapters to opt out of aliasing the subquery generated by render_limited ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) ### Fixes -* Fix adapter-specific cast handling for constraint enforcement +* Fix adapter-specific cast handling for constraint enforcement ([#165](https://github.com/dbt-labs/dbt-adapters/issues/165)) ### Docs -* Use `dbt-adapters` throughout the contributing guide +* Use `dbt-adapters` throughout the contributing guide ([#137](https://github.com/dbt-labs/dbt-adapters/issues/137)) ### Under the Hood -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA -* Validate that dbt-core and dbt-adapters remain de-coupled -* remove dbt_version from query comment test fixture +* Add the option to set the log level of the AdapterRegistered event ([#141](https://github.com/dbt-labs/dbt-adapters/issues/141)) +* Update dependabot config to cover GHA ([#161](https://github.com/dbt-labs/dbt-adapters/issues/161)) +* Validate that dbt-core and dbt-adapters remain de-coupled ([#174](https://github.com/dbt-labs/dbt-adapters/issues/174)) +* remove dbt_version from query comment test fixture ([#184](https://github.com/dbt-labs/dbt-adapters/issues/184)) ### Dependencies -* add support for py3.12 +* add support for py3.12 ([#185](https://github.com/dbt-labs/dbt-adapters/issues/185)) diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md index a1c38c88..9e590f94 100644 --- a/.changes/1.1.1.md +++ b/.changes/1.1.1.md @@ -2,4 +2,4 @@ ### Features -* Enable serialization contexts +* Enable serialization contexts ([#197](https://github.com/dbt-labs/dbt-adapters/issues/197)) diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md index 2aac5674..e554b90b 100644 --- a/.changes/1.2.1.md +++ b/.changes/1.2.1.md @@ -2,9 +2,14 @@ ### Features -* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist ([#203](https://github.com/dbt-labs/dbt-adapters/issues/203)) +* Cross-database `date` macro ([#191](https://github.com/dbt-labs/dbt-adapters/issues/191)) + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes ([#194](https://github.com/dbt-labs/dbt-adapters/issues/194)) ### Under the Hood -* Add query recording for adapters which use SQLConnectionManager -* Improve memory efficiency of process_results() +* Add query recording for adapters which use SQLConnectionManager ([#195](https://github.com/dbt-labs/dbt-adapters/issues/195)) +* Improve memory efficiency of process_results() ([#217](https://github.com/dbt-labs/dbt-adapters/issues/217)) diff --git a/.changes/1.3.0.md b/.changes/1.3.0.md index dae1f819..6a23c3ba 100644 --- a/.changes/1.3.0.md +++ b/.changes/1.3.0.md @@ -2,4 +2,4 @@ ### Features -* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation ([#231](https://github.com/dbt-labs/dbt-adapters/issues/231)) diff --git a/.changes/1.3.2.md b/.changes/1.3.2.md index bdc7f593..6963a4c3 100644 --- a/.changes/1.3.2.md +++ b/.changes/1.3.2.md @@ -2,5 +2,5 @@ ### Under the Hood -* Fix query timer resolution -* Add optional release_connection parameter to connection_named method +* Fix query timer resolution ([#246](https://github.com/dbt-labs/dbt-adapters/issues/246)) +* Add optional release_connection parameter to connection_named method ([#247](https://github.com/dbt-labs/dbt-adapters/issues/247)) diff --git a/.changes/1.3.3.md b/.changes/1.3.3.md index 22ab18e3..c62a0562 100644 --- a/.changes/1.3.3.md +++ b/.changes/1.3.3.md @@ -2,8 +2,8 @@ ### Fixes -* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#213](https://github.com/dbt-labs/dbt-adapters/issues/213)) ### Under the Hood -* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. ([#207](https://github.com/dbt-labs/dbt-adapters/issues/207)) diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md deleted file mode 100644 index e781a985..00000000 --- a/.changes/1.8.0.md +++ /dev/null @@ -1,9 +0,0 @@ -## dbt-adapters 1.8.0 - May 09, 2024 - -### Features - -* Cross-database `date` macro - -### Fixes - -* Update Clone test to reflect core change removing `deferred` attribute from nodes diff --git a/.changes/1.9.0.md b/.changes/1.9.0.md deleted file mode 100644 index 2f016563..00000000 --- a/.changes/1.9.0.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapters 1.9.0 - June 18, 2024 diff --git a/.changes/1.9.1.md b/.changes/1.9.1.md deleted file mode 100644 index 900e6b75..00000000 --- a/.changes/1.9.1.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapters 1.9.1 - June 20, 2024 diff --git a/.changes/1.9.2.md b/.changes/1.9.2.md deleted file mode 100644 index 26505f20..00000000 --- a/.changes/1.9.2.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapters 1.9.2 - July 09, 2024 diff --git a/.changie.yaml b/.changie.yaml index afbafb22..1198cfe0 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -1,27 +1,70 @@ changesDir: .changes unreleasedDir: unreleased headerPath: header.tpl.md +versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md envPrefix: CHANGIE_ versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' -changeFormat: '* {{.Body}}' +changeFormat: |- + {{- $IssueList := list }} + {{- $changes := splitList " " $.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) kinds: - - label: Breaking Changes - - label: Features - - label: Fixes - - label: Docs - - label: Under the Hood - - label: Dependencies - - label: Security +- label: Breaking Changes +- label: Features +- label: Fixes +- label: Docs +- label: Under the Hood +- label: Dependencies + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 +- label: Security + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 newlines: afterChangelogHeader: 1 afterKind: 1 afterChangelogVersion: 1 beforeKind: 1 endOfVersion: 1 - custom: - key: Author label: GitHub Username(s) (separated by a single space if multiple) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a2f1833..8bd48b5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,38 +5,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapters 1.9.2 - July 09, 2024 - -## dbt-adapters 1.9.1 - June 20, 2024 - -## dbt-adapters 1.9.0 - June 18, 2024 - -## dbt-adapters 1.8.0 - May 09, 2024 - -### Features - -* Cross-database `date` macro - -### Fixes - -* Update Clone test to reflect core change removing `deferred` attribute from nodes - ## dbt-adapters 1.3.3 - July 09, 2024 ### Fixes -* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#213](https://github.com/dbt-labs/dbt-adapters/issues/213)) ### Under the Hood -* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. ([#207](https://github.com/dbt-labs/dbt-adapters/issues/207)) ## dbt-adapters 1.3.2 - July 02, 2024 ### Under the Hood -* Fix query timer resolution -* Add optional release_connection parameter to connection_named method +* Fix query timer resolution ([#246](https://github.com/dbt-labs/dbt-adapters/issues/246)) +* Add optional release_connection parameter to connection_named method ([#247](https://github.com/dbt-labs/dbt-adapters/issues/247)) ## dbt-adapters 1.3.1 - June 20, 2024 @@ -44,67 +28,89 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ### Features -* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation ([#231](https://github.com/dbt-labs/dbt-adapters/issues/231)) ## dbt-adapters 1.2.1 - May 21, 2024 ### Features -* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist ([#203](https://github.com/dbt-labs/dbt-adapters/issues/203)) +* Cross-database `date` macro ([#191](https://github.com/dbt-labs/dbt-adapters/issues/191)) + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes ([#194](https://github.com/dbt-labs/dbt-adapters/issues/194)) ### Under the Hood -* Add query recording for adapters which use SQLConnectionManager -* Improve memory efficiency of process_results() +* Add query recording for adapters which use SQLConnectionManager ([#195](https://github.com/dbt-labs/dbt-adapters/issues/195)) +* Improve memory efficiency of process_results() ([#217](https://github.com/dbt-labs/dbt-adapters/issues/217)) ## dbt-adapters 1.1.1 - May 07, 2024 ### Features -* Enable serialization contexts +* Enable serialization contexts ([#197](https://github.com/dbt-labs/dbt-adapters/issues/197)) ## dbt-adapters 1.1.0 - May 01, 2024 ### Features -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing -* Cross-database `cast` macro -* Allow adapters to opt out of aliasing the subquery generated by render_limited -* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing +* Debug log when `type_code` fails to convert to a `data_type` ([#135](https://github.com/dbt-labs/dbt-adapters/issues/135)) +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch ([#127](https://github.com/dbt-labs/dbt-adapters/issues/127)) +* Support for sql fixtures in unit testing ([#146](https://github.com/dbt-labs/dbt-adapters/issues/146)) +* Cross-database `cast` macro ([#173](https://github.com/dbt-labs/dbt-adapters/issues/173)) +* Allow adapters to opt out of aliasing the subquery generated by render_limited ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) ### Fixes -* Fix adapter-specific cast handling for constraint enforcement +* Fix adapter-specific cast handling for constraint enforcement ([#165](https://github.com/dbt-labs/dbt-adapters/issues/165)) ### Docs -* Use `dbt-adapters` throughout the contributing guide +* Use `dbt-adapters` throughout the contributing guide ([#137](https://github.com/dbt-labs/dbt-adapters/issues/137)) ### Under the Hood -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA -* Validate that dbt-core and dbt-adapters remain de-coupled -* remove dbt_version from query comment test fixture +* Add the option to set the log level of the AdapterRegistered event ([#141](https://github.com/dbt-labs/dbt-adapters/issues/141)) +* Update dependabot config to cover GHA ([#161](https://github.com/dbt-labs/dbt-adapters/issues/161)) +* Validate that dbt-core and dbt-adapters remain de-coupled ([#174](https://github.com/dbt-labs/dbt-adapters/issues/174)) +* remove dbt_version from query comment test fixture ([#184](https://github.com/dbt-labs/dbt-adapters/issues/184)) ### Dependencies -* add support for py3.12 +* add support for py3.12 ([#185](https://github.com/dbt-labs/dbt-adapters/issues/185)) ## dbt-adapters 1.0.0 - April 01, 2024 +### Features + +* Update RelationConfig to capture all fields used by adapters ([#30](https://github.com/dbt-labs/dbt-adapters/issues/30)) + ### Fixes -* Add field wrapper to BaseRelation members that were missing it. -* Add "description" and "meta" fields to RelationConfig protocol +* Add field wrapper to BaseRelation members that were missing it. ([#108](https://github.com/dbt-labs/dbt-adapters/issues/108)) +* Add "description" and "meta" fields to RelationConfig protocol ([#119](https://github.com/dbt-labs/dbt-adapters/issues/119)) +* Ignore adapter-level support warnings for 'custom' constraints ([#90](https://github.com/dbt-labs/dbt-adapters/issues/90)) +* Make all adapter zone tests importable by removing "Test" prefix ([#93](https://github.com/dbt-labs/dbt-adapters/issues/93)) + +### Docs + +* Configure `changie` ([#16](https://github.com/dbt-labs/dbt-adapters/issues/16)) +* Setup ADR tracking framework ([#11](https://github.com/dbt-labs/dbt-adapters/issues/11)) +* Create issue templates ([#12](https://github.com/dbt-labs/dbt-adapters/issues/12)) +* Create PR template ([#13](https://github.com/dbt-labs/dbt-adapters/issues/13)) ### Under the Hood -* Lazy load agate to improve dbt-core performance -* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS +* Lazy load agate to improve dbt-core performance ([#125](https://github.com/dbt-labs/dbt-adapters/issues/125)) +* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS ([#131](https://github.com/dbt-labs/dbt-adapters/issues/131)) +* Configure `dependabot` ([#14](https://github.com/dbt-labs/dbt-adapters/issues/14)) +* Implement unit testing in CI ([#22](https://github.com/dbt-labs/dbt-adapters/issues/22)) +* Allow version to be specified in either __version__.py or __about__.py ([#44](https://github.com/dbt-labs/dbt-adapters/issues/44)) +* Remove __init__.py file from dbt.tests ([#96](https://github.com/dbt-labs/dbt-adapters/issues/96)) ### Security -* Pin `black>=24.3` in `pyproject.toml` +* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) From b088e518477ad7023c014eec5d9fad36a273ef86 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:02:25 -0700 Subject: [PATCH 56/66] Adap 821/support for redshift 821 (#270) Co-authored-by: Mila Page <versusfacit@users.noreply.github.com> --- .../macros/unit_test_sql/get_fixture_sql.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index c25a87f7..a3a8173b 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -22,6 +22,7 @@ {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} {%- endfor -%} +{{ validate_fixture_rows(rows, row_number) }} {%- for row in rows -%} {%- set formatted_row = format_row(row, column_name_to_data_types) -%} @@ -93,3 +94,11 @@ union all {%- endfor -%} {{ return(formatted_row) }} {%- endmacro -%} + +{%- macro validate_fixture_rows(rows, row_number) -%} + {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }} +{%- endmacro -%} + +{%- macro default__validate_fixture_rows(rows, row_number) -%} + {# This is an abstract method for adapter overrides as needed #} +{%- endmacro -%} From d23b9e8dc9fc236ce9eae5519c39b19cf758f276 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:11:56 -0400 Subject: [PATCH 57/66] Make `dependabot` ignore patch updates (#267) --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 02f010c7..907926a3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,13 +5,25 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "pip" directory: "/dbt-tests-adapter" schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch From fbfa095c2f3c567b636de601b099f792021d0144 Mon Sep 17 00:00:00 2001 From: Michelle Ark <MichelleArk@users.noreply.github.com> Date: Thu, 25 Jul 2024 10:56:28 -0400 Subject: [PATCH 58/66] render to and to_columns on column and model level fk constraints (#260) --- .../unreleased/Features-20240723-104221.yaml | 6 ++++ .../dbt/tests/adapter/constraints/fixtures.py | 3 +- dbt/adapters/base/impl.py | 34 +++++++++++++------ tests/unit/test_base_adapter.py | 32 +++++++++++++++++ 4 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 .changes/unreleased/Features-20240723-104221.yaml diff --git a/.changes/unreleased/Features-20240723-104221.yaml b/.changes/unreleased/Features-20240723-104221.yaml new file mode 100644 index 00000000..cc038fd4 --- /dev/null +++ b/.changes/unreleased/Features-20240723-104221.yaml @@ -0,0 +1,6 @@ +kind: Features +body: render 'to' and 'to_columns' fields on foreign key constraints +time: 2024-07-23T10:42:21.222203-04:00 +custom: + Author: michelleark + Issue: "271" diff --git a/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py index cfbd5379..2a4f089b 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py @@ -363,7 +363,8 @@ - type: check expression: id >= 1 - type: foreign_key - expression: {schema}.foreign_key_model (id) + to: ref('foreign_key_model') + to_columns: ["id"] - type: unique data_tests: - unique diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index ec590137..e0627c47 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1602,8 +1602,13 @@ def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional rendered_column_constraint = f"unique {constraint_expression}" elif constraint.type == ConstraintType.primary_key: rendered_column_constraint = f"primary key {constraint_expression}" - elif constraint.type == ConstraintType.foreign_key and constraint_expression: - rendered_column_constraint = f"references {constraint_expression}" + elif constraint.type == ConstraintType.foreign_key: + if constraint.to and constraint.to_columns: + rendered_column_constraint = ( + f"references {constraint.to} ({', '.join(constraint.to_columns)})" + ) + elif constraint_expression: + rendered_column_constraint = f"references {constraint_expression}" elif constraint.type == ConstraintType.custom and constraint_expression: rendered_column_constraint = constraint_expression @@ -1682,20 +1687,29 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s rendering.""" constraint_prefix = f"constraint {constraint.name} " if constraint.name else "" column_list = ", ".join(constraint.columns) + rendered_model_constraint = None + if constraint.type == ConstraintType.check and constraint.expression: - return f"{constraint_prefix}check ({constraint.expression})" + rendered_model_constraint = f"{constraint_prefix}check ({constraint.expression})" elif constraint.type == ConstraintType.unique: constraint_expression = f" {constraint.expression}" if constraint.expression else "" - return f"{constraint_prefix}unique{constraint_expression} ({column_list})" + rendered_model_constraint = ( + f"{constraint_prefix}unique{constraint_expression} ({column_list})" + ) elif constraint.type == ConstraintType.primary_key: constraint_expression = f" {constraint.expression}" if constraint.expression else "" - return f"{constraint_prefix}primary key{constraint_expression} ({column_list})" - elif constraint.type == ConstraintType.foreign_key and constraint.expression: - return f"{constraint_prefix}foreign key ({column_list}) references {constraint.expression}" + rendered_model_constraint = ( + f"{constraint_prefix}primary key{constraint_expression} ({column_list})" + ) + elif constraint.type == ConstraintType.foreign_key: + if constraint.to and constraint.to_columns: + rendered_model_constraint = f"{constraint_prefix}foreign key ({column_list}) references {constraint.to} ({', '.join(constraint.to_columns)})" + elif constraint.expression: + rendered_model_constraint = f"{constraint_prefix}foreign key ({column_list}) references {constraint.expression}" elif constraint.type == ConstraintType.custom and constraint.expression: - return f"{constraint_prefix}{constraint.expression}" - else: - return None + rendered_model_constraint = f"{constraint_prefix}{constraint.expression}" + + return rendered_model_constraint @classmethod def capabilities(cls) -> CapabilityDict: diff --git a/tests/unit/test_base_adapter.py b/tests/unit/test_base_adapter.py index 95fe5ae2..5fa109b7 100644 --- a/tests/unit/test_base_adapter.py +++ b/tests/unit/test_base_adapter.py @@ -39,6 +39,14 @@ def connection_manager(self): [{"type": "foreign_key", "expression": "other_table (c1)"}], ["column_name integer references other_table (c1)"], ), + ( + [{"type": "foreign_key", "to": "other_table", "to_columns": ["c1"]}], + ["column_name integer references other_table (c1)"], + ), + ( + [{"type": "foreign_key", "to": "other_table", "to_columns": ["c1", "c2"]}], + ["column_name integer references other_table (c1, c2)"], + ), ([{"type": "check"}, {"type": "unique"}], ["column_name integer unique"]), ([{"type": "custom", "expression": "-- noop"}], ["column_name integer -- noop"]), ] @@ -176,6 +184,30 @@ def test_render_raw_columns_constraints_unsupported( ], ["constraint test_name foreign key (c1, c2) references other_table (c1)"], ), + ( + [ + { + "type": "foreign_key", + "columns": ["c1", "c2"], + "to": "other_table", + "to_columns": ["c1"], + "name": "test_name", + } + ], + ["constraint test_name foreign key (c1, c2) references other_table (c1)"], + ), + ( + [ + { + "type": "foreign_key", + "columns": ["c1", "c2"], + "to": "other_table", + "to_columns": ["c1", "c2"], + "name": "test_name", + } + ], + ["constraint test_name foreign key (c1, c2) references other_table (c1, c2)"], + ), ] @pytest.mark.parametrize("constraints,expected_rendered_constraints", model_constraints) From 4b2755dd129c54f04c3d467cd7e83a46d0afc01a Mon Sep 17 00:00:00 2001 From: Michelle Ark <MichelleArk@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:36:29 -0400 Subject: [PATCH 59/66] bump dbt-common min to 1.6 (#272) --- .changes/unreleased/Features-20240723-104221.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/unreleased/Features-20240723-104221.yaml b/.changes/unreleased/Features-20240723-104221.yaml index cc038fd4..45499971 100644 --- a/.changes/unreleased/Features-20240723-104221.yaml +++ b/.changes/unreleased/Features-20240723-104221.yaml @@ -1,5 +1,5 @@ kind: Features -body: render 'to' and 'to_columns' fields on foreign key constraints +body: render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 time: 2024-07-23T10:42:21.222203-04:00 custom: Author: michelleark diff --git a/pyproject.toml b/pyproject.toml index 55529f34..e794781c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common>=1.3,<2.0", + "dbt-common>=1.6,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", From b64d3a5efd83b8a3ab0612145f6bdeaa8cb0f2ba Mon Sep 17 00:00:00 2001 From: Gerda Shank <gerda.shank@gmail.com> Date: Mon, 29 Jul 2024 16:29:59 -0400 Subject: [PATCH 60/66] When using incremental tables, the column data types are invalidly updated (#269) --- .changes/unreleased/Fixes-20240728-221030.yaml | 6 ++++++ dbt/adapters/base/column.py | 3 --- .../materializations/models/incremental/incremental.sql | 9 ++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240728-221030.yaml diff --git a/.changes/unreleased/Fixes-20240728-221030.yaml b/.changes/unreleased/Fixes-20240728-221030.yaml new file mode 100644 index 00000000..a1cddb24 --- /dev/null +++ b/.changes/unreleased/Fixes-20240728-221030.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Incremental table varchar column definition changed +time: 2024-07-28T22:10:30.673107-04:00 +custom: + Author: gshank + Issue: "276" diff --git a/dbt/adapters/base/column.py b/dbt/adapters/base/column.py index e2e6e1e0..195684a4 100644 --- a/dbt/adapters/base/column.py +++ b/dbt/adapters/base/column.py @@ -123,9 +123,6 @@ def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str: else: return "{}({},{})".format(dtype, precision, scale) - def __repr__(self) -> str: - return "<Column {} ({})>".format(self.name, self.data_type) - @classmethod def from_description(cls, name: str, raw_data_type: str) -> "Column": match = re.match(r"([^(]+)(\([^)]+\))?", raw_data_type) diff --git a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql index e8ff5c1e..f932751a 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql @@ -39,9 +39,12 @@ {% set need_swap = true %} {% else %} {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} - {% do adapter.expand_target_column_types( - from_relation=temp_relation, - to_relation=target_relation) %} + {% set contract_config = config.get('contract') %} + {% if not contract_config or not contract_config.enforced %} + {% do adapter.expand_target_column_types( + from_relation=temp_relation, + to_relation=target_relation) %} + {% endif %} {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} {% if not dest_columns %} From 1799db9bf49414be995abd000e2bca3e9bfd8064 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 30 Jul 2024 16:29:05 +0000 Subject: [PATCH 61/66] Bumping version to 1.4.0 and generate changelog --- .changes/1.4.0.md | 13 +++++++++++++ .changes/unreleased/Features-20240723-104221.yaml | 6 ------ .changes/unreleased/Fixes-20240728-221030.yaml | 6 ------ .../unreleased/Under the Hood-20240625-110833.yaml | 6 ------ CHANGELOG.md | 14 ++++++++++++++ dbt/adapters/__about__.py | 2 +- 6 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 .changes/1.4.0.md delete mode 100644 .changes/unreleased/Features-20240723-104221.yaml delete mode 100644 .changes/unreleased/Fixes-20240728-221030.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240625-110833.yaml diff --git a/.changes/1.4.0.md b/.changes/1.4.0.md new file mode 100644 index 00000000..fc6279db --- /dev/null +++ b/.changes/1.4.0.md @@ -0,0 +1,13 @@ +## dbt-adapters 1.4.0 - July 30, 2024 + +### Features + +- render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 ([#271](https://github.com/dbt-labs/dbt-adapters/issues/271)) + +### Fixes + +- Incremental table varchar column definition changed ([#276](https://github.com/dbt-labs/dbt-adapters/issues/276)) + +### Under the Hood + +- Rework record/replay to record at the database connection level. ([#244](https://github.com/dbt-labs/dbt-adapters/issues/244)) diff --git a/.changes/unreleased/Features-20240723-104221.yaml b/.changes/unreleased/Features-20240723-104221.yaml deleted file mode 100644 index 45499971..00000000 --- a/.changes/unreleased/Features-20240723-104221.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 -time: 2024-07-23T10:42:21.222203-04:00 -custom: - Author: michelleark - Issue: "271" diff --git a/.changes/unreleased/Fixes-20240728-221030.yaml b/.changes/unreleased/Fixes-20240728-221030.yaml deleted file mode 100644 index a1cddb24..00000000 --- a/.changes/unreleased/Fixes-20240728-221030.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Incremental table varchar column definition changed -time: 2024-07-28T22:10:30.673107-04:00 -custom: - Author: gshank - Issue: "276" diff --git a/.changes/unreleased/Under the Hood-20240625-110833.yaml b/.changes/unreleased/Under the Hood-20240625-110833.yaml deleted file mode 100644 index c4486a8f..00000000 --- a/.changes/unreleased/Under the Hood-20240625-110833.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Rework record/replay to record at the database connection level. -time: 2024-06-25T11:08:33.264457-04:00 -custom: - Author: peteralllenwebb - Issue: "244" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bd48b5d..4146e95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.4.0 - July 30, 2024 + +### Features + +- render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 ([#271](https://github.com/dbt-labs/dbt-adapters/issues/271)) + +### Fixes + +- Incremental table varchar column definition changed ([#276](https://github.com/dbt-labs/dbt-adapters/issues/276)) + +### Under the Hood + +- Rework record/replay to record at the database connection level. ([#244](https://github.com/dbt-labs/dbt-adapters/issues/244)) + ## dbt-adapters 1.3.3 - July 09, 2024 ### Fixes diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index cfa2db68..d619c757 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.3" +version = "1.4.0" From 2c4065b10e7aab67c342d6a561411465f9ba6beb Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Tue, 30 Jul 2024 13:43:46 -0500 Subject: [PATCH 62/66] upgrade upload action to v4 in all reference points (#277) --- .github/actions/build-hatch/action.yml | 2 +- .github/actions/publish-results/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml index f2da5935..6d81339a 100644 --- a/.github/actions/build-hatch/action.yml +++ b/.github/actions/build-hatch/action.yml @@ -31,7 +31,7 @@ runs: - name: Upload artifacts if: ${{ inputs.archive-name != '' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ inputs.archive-name }} path: ${{ inputs.working-dir }}dist/ diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml index d863d659..0d5cb7e6 100644 --- a/.github/actions/publish-results/action.yml +++ b/.github/actions/publish-results/action.yml @@ -19,7 +19,7 @@ runs: run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts shell: bash - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.file-name }}_python-${{ inputs.python-version }}_${{ steps.timestamp.outputs.ts }}.csv path: ${{ inputs.source-file }} From ced2f98847c7f53f05f0f0486c51329efd64c73a Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:37:27 -0400 Subject: [PATCH 63/66] Update `setup-python` action to `v5` to move off of a deprecated version of Node (`node16`) (#278) --- .github/actions/setup-hatch/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml index 6b15cdbf..6bf8ea10 100644 --- a/.github/actions/setup-hatch/action.yml +++ b/.github/actions/setup-hatch/action.yml @@ -13,7 +13,7 @@ runs: using: composite steps: - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} From 7935b1380555002c03dbd92bf79475523eef55c6 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 1 Aug 2024 00:40:14 -0400 Subject: [PATCH 64/66] Update `download-artifact` to `v4` (#279) --- .github/actions/publish-pypi/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index deffc6e3..25bc3a8d 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -14,7 +14,7 @@ runs: steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive-name }} path: dist/ From 2bdf580ba269cffb8a3331f626c86f7b0116e51c Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:55:06 -0400 Subject: [PATCH 65/66] Updating changie.yaml to add contributors and PR links (#280) --- .../Under the Hood-20240801-220551.yaml | 6 ++ .changie.yaml | 92 +++++++++++++++---- 2 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240801-220551.yaml diff --git a/.changes/unreleased/Under the Hood-20240801-220551.yaml b/.changes/unreleased/Under the Hood-20240801-220551.yaml new file mode 100644 index 00000000..25b54a65 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240801-220551.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Updating changie.yaml to add contributors and PR links +time: 2024-08-01T22:05:51.327652-04:00 +custom: + Author: leahwicz + Issue: "219" diff --git a/.changie.yaml b/.changie.yaml index 1198cfe0..8f1d8615 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -4,7 +4,7 @@ headerPath: header.tpl.md versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md -envPrefix: CHANGIE_ +envPrefix: "CHANGIE_" versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' changeFormat: |- @@ -15,11 +15,11 @@ changeFormat: |- {{- $IssueList = append $IssueList $changeLink }} {{- end -}} - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) + kinds: - label: Breaking Changes - label: Features - label: Fixes -- label: Docs - label: Under the Hood - label: Dependencies changeFormat: |- @@ -32,14 +32,14 @@ kinds: - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) skipGlobalChoices: true additionalChoices: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: PR - label: GitHub Pull Request Number (separated by a single space if multiple) - type: string - minLength: 1 + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 - label: Security changeFormat: |- {{- $PRList := list }} @@ -51,20 +51,22 @@ kinds: - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) skipGlobalChoices: true additionalChoices: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: PR - label: GitHub Pull Request Number (separated by a single space if multiple) - type: string - minLength: 1 + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 + newlines: afterChangelogHeader: 1 afterKind: 1 afterChangelogVersion: 1 beforeKind: 1 endOfVersion: 1 + custom: - key: Author label: GitHub Username(s) (separated by a single space if multiple) @@ -74,3 +76,57 @@ custom: label: GitHub Issue Number (separated by a single space if multiple) type: string minLength: 1 + + +footerFormat: | + {{- $contributorDict := dict }} + {{- /* ensure all names in this list are all lowercase for later matching purposes */}} + {{- $core_team := splitList " " .Env.CORE_TEAM }} + {{- /* ensure we always skip snyk and dependabot in addition to the core team */}} + {{- $maintainers := list "dependabot[bot]" "snyk-bot"}} + {{- range $team_member := $core_team }} + {{- $team_member_lower := lower $team_member }} + {{- $maintainers = append $maintainers $team_member_lower }} + {{- end }} + {{- range $change := .Changes }} + {{- $authorList := splitList " " $change.Custom.Author }} + {{- /* loop through all authors for a single changelog */}} + {{- range $author := $authorList }} + {{- $authorLower := lower $author }} + {{- /* we only want to include non-core team contributors */}} + {{- if not (has $authorLower $maintainers)}} + {{- $changeList := splitList " " $change.Custom.Author }} + {{- $IssueList := list }} + {{- $changeLink := $change.Kind }} + {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }} + {{- $changes := splitList " " $change.Custom.PR }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- else }} + {{- $changes := splitList " " $change.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- end }} + {{- /* check if this contributor has other changes associated with them already */}} + {{- if hasKey $contributorDict $author }} + {{- $contributionList := get $contributorDict $author }} + {{- $contributionList = concat $contributionList $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- else }} + {{- $contributionList := $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- end }} + {{- end}} + {{- end}} + {{- end }} + {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} + {{- if $contributorDict}} + ### Contributors + {{- range $k,$v := $contributorDict }} + - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) + {{- end }} + {{- end }} From 313d4a506984b5c024d51a110eb9beaeef58fcfa Mon Sep 17 00:00:00 2001 From: Jean Cochrane <jeancochrane@users.noreply.github.com> Date: Thu, 8 Aug 2024 19:13:08 -0500 Subject: [PATCH 66/66] Use model alias for the CTE identifier generated during ephemeral materialization (#236) --- .changes/unreleased/Fixes-20240610-195300.yaml | 6 ++++++ dbt/adapters/base/relation.py | 7 +++++-- tests/unit/test_relation.py | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240610-195300.yaml diff --git a/.changes/unreleased/Fixes-20240610-195300.yaml b/.changes/unreleased/Fixes-20240610-195300.yaml new file mode 100644 index 00000000..1f8cd5a5 --- /dev/null +++ b/.changes/unreleased/Fixes-20240610-195300.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use model alias for the CTE identifier generated during ephemeral materialization +time: 2024-06-10T19:53:00.086488231Z +custom: + Author: jeancochrane + Issue: "5273" diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 210a2dcd..1aab7b2f 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -241,8 +241,11 @@ def create_ephemeral_from( relation_config: RelationConfig, limit: Optional[int] = None, ) -> Self: - # Note that ephemeral models are based on the name. - identifier = cls.add_ephemeral_prefix(relation_config.name) + # Note that ephemeral models are based on the identifier, which will + # point to the model's alias if one exists and otherwise fall back to + # the filename. This is intended to give the user more control over + # the way that the CTE name is constructed + identifier = cls.add_ephemeral_prefix(relation_config.identifier) return cls.create( type=cls.CTE, identifier=identifier, diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py index a1c01c5c..97d56419 100644 --- a/tests/unit/test_relation.py +++ b/tests/unit/test_relation.py @@ -1,4 +1,4 @@ -from dataclasses import replace +from dataclasses import dataclass, replace import pytest @@ -79,3 +79,16 @@ def test_render_limited(limit, require_alias, expected_result): actual_result = my_relation.render_limited() assert actual_result == expected_result assert str(my_relation) == expected_result + + +def test_create_ephemeral_from_uses_identifier(): + @dataclass + class Node: + """Dummy implementation of RelationConfig protocol""" + + name: str + identifier: str + + node = Node(name="name_should_not_be_used", identifier="test") + ephemeral_relation = BaseRelation.create_ephemeral_from(node) + assert str(ephemeral_relation) == "__dbt__cte__test"