From 2e7d04c266f2d4f2f2e73b61244ea78221c5668c Mon Sep 17 00:00:00 2001 From: Isaac Blanco <40709873+AlejandroBlanco2001@users.noreply.github.com> Date: Tue, 14 May 2024 22:28:52 -0500 Subject: [PATCH 01/38] =?UTF-8?q?Improving=20message=20description=20for?= =?UTF-8?q?=20compile=20error=20in=20get=5Ffixtu=E2=80=A6=20(#203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changes/unreleased/Features-20240511-032253.yaml | 7 +++++++ .../macros/unit_test_sql/get_fixture_sql.sql | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Features-20240511-032253.yaml diff --git a/.changes/unreleased/Features-20240511-032253.yaml b/.changes/unreleased/Features-20240511-032253.yaml new file mode 100644 index 00000000..b81ff9c0 --- /dev/null +++ b/.changes/unreleased/Features-20240511-032253.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Improvement of the compile error message in the get_fixture-sql.sql when the + relation or the model not exist +time: 2024-05-11T03:22:53.40817477-05:00 +custom: + Author: AlejandroBlanco2001 + Issue: "10014" diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index 53d7a93b..c25a87f7 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -15,7 +15,7 @@ {%- endif -%} {%- if not column_name_to_data_types -%} - {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this) }} + {{ exceptions.raise_compiler_error("Not able to get columns for unit test '" ~ model.name ~ "' from relation " ~ this ~ " because the relation doesn't exist") }} {%- endif -%} {%- for column_name, column_type in column_name_to_data_types.items() -%} From eda109542efd1703fd08e242006ae4d3fc476b38 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 15 May 2024 16:15:42 -0400 Subject: [PATCH 02/38] Add docstring for "catch all" relation type of `External` (#215) --- dbt/adapters/contracts/relation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/contracts/relation.py b/dbt/adapters/contracts/relation.py index 3560c2b3..42beb579 100644 --- a/dbt/adapters/contracts/relation.py +++ b/dbt/adapters/contracts/relation.py @@ -18,8 +18,9 @@ class RelationType(StrEnum): View = "view" CTE = "cte" MaterializedView = "materialized_view" - External = "external" Ephemeral = "ephemeral" + # this is a "catch all" that is better than `None` == external to anything dbt is aware of + External = "external" class MaterializationContract(Protocol): From fd33aafe8276051b313a3c89557b50d224bc50ed Mon Sep 17 00:00:00 2001 From: Peter Webb Date: Thu, 16 May 2024 17:27:08 -0400 Subject: [PATCH 03/38] Improve memory efficiency of process_results by iterating. (#217) --- .../Under the Hood-20240516-105757.yaml | 6 ++++++ dbt/adapters/sql/connections.py | 15 ++++++--------- tests/unit/test_sql_result.py | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240516-105757.yaml diff --git a/.changes/unreleased/Under the Hood-20240516-105757.yaml b/.changes/unreleased/Under the Hood-20240516-105757.yaml new file mode 100644 index 00000000..a5a47c8c --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240516-105757.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Improve memory efficiency of process_results() +time: 2024-05-16T10:57:57.480672-04:00 +custom: + Author: peterallenwebb + Issue: "217" diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index 9adaafce..d8699fd3 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -1,6 +1,6 @@ import abc import time -from typing import Any, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, TYPE_CHECKING from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event @@ -112,27 +112,24 @@ def get_response(cls, cursor: Any) -> AdapterResponse: @classmethod def process_results( cls, column_names: Iterable[str], rows: Iterable[Any] - ) -> List[Dict[str, Any]]: - # TODO CT-211 + ) -> Iterator[Dict[str, Any]]: unique_col_names = dict() # type: ignore[var-annotated] - # TODO CT-211 for idx in range(len(column_names)): # type: ignore[arg-type] - # TODO CT-211 col_name = column_names[idx] # type: ignore[index] if col_name in unique_col_names: unique_col_names[col_name] += 1 - # TODO CT-211 column_names[idx] = f"{col_name}_{unique_col_names[col_name]}" # type: ignore[index] # noqa else: - # TODO CT-211 unique_col_names[column_names[idx]] = 1 # type: ignore[index] - return [dict(zip(column_names, row)) for row in rows] + + for row in rows: + yield dict(zip(column_names, row)) @classmethod def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Table": from dbt_common.clients.agate_helper import table_from_data_flat - data: List[Any] = [] + data: Iterable[Any] = [] column_names: List[str] = [] if cursor.description is not None: diff --git a/tests/unit/test_sql_result.py b/tests/unit/test_sql_result.py index 12c173cb..454e6572 100644 --- a/tests/unit/test_sql_result.py +++ b/tests/unit/test_sql_result.py @@ -8,13 +8,13 @@ def test_duplicated_columns(self): cols_with_one_dupe = ["a", "b", "a", "d"] rows = [(1, 2, 3, 4)] self.assertEqual( - SQLConnectionManager.process_results(cols_with_one_dupe, rows), + list(SQLConnectionManager.process_results(cols_with_one_dupe, rows)), [{"a": 1, "b": 2, "a_2": 3, "d": 4}], ) cols_with_more_dupes = ["a", "a", "a", "b"] rows = [(1, 2, 3, 4)] self.assertEqual( - SQLConnectionManager.process_results(cols_with_more_dupes, rows), + list(SQLConnectionManager.process_results(cols_with_more_dupes, rows)), [{"a": 1, "a_2": 2, "a_3": 3, "b": 4}], ) From 95e5c77e8a27ac964599cbeeddd0c58b13fbf390 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 21 May 2024 22:42:11 +0000 Subject: [PATCH 04/38] Bumping version to 1.2.1 and generate changelog --- .changes/1.2.1.md | 10 ++++++++++ .changes/unreleased/Features-20240511-032253.yaml | 7 ------- .../unreleased/Under the Hood-20240503-162655.yaml | 6 ------ .../unreleased/Under the Hood-20240516-105757.yaml | 6 ------ CHANGELOG.md | 11 +++++++++++ dbt/adapters/__about__.py | 2 +- 6 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 .changes/1.2.1.md delete mode 100644 .changes/unreleased/Features-20240511-032253.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240503-162655.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240516-105757.yaml diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md new file mode 100644 index 00000000..f838b769 --- /dev/null +++ b/.changes/1.2.1.md @@ -0,0 +1,10 @@ +## dbt-adapter 1.2.1 - May 21, 2024 + +### Features + +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist + +### Under the Hood + +* Add query recording for adapters which use SQLConnectionManager +* Improve memory efficiency of process_results() diff --git a/.changes/unreleased/Features-20240511-032253.yaml b/.changes/unreleased/Features-20240511-032253.yaml deleted file mode 100644 index b81ff9c0..00000000 --- a/.changes/unreleased/Features-20240511-032253.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: Improvement of the compile error message in the get_fixture-sql.sql when the - relation or the model not exist -time: 2024-05-11T03:22:53.40817477-05:00 -custom: - Author: AlejandroBlanco2001 - Issue: "10014" diff --git a/.changes/unreleased/Under the Hood-20240503-162655.yaml b/.changes/unreleased/Under the Hood-20240503-162655.yaml deleted file mode 100644 index 1d2946f0..00000000 --- a/.changes/unreleased/Under the Hood-20240503-162655.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add query recording for adapters which use SQLConnectionManager -time: 2024-05-03T16:26:55.350916-04:00 -custom: - Author: peterallenwebb - Issue: "195" diff --git a/.changes/unreleased/Under the Hood-20240516-105757.yaml b/.changes/unreleased/Under the Hood-20240516-105757.yaml deleted file mode 100644 index a5a47c8c..00000000 --- a/.changes/unreleased/Under the Hood-20240516-105757.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Improve memory efficiency of process_results() -time: 2024-05-16T10:57:57.480672-04:00 -custom: - Author: peterallenwebb - Issue: "217" diff --git a/CHANGELOG.md b/CHANGELOG.md index 76b1928b..aaa477cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,17 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapter 1.2.1 - May 21, 2024 + +### Features + +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist + +### Under the Hood + +* Add query recording for adapters which use SQLConnectionManager +* Improve memory efficiency of process_results() + ## dbt-adapter 1.1.1 - May 07, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 64c8c21b..eb1d9a0f 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.1.1" +version = "1.2.1" From 8ff69435a753f0a94dd1274d0e8dad199805cd13 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:50:44 -0600 Subject: [PATCH 05/38] Update `dbt-adapter` to `dbt-adapters` (#235) --- .changes/1.0.0.md | 2 +- .changes/1.1.0.md | 2 +- .changes/1.1.1.md | 2 +- .changes/1.2.1.md | 2 +- .changes/1.8.0.md | 2 +- .changie.yaml | 2 +- .github/ISSUE_TEMPLATE/bug-report.yml | 6 +++--- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- .github/ISSUE_TEMPLATE/feature-request.yml | 10 +++++----- .github/ISSUE_TEMPLATE/regression-report.yml | 10 +++++----- .github/pull_request_template.md | 2 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 10 +++++----- CONTRIBUTING.md | 2 +- 14 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.changes/1.0.0.md b/.changes/1.0.0.md index b6cc44a9..d381d635 100644 --- a/.changes/1.0.0.md +++ b/.changes/1.0.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.0.0 - April 01, 2024 +## dbt-adapters 1.0.0 - April 01, 2024 ### Fixes diff --git a/.changes/1.1.0.md b/.changes/1.1.0.md index c43ef9aa..9e7db78e 100644 --- a/.changes/1.1.0.md +++ b/.changes/1.1.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.1.0 - May 01, 2024 +## dbt-adapters 1.1.0 - May 01, 2024 ### Features diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md index 0a28c3ad..a1c38c88 100644 --- a/.changes/1.1.1.md +++ b/.changes/1.1.1.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.1.1 - May 07, 2024 +## dbt-adapters 1.1.1 - May 07, 2024 ### Features diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md index f838b769..2aac5674 100644 --- a/.changes/1.2.1.md +++ b/.changes/1.2.1.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.2.1 - May 21, 2024 +## dbt-adapters 1.2.1 - May 21, 2024 ### Features diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md index bbe54825..e781a985 100644 --- a/.changes/1.8.0.md +++ b/.changes/1.8.0.md @@ -1,4 +1,4 @@ -## dbt-adapter 1.8.0 - May 09, 2024 +## dbt-adapters 1.8.0 - May 09, 2024 ### Features diff --git a/.changie.yaml b/.changie.yaml index 9f78b81e..afbafb22 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -4,7 +4,7 @@ headerPath: header.tpl.md changelogPath: CHANGELOG.md versionExt: md envPrefix: CHANGIE_ -versionFormat: '## dbt-adapter {{.Version}} - {{.Time.Format "January 02, 2006"}}' +versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' changeFormat: '* {{.Body}}' kinds: diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index 3bd90bf7..27cb521e 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -1,5 +1,5 @@ name: 🐞 Bug -description: Report a bug or an issue you've found with dbt-adapter +description: Report a bug or an issue you've found with dbt-adapters title: "[Bug] " labels: ["bug", "triage"] body: @@ -62,11 +62,11 @@ body: examples: - **OS**: Ubuntu 20.04 - **Python**: 3.11.6 (`python3 --version`) - - **dbt-adapter**: 1.0.0 + - **dbt-adapters**: 1.0.0 value: | - OS: - Python: - - dbt-adapter: + - dbt-adapters: render: markdown validations: required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 2e23e0fd..a89889af 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -7,5 +7,5 @@ contact_links: url: mailto:support@getdbt.com about: Are you using dbt Cloud? Contact our support team for help! - name: Participate in Discussions - url: https://github.com/dbt-labs/dbt-adapter/discussions - about: Do you have a Big Idea for dbt-adapter? Read open discussions, or start a new one + url: https://github.com/dbt-labs/dbt-adapters/discussions + about: Do you have a Big Idea for dbt-adapters? Read open discussions, or start a new one diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml index 25b28aae..22960c2d 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.yml +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -1,5 +1,5 @@ name: ✨ Feature -description: Propose a straightforward extension of dbt-adapter functionality +description: Propose a straightforward extension of dbt-adapters functionality title: "[Feature] <title>" labels: ["enhancement", "triage"] body: @@ -14,15 +14,15 @@ body: We want to make sure that features are distinct and discoverable, so that other members of the community can find them and offer their thoughts. - Issues are the right place to request straightforward extensions of existing dbt-adapter functionality. - For "big ideas" about future capabilities of dbt-adapter, we ask that you open a - [discussion](https://github.com/dbt-labs/dbt-adapter/discussions/new?category=ideas) in the "Ideas" category instead. + Issues are the right place to request straightforward extensions of existing dbt-adapters functionality. + For "big ideas" about future capabilities of dbt-adapters, we ask that you open a + [discussion](https://github.com/dbt-labs/dbt-adapters/discussions/new?category=ideas) in the "Ideas" category instead. options: - label: I have read the [expectations for open source contributors](https://docs.getdbt.com/docs/contributing/oss-expectations) required: true - label: I have searched the existing issues, and I could not find an existing issue for this feature required: true - - label: I am requesting a straightforward extension of existing dbt-adapter functionality, rather than a Big Idea better suited to a discussion + - label: I am requesting a straightforward extension of existing dbt-adapters functionality, rather than a Big Idea better suited to a discussion required: true - type: textarea attributes: diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml index 01775507..6831ede2 100644 --- a/.github/ISSUE_TEMPLATE/regression-report.yml +++ b/.github/ISSUE_TEMPLATE/regression-report.yml @@ -1,5 +1,5 @@ name: ☣️ Regression -description: Report a regression you've observed in a newer version of dbt-adapter +description: Report a regression you've observed in a newer version of dbt-adapters title: "[Regression] <title>" labels: ["regression", "triage"] body: @@ -57,13 +57,13 @@ body: examples: - **OS**: Ubuntu 20.04 - **Python**: 3.11.6 (`python3 --version`) - - **dbt-adapter (working version)**: 1.1.0 - - **dbt-adapter (regression version)**: 1.2.0 + - **dbt-adapters (working version)**: 1.1.0 + - **dbt-adapters (regression version)**: 1.2.0 value: | - OS: - Python: - - dbt-adapter (working version): - - dbt-adapter (regression version): + - dbt-adapters (working version): + - dbt-adapters (regression version): render: markdown validations: required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4fc2fcf8..3879b653 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -29,7 +29,7 @@ resolves # ### Checklist -- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-adapter/blob/main/CONTRIBUTING.md) and understand what's expected of me +- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md) and understand what's expected of me - [ ] I have run this code in development, and it appears to resolve the stated issue - [ ] This PR includes tests, or tests are not required/relevant for this PR - [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1135adb8..828350dd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -151,7 +151,7 @@ jobs: github-release: name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapter assets to the same release + # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapters assets to the same release if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} needs: [release-inputs, build-and-test, bump-version-generate-changelog] uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa477cf..43c3937e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapter 1.8.0 - May 09, 2024 +## dbt-adapters 1.8.0 - May 09, 2024 ### Features @@ -15,7 +15,7 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes -## dbt-adapter 1.2.1 - May 21, 2024 +## dbt-adapters 1.2.1 - May 21, 2024 ### Features @@ -26,13 +26,13 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Add query recording for adapters which use SQLConnectionManager * Improve memory efficiency of process_results() -## dbt-adapter 1.1.1 - May 07, 2024 +## dbt-adapters 1.1.1 - May 07, 2024 ### Features * Enable serialization contexts -## dbt-adapter 1.1.0 - May 01, 2024 +## dbt-adapters 1.1.0 - May 01, 2024 ### Features @@ -62,7 +62,7 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * add support for py3.12 -## dbt-adapter 1.0.0 - April 01, 2024 +## dbt-adapters 1.0.0 - April 01, 2024 ### Fixes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c87ea23d..28f13f6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -163,7 +163,7 @@ Remember to commit and push the file that's created. ### Signing the CLA -> **_NOTE:_** All contributors to `dbt-adapter` must sign the +> **_NOTE:_** All contributors to `dbt-adapters` must sign the > [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA). Maintainers will be unable to merge contributions until the contributor signs the CLA. From 6687ccc7cdf7b16ab5dbaab57d8aae5f9905b21a Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Tue, 18 Jun 2024 21:27:35 +0100 Subject: [PATCH 06/38] Add `get_catalog_for_single_relation` macro and capability (#231) --- .../unreleased/Features-20240528-013623.yaml | 6 ++ .../test_get_catalog_for_single_relation.py | 87 +++++++++++++++++++ dbt/adapters/base/impl.py | 15 +++- dbt/adapters/capability.py | 7 +- dbt/adapters/sql/impl.py | 9 +- .../macros/adapters/metadata.sql | 9 ++ pyproject.toml | 5 +- 7 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Features-20240528-013623.yaml create mode 100644 dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py diff --git a/.changes/unreleased/Features-20240528-013623.yaml b/.changes/unreleased/Features-20240528-013623.yaml new file mode 100644 index 00000000..2c01e972 --- /dev/null +++ b/.changes/unreleased/Features-20240528-013623.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation +time: 2024-05-28T01:36:23.588295+01:00 +custom: + Author: aranke + Issue: "231" diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py b/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py new file mode 100644 index 00000000..78055cc5 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py @@ -0,0 +1,87 @@ +import pytest + +from dbt.tests.util import run_dbt, get_connection + +models__my_table_model_sql = """ +select * from {{ ref('my_seed') }} +""" + + +models__my_view_model_sql = """ +{{ + config( + materialized='view', + ) +}} + +select * from {{ ref('my_seed') }} +""" + +seed__my_seed_csv = """id,first_name,email,ip_address,updated_at +1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 +""" + + +class BaseGetCatalogForSingleRelation: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "get_catalog_for_single_relation"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "my_seed.csv": seed__my_seed_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "my_view_model.sql": models__my_view_model_sql, + "my_table_model.sql": models__my_table_model_sql, + } + + @pytest.fixture(scope="class") + def expected_catalog_my_seed(self, project): + raise NotImplementedError( + "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`." + ) + + @pytest.fixture(scope="class") + def expected_catalog_my_model(self, project): + raise NotImplementedError( + "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`." + ) + + def get_relation_for_identifier(self, project, identifier): + return project.adapter.get_relation( + database=project.database, + schema=project.test_schema, + identifier=identifier, + ) + + def test_get_catalog_for_single_relation( + self, project, expected_catalog_my_seed, expected_catalog_my_view_model + ): + results = run_dbt(["seed"]) + assert len(results) == 1 + + my_seed_relation = self.get_relation_for_identifier(project, "my_seed") + + with get_connection(project.adapter): + actual_catalog_my_seed = project.adapter.get_catalog_for_single_relation( + my_seed_relation + ) + + assert actual_catalog_my_seed == expected_catalog_my_seed + + results = run_dbt(["run"]) + assert len(results) == 2 + + my_view_model_relation = self.get_relation_for_identifier(project, "my_view_model") + + with get_connection(project.adapter): + actual_catalog_my_view_model = project.adapter.get_catalog_for_single_relation( + my_view_model_relation + ) + + assert actual_catalog_my_view_model == expected_catalog_my_view_model diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index f58f8aba..1fdbce48 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1,10 +1,10 @@ import abc +import time from concurrent.futures import as_completed, Future from contextlib import contextmanager from datetime import datetime from enum import Enum from multiprocessing.context import SpawnContext -import time from typing import ( Any, Callable, @@ -23,12 +23,15 @@ TYPE_CHECKING, ) +import pytz from dbt_common.clients.jinja import CallableMacroGenerator from dbt_common.contracts.constraints import ( ColumnLevelConstraint, ConstraintType, ModelLevelConstraint, ) +from dbt_common.contracts.metadata import CatalogTable +from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.exceptions import ( DbtInternalError, DbtRuntimeError, @@ -38,14 +41,12 @@ NotImplementedError, UnexpectedNullError, ) -from dbt_common.events.functions import fire_event, warn_or_error from dbt_common.utils import ( AttrDict, cast_to_str, executor, filter_null_values, ) -import pytz from dbt.adapters.base.column import Column as BaseColumn from dbt.adapters.base.connections import ( @@ -222,6 +223,7 @@ class BaseAdapter(metaclass=AdapterMeta): - truncate_relation - rename_relation - get_columns_in_relation + - get_catalog_for_single_relation - get_column_schema_from_query - expand_column_types - list_relations_without_caching @@ -627,6 +629,13 @@ def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" raise NotImplementedError("`get_columns_in_relation` is not implemented for this adapter!") + @abc.abstractmethod + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + """Get catalog information including table-level and column-level metadata for a single relation.""" + raise NotImplementedError( + "`get_catalog_for_single_relation` is not implemented for this adapter!" + ) + @available.deprecated("get_columns_in_relation", lambda *a, **k: []) def get_columns_in_table(self, schema: str, identifier: str) -> List[BaseColumn]: """DEPRECATED: Get a list of the columns in the given table.""" diff --git a/dbt/adapters/capability.py b/dbt/adapters/capability.py index 305604c7..2bd49112 100644 --- a/dbt/adapters/capability.py +++ b/dbt/adapters/capability.py @@ -14,7 +14,12 @@ class Capability(str, Enum): """Indicates support for determining the time of the last table modification by querying database metadata.""" TableLastModifiedMetadataBatch = "TableLastModifiedMetadataBatch" - """Indicates support for performantly determining the time of the last table modification by querying database metadata in batch.""" + """Indicates support for performantly determining the time of the last table modification by querying database + metadata in batch.""" + + GetCatalogForSingleRelation = "GetCatalogForSingleRelation" + """Indicates support for getting catalog information including table-level and column-level metadata for a single + relation.""" class Support(str, Enum): diff --git a/dbt/adapters/sql/impl.py b/dbt/adapters/sql/impl.py index 8c6e0e8e..91ca3ebb 100644 --- a/dbt/adapters/sql/impl.py +++ b/dbt/adapters/sql/impl.py @@ -1,5 +1,6 @@ from typing import Any, List, Optional, Tuple, Type, TYPE_CHECKING +from dbt_common.contracts.metadata import CatalogTable from dbt_common.events.functions import fire_event from dbt.adapters.base import BaseAdapter, BaseRelation, available @@ -9,9 +10,9 @@ from dbt.adapters.exceptions import RelationTypeNullError from dbt.adapters.sql.connections import SQLConnectionManager - LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" +GET_CATALOG_FOR_SINGLE_RELATION_NAME = "get_catalog_for_single_relation" LIST_SCHEMAS_MACRO_NAME = "list_schemas" CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists" CREATE_SCHEMA_MACRO_NAME = "create_schema" @@ -41,6 +42,7 @@ class SQLAdapter(BaseAdapter): - get_catalog - list_relations_without_caching - get_columns_in_relation + - get_catalog_for_single_relation """ ConnectionManager: Type[SQLConnectionManager] @@ -158,6 +160,11 @@ def get_columns_in_relation(self, relation): GET_COLUMNS_IN_RELATION_MACRO_NAME, kwargs={"relation": relation} ) + def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: + return self.execute_macro( + GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs={"relation": relation} + ) + def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) diff --git a/dbt/include/global_project/macros/adapters/metadata.sql b/dbt/include/global_project/macros/adapters/metadata.sql index c8e8a414..0aa7aabb 100644 --- a/dbt/include/global_project/macros/adapters/metadata.sql +++ b/dbt/include/global_project/macros/adapters/metadata.sql @@ -77,6 +77,15 @@ 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }} {% endmacro %} +{% macro get_catalog_for_single_relation(relation) %} + {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }} +{% endmacro %} + +{% macro default__get_catalog_for_single_relation(relation) %} + {{ exceptions.raise_not_implemented( + 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }} +{% endmacro %} + {% macro get_relations() %} {{ return(adapter.dispatch('get_relations', 'dbt')()) }} {% endmacro %} diff --git a/pyproject.toml b/pyproject.toml index a4b011a8..e50aa63a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common<2.0", + "dbt-common>=1.3,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", @@ -43,6 +43,9 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] path = "dbt/adapters/__about__.py" From 9f504d3fc7a822ebeab7da1000ad78b9a7efc8b3 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 18 Jun 2024 22:01:45 +0000 Subject: [PATCH 07/38] Bumping version to 1.3.0 and generate changelog --- .changes/1.3.0.md | 5 +++++ .changes/unreleased/Features-20240528-013623.yaml | 6 ------ CHANGELOG.md | 6 ++++++ dbt/adapters/__about__.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 .changes/1.3.0.md delete mode 100644 .changes/unreleased/Features-20240528-013623.yaml diff --git a/.changes/1.3.0.md b/.changes/1.3.0.md new file mode 100644 index 00000000..dae1f819 --- /dev/null +++ b/.changes/1.3.0.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.3.0 - June 18, 2024 + +### Features + +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation diff --git a/.changes/unreleased/Features-20240528-013623.yaml b/.changes/unreleased/Features-20240528-013623.yaml deleted file mode 100644 index 2c01e972..00000000 --- a/.changes/unreleased/Features-20240528-013623.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation -time: 2024-05-28T01:36:23.588295+01:00 -custom: - Author: aranke - Issue: "231" diff --git a/CHANGELOG.md b/CHANGELOG.md index 43c3937e..5f7560f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.0 - June 18, 2024 + +### Features + +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation + ## dbt-adapters 1.2.1 - May 21, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index eb1d9a0f..d28b3ddc 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.2.1" +version = "1.3.0" From 3460543a155eeb4ccb749fe185d65c8f579dbd04 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 18 Jun 2024 22:58:01 +0000 Subject: [PATCH 08/38] Bumping version to 1.9.0 and generate changelog --- .changes/1.9.0.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.9.0.md diff --git a/.changes/1.9.0.md b/.changes/1.9.0.md new file mode 100644 index 00000000..2f016563 --- /dev/null +++ b/.changes/1.9.0.md @@ -0,0 +1 @@ +## dbt-adapters 1.9.0 - June 18, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f7560f7..016c13ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.9.0 - June 18, 2024 + ## dbt-adapters 1.8.0 - May 09, 2024 ### Features diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 6aaa73b8..7aba6409 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.8.0" +version = "1.9.0" From 46e470cbbe8cefc98b00a2f0cd7223c43c2be4d1 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Wed, 19 Jun 2024 16:50:08 +0100 Subject: [PATCH 09/38] Make `get_catalog_for_single_relation` a concrete method in `BaseAdapter` (#240) --- dbt/adapters/base/impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 1fdbce48..3b36022f 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -629,7 +629,6 @@ def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]: """Get a list of the columns in the given Relation.""" raise NotImplementedError("`get_columns_in_relation` is not implemented for this adapter!") - @abc.abstractmethod def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: """Get catalog information including table-level and column-level metadata for a single relation.""" raise NotImplementedError( From ecf3e1d52bc31a4814f25fd67bc186bbb4ed132b Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Thu, 20 Jun 2024 17:41:48 +0100 Subject: [PATCH 10/38] Remove get_catalog_for_single_relation from sql/impl.py (#241) --- dbt/adapters/sql/impl.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dbt/adapters/sql/impl.py b/dbt/adapters/sql/impl.py index 91ca3ebb..8a8473f2 100644 --- a/dbt/adapters/sql/impl.py +++ b/dbt/adapters/sql/impl.py @@ -1,6 +1,5 @@ from typing import Any, List, Optional, Tuple, Type, TYPE_CHECKING -from dbt_common.contracts.metadata import CatalogTable from dbt_common.events.functions import fire_event from dbt.adapters.base import BaseAdapter, BaseRelation, available @@ -12,7 +11,6 @@ LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching" GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation" -GET_CATALOG_FOR_SINGLE_RELATION_NAME = "get_catalog_for_single_relation" LIST_SCHEMAS_MACRO_NAME = "list_schemas" CHECK_SCHEMA_EXISTS_MACRO_NAME = "check_schema_exists" CREATE_SCHEMA_MACRO_NAME = "create_schema" @@ -160,11 +158,6 @@ def get_columns_in_relation(self, relation): GET_COLUMNS_IN_RELATION_MACRO_NAME, kwargs={"relation": relation} ) - def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]: - return self.execute_macro( - GET_CATALOG_FOR_SINGLE_RELATION_NAME, kwargs={"relation": relation} - ) - def create_schema(self, relation: BaseRelation) -> None: relation = relation.without_identifier() fire_event(SchemaCreation(relation=_make_ref_key_dict(relation))) From 10f197a09ebca9404e79710b49ec04ac71d23b99 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Thu, 20 Jun 2024 19:17:15 +0000 Subject: [PATCH 11/38] Bumping version to 1.3.1 and generate changelog --- .changes/1.3.1.md | 1 + CHANGELOG.md | 2 ++ dbt/adapters/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.3.1.md diff --git a/.changes/1.3.1.md b/.changes/1.3.1.md new file mode 100644 index 00000000..b8ec7374 --- /dev/null +++ b/.changes/1.3.1.md @@ -0,0 +1 @@ +## dbt-adapters 1.3.1 - June 20, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 016c13ee..b06ec9f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.1 - June 20, 2024 + ## dbt-adapters 1.3.0 - June 18, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index d28b3ddc..35ec9a37 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.0" +version = "1.3.1" From f4880e57ae59215d6f4beedcaee290b5ffe22ae3 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Thu, 20 Jun 2024 19:27:26 +0000 Subject: [PATCH 12/38] Bumping version to 1.9.1 and generate changelog --- .changes/1.9.1.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.9.1.md diff --git a/.changes/1.9.1.md b/.changes/1.9.1.md new file mode 100644 index 00000000..900e6b75 --- /dev/null +++ b/.changes/1.9.1.md @@ -0,0 +1 @@ +## dbt-adapters 1.9.1 - June 20, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index b06ec9f2..9114bcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.9.1 - June 20, 2024 + ## dbt-adapters 1.9.0 - June 18, 2024 ## dbt-adapters 1.8.0 - May 09, 2024 diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 7aba6409..70227976 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.9.0" +version = "1.9.1" From 267cf5e31a132be6bf994c95ccce1d628043be64 Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:44:33 -0500 Subject: [PATCH 13/38] update user docs-issue workflow (#238) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/docs-issue.yml | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/docs-issue.yml diff --git a/.github/workflows/docs-issue.yml b/.github/workflows/docs-issue.yml new file mode 100644 index 00000000..f49cf517 --- /dev/null +++ b/.github/workflows/docs-issue.yml @@ -0,0 +1,41 @@ +# **what?** +# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed + +# **why?** +# To reduce barriers for keeping docs up to date + +# **when?** +# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. + + +name: Open issues in docs.getdbt.com repo when an issue is labeled +run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" + +on: + issues: + types: [labeled, closed] + +defaults: + run: + shell: bash + +permissions: + issues: write # comments on issues + +jobs: + open_issues: + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs')) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit From e964302121f70bacc8f827683c329e06696216cf Mon Sep 17 00:00:00 2001 From: Peter Webb <peterallenwebb@gmail.com> Date: Fri, 21 Jun 2024 15:21:15 -0400 Subject: [PATCH 14/38] Fix Query Timer Resolution (#246) --- .changes/unreleased/Under the Hood-20240621-150837.yaml | 6 ++++++ dbt/adapters/events/types.py | 2 +- dbt/adapters/sql/connections.py | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240621-150837.yaml diff --git a/.changes/unreleased/Under the Hood-20240621-150837.yaml b/.changes/unreleased/Under the Hood-20240621-150837.yaml new file mode 100644 index 00000000..fab070c7 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240621-150837.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Fix query timer resolution +time: 2024-06-21T15:08:37.43527-04:00 +custom: + Author: peterallenwebb + Issue: "246" diff --git a/dbt/adapters/events/types.py b/dbt/adapters/events/types.py index 7d98269d..47c48da6 100644 --- a/dbt/adapters/events/types.py +++ b/dbt/adapters/events/types.py @@ -190,7 +190,7 @@ def code(self) -> str: return "E017" def message(self) -> str: - return f"SQL status: {self.status} in {self.elapsed} seconds" + return f"SQL status: {self.status} in {self.elapsed:.3f} seconds" class SQLCommit(DebugLevel): diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index d8699fd3..13111a6e 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -88,7 +88,8 @@ def add_query( node_info=get_node_info(), ) ) - pre = time.time() + + pre = time.perf_counter() cursor = connection.handle.cursor() cursor.execute(sql, bindings) @@ -96,7 +97,7 @@ def add_query( fire_event( SQLQueryStatus( status=str(self.get_response(cursor)), - elapsed=round((time.time() - pre)), + elapsed=time.perf_counter() - pre, node_info=get_node_info(), ) ) From 0739850873de0af14bc5506accb597256f0cdad1 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke <kshitij.aranke@dbtlabs.com> Date: Wed, 26 Jun 2024 23:47:14 +0100 Subject: [PATCH 15/38] Add optional `should_release_connection` parameter to `connection_named` method (#247) --- .changes/unreleased/Under the Hood-20240624-161108.yaml | 6 ++++++ dbt/adapters/base/impl.py | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240624-161108.yaml diff --git a/.changes/unreleased/Under the Hood-20240624-161108.yaml b/.changes/unreleased/Under the Hood-20240624-161108.yaml new file mode 100644 index 00000000..9042547f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240624-161108.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add optional release_connection parameter to connection_named method +time: 2024-06-24T16:11:08.773419+01:00 +custom: + Author: aranke + Issue: "247" diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 3b36022f..ec590137 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -319,14 +319,18 @@ def nice_connection_name(self) -> str: return conn.name @contextmanager - def connection_named(self, name: str, query_header_context: Any = None) -> Iterator[None]: + def connection_named( + self, name: str, query_header_context: Any = None, should_release_connection=True + ) -> Iterator[None]: try: if self.connections.query_header is not None: self.connections.query_header.set(name, query_header_context) self.acquire_connection(name) yield finally: - self.release_connection() + if should_release_connection: + self.release_connection() + if self.connections.query_header is not None: self.connections.query_header.reset() From d6f736956ef0948cb3fa61d2f5a00d4aa9a6a2aa Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 2 Jul 2024 15:45:03 +0000 Subject: [PATCH 16/38] Bumping version to 1.3.2 and generate changelog --- .changes/1.3.2.md | 6 ++++++ .changes/unreleased/Under the Hood-20240621-150837.yaml | 6 ------ .changes/unreleased/Under the Hood-20240624-161108.yaml | 6 ------ CHANGELOG.md | 7 +++++++ dbt/adapters/__about__.py | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 .changes/1.3.2.md delete mode 100644 .changes/unreleased/Under the Hood-20240621-150837.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240624-161108.yaml diff --git a/.changes/1.3.2.md b/.changes/1.3.2.md new file mode 100644 index 00000000..bdc7f593 --- /dev/null +++ b/.changes/1.3.2.md @@ -0,0 +1,6 @@ +## dbt-adapters 1.3.2 - July 02, 2024 + +### Under the Hood + +* Fix query timer resolution +* Add optional release_connection parameter to connection_named method diff --git a/.changes/unreleased/Under the Hood-20240621-150837.yaml b/.changes/unreleased/Under the Hood-20240621-150837.yaml deleted file mode 100644 index fab070c7..00000000 --- a/.changes/unreleased/Under the Hood-20240621-150837.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Fix query timer resolution -time: 2024-06-21T15:08:37.43527-04:00 -custom: - Author: peterallenwebb - Issue: "246" diff --git a/.changes/unreleased/Under the Hood-20240624-161108.yaml b/.changes/unreleased/Under the Hood-20240624-161108.yaml deleted file mode 100644 index 9042547f..00000000 --- a/.changes/unreleased/Under the Hood-20240624-161108.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add optional release_connection parameter to connection_named method -time: 2024-06-24T16:11:08.773419+01:00 -custom: - Author: aranke - Issue: "247" diff --git a/CHANGELOG.md b/CHANGELOG.md index 9114bcd1..ce8d7837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.2 - July 02, 2024 + +### Under the Hood + +* Fix query timer resolution +* Add optional release_connection parameter to connection_named method + ## dbt-adapters 1.3.1 - June 20, 2024 ## dbt-adapters 1.3.0 - June 18, 2024 diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 35ec9a37..2238c8f6 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.1" +version = "1.3.2" From d2842a73d11b2e49d72d87429314e96eae0574fe Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:42:58 +0000 Subject: [PATCH 17/38] [Adap-207] remove subshell around get show sql with limit predicate (#249) Co-authored-by: Mila Page <versusfacit@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../Under the Hood-20240624-231955.yaml | 7 ++++ .gitignore | 3 ++ .../tests/adapter/dbt_show/test_dbt_show.py | 17 ++++++++++ .../global_project/macros/adapters/show.sql | 34 +++++++++++-------- 4 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240624-231955.yaml diff --git a/.changes/unreleased/Under the Hood-20240624-231955.yaml b/.changes/unreleased/Under the Hood-20240624-231955.yaml new file mode 100644 index 00000000..e3eaccb7 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240624-231955.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: --limit flag no longer subshells the query. This resolves the dbt Cloud experience + issue where limit prevents ordering elements.. +time: 2024-06-24T23:19:55.611142-07:00 +custom: + Author: versusfacit + Issue: "207" diff --git a/.gitignore b/.gitignore index a14d6d0d..807d3d41 100644 --- a/.gitignore +++ b/.gitignore @@ -154,3 +154,6 @@ cython_debug/ # PyCharm .idea/ + +# Vim +*.swp diff --git a/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py b/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py index c00491aa..3f3d36c5 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py +++ b/dbt-tests-adapter/dbt/tests/adapter/dbt_show/test_dbt_show.py @@ -1,5 +1,6 @@ import pytest +from dbt_common.exceptions import DbtRuntimeError from dbt.tests.adapter.dbt_show import fixtures from dbt.tests.util import run_dbt @@ -47,9 +48,25 @@ def test_sql_header(self, project): run_dbt(["show", "--select", "sql_header", "--vars", "timezone: Asia/Kolkata"]) +class BaseShowDoesNotHandleDoubleLimit: + """see issue: https://github.com/dbt-labs/dbt-adapters/issues/207""" + + DATABASE_ERROR_MESSAGE = 'syntax error at or near "limit"' + + def test_double_limit_throws_syntax_error(self, project): + with pytest.raises(DbtRuntimeError) as e: + run_dbt(["show", "--limit", "1", "--inline", "select 1 limit 1"]) + + assert self.DATABASE_ERROR_MESSAGE in str(e) + + class TestPostgresShowSqlHeader(BaseShowSqlHeader): pass class TestPostgresShowLimit(BaseShowLimit): pass + + +class TestShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass diff --git a/dbt/include/global_project/macros/adapters/show.sql b/dbt/include/global_project/macros/adapters/show.sql index 33a93f3d..3a5faa98 100644 --- a/dbt/include/global_project/macros/adapters/show.sql +++ b/dbt/include/global_project/macros/adapters/show.sql @@ -1,22 +1,26 @@ +{# + We expect a syntax error if dbt show is invoked both with a --limit flag to show + and with a limit predicate embedded in its inline query. No special handling is + provided out-of-box. +#} {% macro get_show_sql(compiled_code, sql_header, limit) -%} - {%- if sql_header -%} + {%- if sql_header is not none -%} {{ sql_header }} - {%- endif -%} - {%- if limit is not none -%} + {%- endif %} {{ get_limit_subquery_sql(compiled_code, limit) }} - {%- else -%} - {{ compiled_code }} - {%- endif -%} {% endmacro %} -{% macro get_limit_subquery_sql(sql, limit) %} - {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }} -{% endmacro %} +{# + Not necessarily a true subquery anymore. Now, merely a query subordinate + to the calling macro. +#} +{%- macro get_limit_subquery_sql(sql, limit) -%} + {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }} +{%- endmacro -%} -{% macro default__get_limit_subquery_sql(sql, limit) %} - select * - from ( - {{ sql }} - ) as model_limit_subq - limit {{ limit }} +{% macro default__get_limit_sql(sql, limit) %} + {{ compiled_code }} + {% if limit is not none %} + limit {{ limit }} + {%- endif -%} {% endmacro %} From 99e81943d39baf74e9cbd9c797227a071949a66e Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:59:45 -0400 Subject: [PATCH 18/38] [Bug] Fix scenarios where `--empty` flag pushes limit 0 into metadata queries (#257) --- .../unreleased/Fixes-20240702-161935.yaml | 6 + .gitignore | 7 ++ .../dbt/tests/adapter/empty/_models.py | 111 ++++++++++++++++++ .../dbt/tests/adapter/empty/test_empty.py | 99 ++++++++-------- .../macros/adapters/apply_grants.sql | 8 +- .../macros/adapters/columns.sql | 10 +- .../macros/adapters/relation.sql | 2 +- .../materializations/models/clone/clone.sql | 4 +- .../models/clone/create_or_replace_clone.sql | 2 +- .../models/materialized_view.sql | 4 +- .../macros/materializations/seeds/helpers.sql | 2 +- .../macros/materializations/seeds/seed.sql | 2 +- .../materializations/snapshots/helpers.sql | 2 +- .../snapshots/snapshot_merge.sql | 2 +- .../global_project/macros/relations/drop.sql | 2 +- .../relations/materialized_view/drop.sql | 2 +- .../macros/relations/rename.sql | 2 +- .../macros/relations/table/drop.sql | 2 +- .../macros/relations/view/create.sql | 2 +- .../macros/relations/view/drop.sql | 2 +- .../macros/relations/view/replace.sql | 2 +- 21 files changed, 202 insertions(+), 73 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240702-161935.yaml create mode 100644 dbt-tests-adapter/dbt/tests/adapter/empty/_models.py diff --git a/.changes/unreleased/Fixes-20240702-161935.yaml b/.changes/unreleased/Fixes-20240702-161935.yaml new file mode 100644 index 00000000..0b0f21ee --- /dev/null +++ b/.changes/unreleased/Fixes-20240702-161935.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses +time: 2024-07-02T16:19:35.457997-04:00 +custom: + Author: mikealfare + Issue: "213" diff --git a/.gitignore b/.gitignore index 807d3d41..31258db2 100644 --- a/.gitignore +++ b/.gitignore @@ -155,5 +155,12 @@ cython_debug/ # PyCharm .idea/ +# MacOS +.DS_Store + +# vscode +.vscode/ +.venv/ + # Vim *.swp diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py b/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py new file mode 100644 index 00000000..f5e684f7 --- /dev/null +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/_models.py @@ -0,0 +1,111 @@ +model_input_sql = """ +select 1 as id +""" + +ephemeral_model_input_sql = """ +{{ config(materialized='ephemeral') }} +select 2 as id +""" + +raw_source_csv = """id +3 +""" + + +model_sql = """ +select * +from {{ ref('model_input') }} +union all +select * +from {{ ref('ephemeral_model_input') }} +union all +select * +from {{ source('seed_sources', 'raw_source') }} +""" + + +model_inline_sql = """ +select * from {{ source('seed_sources', 'raw_source') }} as raw_source +""" + +schema_sources_yml = """ +sources: + - name: seed_sources + schema: "{{ target.schema }}" + tables: + - name: raw_source +""" + + +SEED = """ +my_id,my_value +1,a +2,b +3,c +""".strip() + + +SCHEMA = """ +version: 2 + +seeds: + - name: my_seed + description: "This is my_seed" + columns: + - name: id + description: "This is my_seed.my_id" +""" + +CONTROL = """ +select * from {{ ref("my_seed") }} +""" + + +GET_COLUMNS_IN_RELATION = """ +{{ config(materialized="table") }} +{% set columns = adapter.get_columns_in_relation(ref("my_seed")) %} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_TYPE = """ +{{ config(materialized="table") }} +{{ alter_column_type(ref("my_seed"), "MY_VALUE", "varchar") }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"relations": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_COLUMN_COMMENT = """ +{{ config( + materialized="table", + persist_docs={"columns": True}, +) }} +select * from {{ ref("my_seed") }} +""" + + +ALTER_RELATION_ADD_REMOVE_COLUMNS = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{% set my_column = api.Column("my_column", "varchar") %} +{% do alter_relation_add_remove_columns(my_seed, [my_column], none) %} +{% do alter_relation_add_remove_columns(my_seed, none, [my_column]) %} +select * from {{ ref("my_seed") }} +""" + + +TRUNCATE_RELATION = """ +{{ config(materialized="table") }} +{% set my_seed = adapter.Relation.create(this.database, this.schema, "my_seed", "table") %} +{{ truncate_relation(my_seed) }} +select * from {{ ref("my_seed") }} +""" diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py index 2249d98d..a0a44edc 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py @@ -1,57 +1,23 @@ -import pytest - from dbt.tests.util import relation_from_name, run_dbt +import pytest - -model_input_sql = """ -select 1 as id -""" - -ephemeral_model_input_sql = """ -{{ config(materialized='ephemeral') }} -select 2 as id -""" - -raw_source_csv = """id -3 -""" - - -model_sql = """ -select * -from {{ ref('model_input') }} -union all -select * -from {{ ref('ephemeral_model_input') }} -union all -select * -from {{ source('seed_sources', 'raw_source') }} -""" - - -schema_sources_yml = """ -sources: - - name: seed_sources - schema: "{{ target.schema }}" - tables: - - name: raw_source -""" +from tests.functional.adapter.empty import _models class BaseTestEmpty: @pytest.fixture(scope="class") def seeds(self): return { - "raw_source.csv": raw_source_csv, + "raw_source.csv": _models.raw_source_csv, } @pytest.fixture(scope="class") def models(self): return { - "model_input.sql": model_input_sql, - "ephemeral_model_input.sql": ephemeral_model_input_sql, - "model.sql": model_sql, - "sources.yml": schema_sources_yml, + "model_input.sql": _models.model_input_sql, + "ephemeral_model_input.sql": _models.ephemeral_model_input_sql, + "model.sql": _models.model_sql, + "sources.yml": _models.schema_sources_yml, } def assert_row_count(self, project, relation_name: str, expected_row_count: int): @@ -75,13 +41,9 @@ def test_run_with_empty(self, project): class BaseTestEmptyInlineSourceRef(BaseTestEmpty): @pytest.fixture(scope="class") def models(self): - model_sql = """ - select * from {{ source('seed_sources', 'raw_source') }} as raw_source - """ - return { - "model.sql": model_sql, - "sources.yml": schema_sources_yml, + "model.sql": _models.model_inline_sql, + "sources.yml": _models.schema_sources_yml, } def test_run_with_empty(self, project): @@ -92,4 +54,47 @@ def test_run_with_empty(self, project): class TestEmpty(BaseTestEmpty): + """ + Though we don't create these classes anymore, we need to keep this one in case an adapter wanted to import the test as-is to automatically run it. + We should consider adding a deprecation warning that suggests moving this into the concrete adapter and importing `BaseTestEmpty` instead. + """ + pass + + +class MetadataWithEmptyFlag: + @pytest.fixture(scope="class") + def seeds(self): + return {"my_seed.csv": _models.SEED} + + @pytest.fixture(scope="class") + def models(self): + return { + "schema.yml": _models.SCHEMA, + "control.sql": _models.CONTROL, + "get_columns_in_relation.sql": _models.GET_COLUMNS_IN_RELATION, + "alter_column_type.sql": _models.ALTER_COLUMN_TYPE, + "alter_relation_comment.sql": _models.ALTER_RELATION_COMMENT, + "alter_column_comment.sql": _models.ALTER_COLUMN_COMMENT, + "alter_relation_add_remove_columns.sql": _models.ALTER_RELATION_ADD_REMOVE_COLUMNS, + "truncate_relation.sql": _models.TRUNCATE_RELATION, + } + + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + run_dbt(["seed"]) + + @pytest.mark.parametrize( + "model", + [ + "control", + "get_columns_in_relation", + "alter_column_type", + "alter_relation_comment", + "alter_column_comment", + "alter_relation_add_remove_columns", + "truncate_relation", + ], + ) + def test_run(self, project, model): + run_dbt(["run", "--empty", "--select", model]) diff --git a/dbt/include/global_project/macros/adapters/apply_grants.sql b/dbt/include/global_project/macros/adapters/apply_grants.sql index 10906e7f..c75eef89 100644 --- a/dbt/include/global_project/macros/adapters/apply_grants.sql +++ b/dbt/include/global_project/macros/adapters/apply_grants.sql @@ -61,7 +61,7 @@ {% endmacro %} {% macro default__get_show_grant_sql(relation) %} - show grants on {{ relation }} + show grants on {{ relation.render() }} {% endmacro %} @@ -70,7 +70,7 @@ {% endmacro %} {%- macro default__get_grant_sql(relation, privilege, grantees) -%} - grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }} + grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }} {%- endmacro -%} @@ -79,7 +79,7 @@ {% endmacro %} {%- macro default__get_revoke_sql(relation, privilege, grantees) -%} - revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }} + revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }} {%- endmacro -%} @@ -147,7 +147,7 @@ {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %} {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %} {% if not (needs_granting or needs_revoking) %} - {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}} + {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}} {% endif %} {% else %} {#-- We don't think there's any chance of previous grants having carried over. --#} diff --git a/dbt/include/global_project/macros/adapters/columns.sql b/dbt/include/global_project/macros/adapters/columns.sql index 663a827b..96e6f3f2 100644 --- a/dbt/include/global_project/macros/adapters/columns.sql +++ b/dbt/include/global_project/macros/adapters/columns.sql @@ -96,10 +96,10 @@ {%- set tmp_column = column_name + "__dbt_alter" -%} {% call statement('alter_column_type') %} - alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; - update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; - alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade; - alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} + alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }}; + update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }}; + alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade; + alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }} {% endcall %} {% endmacro %} @@ -120,7 +120,7 @@ {% set sql -%} - alter {{ relation.type }} {{ relation }} + alter {{ relation.type }} {{ relation.render() }} {% for column in add_columns %} add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }} diff --git a/dbt/include/global_project/macros/adapters/relation.sql b/dbt/include/global_project/macros/adapters/relation.sql index 1c2bd880..b9af4969 100644 --- a/dbt/include/global_project/macros/adapters/relation.sql +++ b/dbt/include/global_project/macros/adapters/relation.sql @@ -38,7 +38,7 @@ {% macro default__truncate_relation(relation) -%} {% call statement('truncate_relation') -%} - truncate table {{ relation }} + truncate table {{ relation.render() }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/models/clone/clone.sql b/dbt/include/global_project/macros/materializations/models/clone/clone.sql index 01c8c393..56d80082 100644 --- a/dbt/include/global_project/macros/materializations/models/clone/clone.sql +++ b/dbt/include/global_project/macros/materializations/models/clone/clone.sql @@ -27,14 +27,14 @@ {%- set target_relation = this.incorporate(type='table') -%} {% if existing_relation is not none and not existing_relation.is_table %} - {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }} + {{ log("Dropping relation " ~ existing_relation.render() ~ " because it is of type " ~ existing_relation.type) }} {{ drop_relation_if_exists(existing_relation) }} {% endif %} -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace' {% call statement('main') %} {% if target_relation and defer_relation and target_relation == defer_relation %} - {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation) }} + {{ log("Target relation and defer relation are the same, skipping clone for relation: " ~ target_relation.render()) }} {% else %} {{ create_or_replace_clone(target_relation, defer_relation) }} {% endif %} diff --git a/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql b/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql index 204e9e87..cdb2559c 100644 --- a/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql +++ b/dbt/include/global_project/macros/materializations/models/clone/create_or_replace_clone.sql @@ -3,5 +3,5 @@ {% endmacro %} {% macro default__create_or_replace_clone(this_relation, defer_relation) %} - create or replace table {{ this_relation }} clone {{ defer_relation }} + create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/models/materialized_view.sql b/dbt/include/global_project/macros/materializations/models/materialized_view.sql index 6dc30bf9..a39f8aa2 100644 --- a/dbt/include/global_project/macros/materializations/models/materialized_view.sql +++ b/dbt/include/global_project/macros/materializations/models/materialized_view.sql @@ -71,9 +71,9 @@ {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %} {% elif on_configuration_change == 'continue' %} {% set build_sql = '' %} - {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation ~ "`") }} + {{ exceptions.warn("Configuration changes were identified and `on_configuration_change` was set to `continue` for `" ~ target_relation.render() ~ "`") }} {% elif on_configuration_change == 'fail' %} - {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation ~ "`") }} + {{ exceptions.raise_fail_fast_error("Configuration changes were identified and `on_configuration_change` was set to `fail` for `" ~ target_relation.render() ~ "`") }} {% else %} -- this only happens if the user provides a value other than `apply`, 'skip', 'fail' diff --git a/dbt/include/global_project/macros/materializations/seeds/helpers.sql b/dbt/include/global_project/macros/materializations/seeds/helpers.sql index 44dbf370..d87c258b 100644 --- a/dbt/include/global_project/macros/materializations/seeds/helpers.sql +++ b/dbt/include/global_project/macros/materializations/seeds/helpers.sql @@ -37,7 +37,7 @@ {% set sql = create_csv_table(model, agate_table) %} {% else %} {{ adapter.truncate_relation(old_relation) }} - {% set sql = "truncate table " ~ old_relation %} + {% set sql = "truncate table " ~ old_relation.render() %} {% endif %} {{ return(sql) }} diff --git a/dbt/include/global_project/macros/materializations/seeds/seed.sql b/dbt/include/global_project/macros/materializations/seeds/seed.sql index 3b66252d..4ee4fb80 100644 --- a/dbt/include/global_project/macros/materializations/seeds/seed.sql +++ b/dbt/include/global_project/macros/materializations/seeds/seed.sql @@ -22,7 +22,7 @@ -- build model {% set create_table_sql = "" %} {% if exists_as_view %} - {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }} + {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation.render())) }} {% elif exists_as_table %} {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %} {% else %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 7fd4bfd5..bb71974c 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -8,7 +8,7 @@ {% macro default__create_columns(relation, columns) %} {% for column in columns %} {% call statement() %} - alter table {{ relation }} add column "{{ column.name }}" {{ column.data_type }}; + alter table {{ relation.render() }} add column "{{ column.name }}" {{ column.data_type }}; {% endcall %} {% endfor %} {% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 6bc50fd3..56798811 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -7,7 +7,7 @@ {% macro default__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - merge into {{ target }} as DBT_INTERNAL_DEST + merge into {{ target.render() }} as DBT_INTERNAL_DEST using {{ source }} as DBT_INTERNAL_SOURCE on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id diff --git a/dbt/include/global_project/macros/relations/drop.sql b/dbt/include/global_project/macros/relations/drop.sql index 58abd14d..e66511da 100644 --- a/dbt/include/global_project/macros/relations/drop.sql +++ b/dbt/include/global_project/macros/relations/drop.sql @@ -16,7 +16,7 @@ {{ drop_materialized_view(relation) }} {%- else -%} - drop {{ relation.type }} if exists {{ relation }} cascade + drop {{ relation.type }} if exists {{ relation.render() }} cascade {%- endif -%} diff --git a/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/dbt/include/global_project/macros/relations/materialized_view/drop.sql index b218d0f3..8235b1c6 100644 --- a/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_materialized_view(relation) -%} - drop materialized view if exists {{ relation }} cascade + drop materialized view if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/rename.sql b/dbt/include/global_project/macros/relations/rename.sql index d7f3a72e..4b913df3 100644 --- a/dbt/include/global_project/macros/relations/rename.sql +++ b/dbt/include/global_project/macros/relations/rename.sql @@ -30,6 +30,6 @@ {% macro default__rename_relation(from_relation, to_relation) -%} {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %} {% call statement('rename_relation') -%} - alter table {{ from_relation }} rename to {{ target_name }} + alter table {{ from_relation.render() }} rename to {{ target_name }} {%- endcall %} {% endmacro %} diff --git a/dbt/include/global_project/macros/relations/table/drop.sql b/dbt/include/global_project/macros/relations/table/drop.sql index d7d5941c..038ded9e 100644 --- a/dbt/include/global_project/macros/relations/table/drop.sql +++ b/dbt/include/global_project/macros/relations/table/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_table(relation) -%} - drop table if exists {{ relation }} cascade + drop table if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/view/create.sql b/dbt/include/global_project/macros/relations/view/create.sql index 41cd196c..ee83befa 100644 --- a/dbt/include/global_project/macros/relations/view/create.sql +++ b/dbt/include/global_project/macros/relations/view/create.sql @@ -16,7 +16,7 @@ {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create view {{ relation }} + create view {{ relation.render() }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} {{ get_assert_columns_equivalent(sql) }} diff --git a/dbt/include/global_project/macros/relations/view/drop.sql b/dbt/include/global_project/macros/relations/view/drop.sql index 7e1924fa..84c91a36 100644 --- a/dbt/include/global_project/macros/relations/view/drop.sql +++ b/dbt/include/global_project/macros/relations/view/drop.sql @@ -10,5 +10,5 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro default__drop_view(relation) -%} - drop view if exists {{ relation }} cascade + drop view if exists {{ relation.render() }} cascade {%- endmacro %} diff --git a/dbt/include/global_project/macros/relations/view/replace.sql b/dbt/include/global_project/macros/relations/view/replace.sql index 1da06134..a0f0dc76 100644 --- a/dbt/include/global_project/macros/relations/view/replace.sql +++ b/dbt/include/global_project/macros/relations/view/replace.sql @@ -61,6 +61,6 @@ {% endmacro %} {% macro default__handle_existing_table(full_refresh, old_relation) %} - {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }} + {{ log("Dropping relation " ~ old_relation.render() ~ " because it is of type " ~ old_relation.type) }} {{ adapter.drop_relation(old_relation) }} {% endmacro %} From 7e9731cd819d521ffd638f0e1b96a96e58504eb5 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:25:11 -0400 Subject: [PATCH 19/38] Fix import statement for `--empty` tests (#258) --- dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py index a0a44edc..de15bd5b 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py +++ b/dbt-tests-adapter/dbt/tests/adapter/empty/test_empty.py @@ -1,7 +1,7 @@ from dbt.tests.util import relation_from_name, run_dbt import pytest -from tests.functional.adapter.empty import _models +from dbt.tests.adapter.empty import _models class BaseTestEmpty: From 78b2b4c5867e13d07f5c5d9bb4ffcd6081aedd22 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 9 Jul 2024 20:07:46 +0000 Subject: [PATCH 20/38] Bumping version to 1.3.3 and generate changelog --- .changes/1.3.3.md | 9 +++++++++ .changes/unreleased/Fixes-20240702-161935.yaml | 6 ------ .../unreleased/Under the Hood-20240624-231955.yaml | 7 ------- CHANGELOG.md | 10 ++++++++++ dbt/adapters/__about__.py | 2 +- 5 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 .changes/1.3.3.md delete mode 100644 .changes/unreleased/Fixes-20240702-161935.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240624-231955.yaml diff --git a/.changes/1.3.3.md b/.changes/1.3.3.md new file mode 100644 index 00000000..22ab18e3 --- /dev/null +++ b/.changes/1.3.3.md @@ -0,0 +1,9 @@ +## dbt-adapters 1.3.3 - July 09, 2024 + +### Fixes + +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses + +### Under the Hood + +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. diff --git a/.changes/unreleased/Fixes-20240702-161935.yaml b/.changes/unreleased/Fixes-20240702-161935.yaml deleted file mode 100644 index 0b0f21ee..00000000 --- a/.changes/unreleased/Fixes-20240702-161935.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses -time: 2024-07-02T16:19:35.457997-04:00 -custom: - Author: mikealfare - Issue: "213" diff --git a/.changes/unreleased/Under the Hood-20240624-231955.yaml b/.changes/unreleased/Under the Hood-20240624-231955.yaml deleted file mode 100644 index e3eaccb7..00000000 --- a/.changes/unreleased/Under the Hood-20240624-231955.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Under the Hood -body: --limit flag no longer subshells the query. This resolves the dbt Cloud experience - issue where limit prevents ordering elements.. -time: 2024-06-24T23:19:55.611142-07:00 -custom: - Author: versusfacit - Issue: "207" diff --git a/CHANGELOG.md b/CHANGELOG.md index ce8d7837..8834049e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,16 @@ and is generated by [Changie](https://github.com/miniscruff/changie). * Update Clone test to reflect core change removing `deferred` attribute from nodes +## dbt-adapters 1.3.3 - July 09, 2024 + +### Fixes + +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses + +### Under the Hood + +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. + ## dbt-adapters 1.3.2 - July 02, 2024 ### Under the Hood diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 2238c8f6..cfa2db68 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.2" +version = "1.3.3" From 594ad067ea1e85f4cef4fb2b2b420695c6ba936f Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 9 Jul 2024 20:22:10 +0000 Subject: [PATCH 21/38] Bumping version to 1.9.2 and generate changelog --- .changes/1.9.2.md | 1 + CHANGELOG.md | 2 ++ dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .changes/1.9.2.md diff --git a/.changes/1.9.2.md b/.changes/1.9.2.md new file mode 100644 index 00000000..26505f20 --- /dev/null +++ b/.changes/1.9.2.md @@ -0,0 +1 @@ +## dbt-adapters 1.9.2 - July 09, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8834049e..1a2f1833 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.9.2 - July 09, 2024 + ## dbt-adapters 1.9.1 - June 20, 2024 ## dbt-adapters 1.9.0 - June 18, 2024 diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 70227976..1b022739 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.9.1" +version = "1.9.2" From 6e50885b94af636bd8c39d92a201ff11c75b05af Mon Sep 17 00:00:00 2001 From: Peter Webb <peterallenwebb@gmail.com> Date: Tue, 16 Jul 2024 17:10:28 -0400 Subject: [PATCH 22/38] Rework record/replay to record at the database connection level. (#244) Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../Under the Hood-20240625-110833.yaml | 6 ++ dbt/adapters/record.py | 67 ------------------- dbt/adapters/record/__init__.py | 2 + dbt/adapters/record/cursor/cursor.py | 54 +++++++++++++++ dbt/adapters/record/cursor/description.py | 37 ++++++++++ dbt/adapters/record/cursor/execute.py | 20 ++++++ dbt/adapters/record/cursor/fetchall.py | 66 ++++++++++++++++++ dbt/adapters/record/cursor/fetchmany.py | 23 +++++++ dbt/adapters/record/cursor/fetchone.py | 23 +++++++ dbt/adapters/record/cursor/rowcount.py | 23 +++++++ dbt/adapters/record/handle.py | 24 +++++++ dbt/adapters/sql/connections.py | 3 - docs/guides/record_replay.md | 16 +---- 13 files changed, 280 insertions(+), 84 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240625-110833.yaml delete mode 100644 dbt/adapters/record.py create mode 100644 dbt/adapters/record/__init__.py create mode 100644 dbt/adapters/record/cursor/cursor.py create mode 100644 dbt/adapters/record/cursor/description.py create mode 100644 dbt/adapters/record/cursor/execute.py create mode 100644 dbt/adapters/record/cursor/fetchall.py create mode 100644 dbt/adapters/record/cursor/fetchmany.py create mode 100644 dbt/adapters/record/cursor/fetchone.py create mode 100644 dbt/adapters/record/cursor/rowcount.py create mode 100644 dbt/adapters/record/handle.py diff --git a/.changes/unreleased/Under the Hood-20240625-110833.yaml b/.changes/unreleased/Under the Hood-20240625-110833.yaml new file mode 100644 index 00000000..c4486a8f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240625-110833.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Rework record/replay to record at the database connection level. +time: 2024-06-25T11:08:33.264457-04:00 +custom: + Author: peteralllenwebb + Issue: "244" diff --git a/dbt/adapters/record.py b/dbt/adapters/record.py deleted file mode 100644 index 5204f59c..00000000 --- a/dbt/adapters/record.py +++ /dev/null @@ -1,67 +0,0 @@ -import dataclasses -from io import StringIO -import json -import re -from typing import Any, Optional, Mapping - -from agate import Table - -from dbt_common.events.contextvars import get_node_info -from dbt_common.record import Record, Recorder - -from dbt.adapters.contracts.connection import AdapterResponse - - -@dataclasses.dataclass -class QueryRecordParams: - sql: str - auto_begin: bool = False - fetch: bool = False - limit: Optional[int] = None - node_unique_id: Optional[str] = None - - def __post_init__(self) -> None: - if self.node_unique_id is None: - node_info = get_node_info() - self.node_unique_id = node_info["unique_id"] if node_info else "" - - @staticmethod - def _clean_up_sql(sql: str) -> str: - sql = re.sub(r"--.*?\n", "", sql) # Remove single-line comments (--) - sql = re.sub(r"/\*.*?\*/", "", sql, flags=re.DOTALL) # Remove multi-line comments (/* */) - return sql.replace(" ", "").replace("\n", "") - - def _matches(self, other: "QueryRecordParams") -> bool: - return self.node_unique_id == other.node_unique_id and self._clean_up_sql( - self.sql - ) == self._clean_up_sql(other.sql) - - -@dataclasses.dataclass -class QueryRecordResult: - adapter_response: Optional["AdapterResponse"] - table: Optional[Table] - - def _to_dict(self) -> Any: - buf = StringIO() - self.table.to_json(buf) # type: ignore - - return { - "adapter_response": self.adapter_response.to_dict(), # type: ignore - "table": buf.getvalue(), - } - - @classmethod - def _from_dict(cls, dct: Mapping) -> "QueryRecordResult": - return QueryRecordResult( - adapter_response=AdapterResponse.from_dict(dct["adapter_response"]), - table=Table.from_object(json.loads(dct["table"])), - ) - - -class QueryRecord(Record): - params_cls = QueryRecordParams - result_cls = QueryRecordResult - - -Recorder.register_record_type(QueryRecord) diff --git a/dbt/adapters/record/__init__.py b/dbt/adapters/record/__init__.py new file mode 100644 index 00000000..afde4a01 --- /dev/null +++ b/dbt/adapters/record/__init__.py @@ -0,0 +1,2 @@ +from dbt.adapters.record.handle import RecordReplayHandle +from dbt.adapters.record.cursor.cursor import RecordReplayCursor diff --git a/dbt/adapters/record/cursor/cursor.py b/dbt/adapters/record/cursor/cursor.py new file mode 100644 index 00000000..577178db --- /dev/null +++ b/dbt/adapters/record/cursor/cursor.py @@ -0,0 +1,54 @@ +from typing import Any, Optional + +from dbt_common.record import record_function + +from dbt.adapters.contracts.connection import Connection +from dbt.adapters.record.cursor.description import CursorGetDescriptionRecord +from dbt.adapters.record.cursor.execute import CursorExecuteRecord +from dbt.adapters.record.cursor.fetchone import CursorFetchOneRecord +from dbt.adapters.record.cursor.fetchmany import CursorFetchManyRecord +from dbt.adapters.record.cursor.fetchall import CursorFetchAllRecord +from dbt.adapters.record.cursor.rowcount import CursorGetRowCountRecord + + +class RecordReplayCursor: + """A proxy object used to wrap native database cursors under record/replay + modes. In record mode, this proxy notes the parameters and return values + of the methods and properties it implements, which closely match the Python + DB API 2.0 cursor methods used by many dbt adapters to interact with the + database or DWH. In replay mode, it mocks out those calls using previously + recorded calls, so that no interaction with a database actually occurs.""" + + def __init__(self, native_cursor: Any, connection: Connection) -> None: + self.native_cursor = native_cursor + self.connection = connection + + @record_function(CursorExecuteRecord, method=True, id_field_name="connection_name") + def execute(self, operation, parameters=None) -> None: + self.native_cursor.execute(operation, parameters) + + @record_function(CursorFetchOneRecord, method=True, id_field_name="connection_name") + def fetchone(self) -> Any: + return self.native_cursor.fetchone() + + @record_function(CursorFetchManyRecord, method=True, id_field_name="connection_name") + def fetchmany(self, size: int) -> Any: + return self.native_cursor.fetchmany(size) + + @record_function(CursorFetchAllRecord, method=True, id_field_name="connection_name") + def fetchall(self) -> Any: + return self.native_cursor.fetchall() + + @property + def connection_name(self) -> Optional[str]: + return self.connection.name + + @property + @record_function(CursorGetRowCountRecord, method=True, id_field_name="connection_name") + def rowcount(self) -> int: + return self.native_cursor.rowcount + + @property + @record_function(CursorGetDescriptionRecord, method=True, id_field_name="connection_name") + def description(self) -> str: + return self.native_cursor.description diff --git a/dbt/adapters/record/cursor/description.py b/dbt/adapters/record/cursor/description.py new file mode 100644 index 00000000..d6ba15d9 --- /dev/null +++ b/dbt/adapters/record/cursor/description.py @@ -0,0 +1,37 @@ +import dataclasses +from typing import Any, Iterable, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetDescriptionParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetDescriptionResult: + columns: Iterable[Any] + + def _to_dict(self) -> Any: + column_dicts = [] + for c in self.columns: + # This captures the mandatory column information, but we might need + # more for some adapters. + # See https://peps.python.org/pep-0249/#description + column_dicts.append((c[0], c[1])) + + return {"columns": column_dicts} + + @classmethod + def _from_dict(cls, dct: Mapping) -> "CursorGetDescriptionResult": + return CursorGetDescriptionResult(columns=dct["columns"]) + + +@Recorder.register_record_type +class CursorGetDescriptionRecord(Record): + """Implements record/replay support for the cursor.description property.""" + + params_cls = CursorGetDescriptionParams + result_cls = CursorGetDescriptionResult + group = "Database" diff --git a/dbt/adapters/record/cursor/execute.py b/dbt/adapters/record/cursor/execute.py new file mode 100644 index 00000000..e7e69859 --- /dev/null +++ b/dbt/adapters/record/cursor/execute.py @@ -0,0 +1,20 @@ +import dataclasses +from typing import Any, Iterable, Union, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorExecuteParams: + connection_name: str + operation: str + parameters: Union[Iterable[Any], Mapping[str, Any]] + + +@Recorder.register_record_type +class CursorExecuteRecord(Record): + """Implements record/replay support for the cursor.execute() method.""" + + params_cls = CursorExecuteParams + result_cls = None + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchall.py b/dbt/adapters/record/cursor/fetchall.py new file mode 100644 index 00000000..090cc160 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchall.py @@ -0,0 +1,66 @@ +import dataclasses +import datetime +from typing import Any, Dict, List, Mapping + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchAllParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchAllResult: + results: List[Any] + + def _to_dict(self) -> Dict[str, Any]: + processed_results = [] + for result in self.results: + result = tuple(map(self._process_value, result)) + processed_results.append(result) + + return {"results": processed_results} + + @classmethod + def _from_dict(cls, dct: Mapping) -> "CursorFetchAllResult": + unprocessed_results = [] + for result in dct["results"]: + result = tuple(map(cls._unprocess_value, result)) + unprocessed_results.append(result) + + return CursorFetchAllResult(unprocessed_results) + + @classmethod + def _process_value(cls, value: Any) -> Any: + if type(value) is datetime.date: + return {"type": "date", "value": value.isoformat()} + elif type(value) is datetime.datetime: + return {"type": "datetime", "value": value.isoformat()} + else: + return value + + @classmethod + def _unprocess_value(cls, value: Any) -> Any: + if type(value) is dict: + value_type = value.get("type") + if value_type == "date": + date_string = value.get("value") + assert isinstance(date_string, str) + return datetime.date.fromisoformat(date_string) + elif value_type == "datetime": + date_string = value.get("value") + assert isinstance(date_string, str) + return datetime.datetime.fromisoformat(date_string) + return value + else: + return value + + +@Recorder.register_record_type +class CursorFetchAllRecord(Record): + """Implements record/replay support for the cursor.fetchall() method.""" + + params_cls = CursorFetchAllParams + result_cls = CursorFetchAllResult + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchmany.py b/dbt/adapters/record/cursor/fetchmany.py new file mode 100644 index 00000000..86f15440 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchmany.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Any, List + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchManyParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchManyResult: + results: List[Any] + + +@Recorder.register_record_type +class CursorFetchManyRecord(Record): + """Implements record/replay support for the cursor.fetchmany() method.""" + + params_cls = CursorFetchManyParams + result_cls = CursorFetchManyResult + group = "Database" diff --git a/dbt/adapters/record/cursor/fetchone.py b/dbt/adapters/record/cursor/fetchone.py new file mode 100644 index 00000000..42ffe210 --- /dev/null +++ b/dbt/adapters/record/cursor/fetchone.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Any + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorFetchOneParams: + connection_name: str + + +@dataclasses.dataclass +class CursorFetchOneResult: + result: Any + + +@Recorder.register_record_type +class CursorFetchOneRecord(Record): + """Implements record/replay support for the cursor.fetchone() method.""" + + params_cls = CursorFetchOneParams + result_cls = CursorFetchOneResult + group = "Database" diff --git a/dbt/adapters/record/cursor/rowcount.py b/dbt/adapters/record/cursor/rowcount.py new file mode 100644 index 00000000..c024817e --- /dev/null +++ b/dbt/adapters/record/cursor/rowcount.py @@ -0,0 +1,23 @@ +import dataclasses +from typing import Optional + +from dbt_common.record import Record, Recorder + + +@dataclasses.dataclass +class CursorGetRowCountParams: + connection_name: str + + +@dataclasses.dataclass +class CursorGetRowCountResult: + rowcount: Optional[int] + + +@Recorder.register_record_type +class CursorGetRowCountRecord(Record): + """Implements record/replay support for the cursor.rowcount property.""" + + params_cls = CursorGetRowCountParams + result_cls = CursorGetRowCountResult + group = "Database" diff --git a/dbt/adapters/record/handle.py b/dbt/adapters/record/handle.py new file mode 100644 index 00000000..31817c37 --- /dev/null +++ b/dbt/adapters/record/handle.py @@ -0,0 +1,24 @@ +from typing import Any + +from dbt.adapters.contracts.connection import Connection + +from dbt.adapters.record.cursor.cursor import RecordReplayCursor + + +class RecordReplayHandle: + """A proxy object used for record/replay modes. What adapters call a + 'handle' is typically a native database connection, but should not be + confused with the Connection protocol, which is a dbt-adapters concept. + + Currently, the only function of the handle proxy is to provide a record/replay + aware cursor object when cursor() is called.""" + + def __init__(self, native_handle: Any, connection: Connection) -> None: + self.native_handle = native_handle + self.connection = connection + + def cursor(self) -> Any: + # The native handle could be None if we are in replay mode, because no + # actual database access should be performed in that mode. + cursor = None if self.native_handle is None else self.native_handle.cursor() + return RecordReplayCursor(cursor, self.connection) diff --git a/dbt/adapters/sql/connections.py b/dbt/adapters/sql/connections.py index 13111a6e..4d450c88 100644 --- a/dbt/adapters/sql/connections.py +++ b/dbt/adapters/sql/connections.py @@ -5,7 +5,6 @@ from dbt_common.events.contextvars import get_node_info from dbt_common.events.functions import fire_event from dbt_common.exceptions import DbtInternalError, NotImplementedError -from dbt_common.record import record_function from dbt_common.utils import cast_to_str from dbt.adapters.base import BaseConnectionManager @@ -20,7 +19,6 @@ SQLQuery, SQLQueryStatus, ) -from dbt.adapters.record import QueryRecord if TYPE_CHECKING: import agate @@ -143,7 +141,6 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> "agate.Tab return table_from_data_flat(data, column_names) - @record_function(QueryRecord, method=True, tuple_result=True) def execute( self, sql: str, diff --git a/docs/guides/record_replay.md b/docs/guides/record_replay.md index 670bb843..5bcbec06 100644 --- a/docs/guides/record_replay.md +++ b/docs/guides/record_replay.md @@ -4,24 +4,12 @@ This document describes how to implement support for dbt's Record/Replay Subsyst ## Recording and Replaying Warehouse Interaction -The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most obvious. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. (It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, and so forth.) +The goal of the Record/Replay Subsystem is to record all interactions between dbt and external systems, of which the data warehouse is the most important. Since, warehouse interaction is mediated by adapters, full Record/Replay support requires that adapters record all interactions they have with the warehouse. It also requires that they record access to the local filesystem or external service, if that is access is not mediated by dbt itself. This includes authentication steps, opening and closing connections, beginning and ending transactions, etc. -In practice, this means that any request sent to the warehouse must be recorded, along with the corresponding response. If this is done correctly, as described in the document linked in the intro, the Record portion of the Record/Replay subsystem should work as expected. - -At the time of this writing, there is only an incomplete implementation of this goal, which can be found in `dbt-adapters/dbt/adapters/record.py`. - -There are some important things to notice about this implementation. First, the QueryRecordResult class provides custom serialization methods `to_dict()` and `from_dict()`. This is necessary because the `AdapterResponse` and `Agate.Table` types cannot be automatically converted to and from JSON by the dataclass library, and JSON is the format used to persist recordings to disk and reload them for replay. - -Another important feature is that `QueryRecordParams` implements the `_matches()` method. This method allows `dbt-adapters` to customize the way that the Record/Replay determines whether a query issued by dbt matches a previously recorded query. In this case, the method performs a comparison which attempts to ignore comments and whitespace which would not affect query behavior. +A basic implementation of Record/Replay functionality, suitable for most adapters which extend the `SQLAdapter` class, can be found in `dbt-adapters/dbt/adapters/record`. The `RecordReplayHandle` and `RecordReplayCursor` classes defined there are used to intercept and record or replay all DWH interactions. They are an excellent starting point for adapters which extend `SQLAdapter` and use a database library which substantially conforms to Python's DB API v2.0 (PEP 249). Examples of how library-specific deviations from that API can be found in the dbt-postgress and dbt-snowflake repositories. ## Misc. Notes and Suggestions -Currently, support for recording data warehouse interaction is very rudimentary, however, even rudimentary support is valuable and we should be concentrating on extending it in a way that adds the most value with the least work. Usefulness, rather than perfection, is the initial goal. - -Picking the right functions to record, at the right level of abstraction, will probably be the most important part of carrying this work forward. - Not every interaction with an external system has to be recorded in full detail, and authentication might prove to be a place where we exclude sensitive secrets from the recording. For example, since replay will not actually be communicating with the warehouse, it may be possible to exclude passwords and auth keys from the parameters recorded, and to exclude auth tokens from the results. In addition to adding an appropriate decorator to functions which communicate with external systems, you should check those functions for side-effects. Since the function's calls will be mocked out in replay mode, those side-effects will not be carried out during replay. At present, we are focusing on support for recording and comparing recordings, but this is worth keeping in mind. - -The current implementation records which dbt node issues a query, and uses that information to ensure a match during replay. The same node should issue the same query. A better model might be to monitor which connection issued which query, and associate the same connection with open/close operations, transaction starts/stops and so forth. From 43ea32d821ed041cdc03175308e1f411211847a3 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 16 Jul 2024 18:20:20 -0400 Subject: [PATCH 23/38] Update contributor guide for `hatch` (#251) --- .github/actions/build-hatch/action.yml | 3 +- .github/workflows/build.yml | 31 --------- .github/workflows/release_prep_hatch.yml | 6 +- .github/workflows/unit-tests.yml | 2 +- .gitignore | 2 +- CONTRIBUTING.md | 88 +++++++++++++----------- dbt-tests-adapter/pyproject.toml | 14 ++-- pyproject.toml | 19 ++--- 8 files changed, 64 insertions(+), 101 deletions(-) diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml index fe9825d4..f2da5935 100644 --- a/.github/actions/build-hatch/action.yml +++ b/.github/actions/build-hatch/action.yml @@ -13,7 +13,7 @@ inputs: default: "./" archive-name: description: Where to upload the artifacts - required: true + default: "" runs: using: composite @@ -30,6 +30,7 @@ runs: working-directory: ${{ inputs.working-dir }} - name: Upload artifacts + if: ${{ inputs.archive-name != '' }} uses: actions/upload-artifact@v3 with: name: ${{ inputs.archive-name }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33d94ff4..00afd704 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,11 +20,6 @@ on: types: [checks_requested] workflow_dispatch: workflow_call: - inputs: - changelog_path: - description: "Path to changelog file" - required: true - type: string permissions: read-all @@ -51,35 +46,9 @@ jobs: uses: ./.github/actions/setup-hatch - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} uses: ./.github/actions/build-hatch - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} uses: ./.github/actions/build-hatch with: working-dir: "./dbt-tests-adapter/" - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - if: ${{ inputs.package == 'dbt-adapters' }} - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - if: ${{ inputs.package == 'dbt-tests-adapter' }} - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" - - # this step is only needed for the release process - - name: "Upload Build Artifact" - if: ${{ github.event_name == 'workflow_call' }} - uses: actions/upload-artifact@v3 - with: - name: ${{ steps.version.outputs.version_number }} - path: | - ${{ inputs.changelog_path }} - ./dist/ - retention-days: 3 diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index b043e19e..ec029ad5 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -34,7 +34,7 @@ # name: Version Bump and Changelog Generation -run-name: Bump ${{ inputs.package }}==${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog +run-name: Bump to ${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog on: workflow_call: inputs: @@ -398,7 +398,7 @@ jobs: - name: "Setup `hatch`" uses: ./.github/actions/setup-hatch - name: "Run Unit Tests" - run: hatch run unit-tests:all + run: hatch run unit-tests run-integration-tests: runs-on: ubuntu-20.04 @@ -447,7 +447,7 @@ jobs: python-version: ${{ env.PYTHON_TARGET_VERSION }} - name: Run tests - run: hatch run integration-tests:all + run: hatch run integration-tests merge-changes-into-target-branch: runs-on: ubuntu-latest diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7d720655..b61c83d7 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -37,7 +37,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run unit tests - run: hatch run unit-tests:all + run: hatch run unit-tests shell: bash - name: Publish results diff --git a/.gitignore b/.gitignore index 31258db2..29c470c5 100644 --- a/.gitignore +++ b/.gitignore @@ -158,7 +158,7 @@ cython_debug/ # MacOS .DS_Store -# vscode +# VSCode .vscode/ .venv/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28f13f6c..1a6e92a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,13 +17,10 @@ This guide assumes users are developing on a Linux or MacOS system. The following utilities are needed or will be installed in this guide: - `pip` -- `virturalenv` +- `hatch` - `git` - `changie` -If local functional testing is required, then a database instance -and appropriate credentials are also required. - In addition to this guide, users are highly encouraged to read the `dbt-core` [CONTRIBUTING.md](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md). Almost all information there is applicable here. @@ -66,25 +63,39 @@ Rather than forking `dbt-labs/dbt-adapters`, use `dbt-labs/dbt-adapters` directl ### Installation -1. Ensure the latest version of `pip` is installed: +1. Ensure the latest versions of `pip` and `hatch` are installed: ```shell - pip install --upgrade pip + pip install --user --upgrade pip hatch + ``` +2. This step is optional, but it's recommended. Configure `hatch` to create its virtual environments in the project. Add this block to your `hatch` `config.toml` file: + ```toml + # MacOS: ~/Library/Application Support/hatch/config.toml + [dirs.env] + virtual = ".hatch" ``` -2. Configure and activate a virtual environment using `virtualenv` as described in -[Setting up an environment](https://github.com/dbt-labs/dbt-core/blob/HEAD/CONTRIBUTING.md#setting-up-an-environment) -3. Install `dbt-adapters` and development dependencies in the virtual environment + This makes `hatch` create all virtual environments in the project root inside of the directory `/.hatch`, similar to `/.tox` for `tox`. + It also makes it easier to add this environment as a runner in common IDEs like VSCode and PyCharm. +3. Create a `hatch` environment with all of the development dependencies and activate it: ```shell - pip install -e .[dev] + hatch run setup + hatch shell + ``` +4. Run any commands within the virtual environment by prefixing the command with `hatch run`: + ```shell + hatch run <command> ``` -When `dbt-adapters` is installed this way, any changes made to the `dbt-adapters` source code -will be reflected in the virtual environment immediately. +## Testing +`dbt-adapters` contains [code quality checks](https://github.com/dbt-labs/dbt-adapters/tree/main/.pre-commit-config.yaml) and [unit tests](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit). +While `dbt-tests-adapter` is also hosted in this repo, it requires a concrete adapter to run. -## Testing +### Code quality -`dbt-adapters` contains [unit](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/unit) -and [functional](https://github.com/dbt-labs/dbt-adapters/tree/main/tests/functional) tests. +Code quality checks can run with a single command: +```shell +hatch run code-quality +``` ### Unit tests @@ -93,43 +104,38 @@ Unit tests can be run locally without setting up a database connection: ```shell # Note: replace $strings with valid names +# run all unit tests +hatch run unit-test + # run all unit tests in a module -python -m pytest tests/unit/$test_file_name.py +hatch run unit-test tests/unit/$test_file_name.py + # run a specific unit test -python -m pytest tests/unit/$test_file_name.py::$test_class_name::$test_method_name +hatch run unit-test tests/unit/$test_file_name.py::$test_class_name::$test_method_name ``` -### Functional tests - -Functional tests require a database to test against. There are two primary ways to run functional tests: +### Testing against a development branch -- Tests will run automatically against a dbt Labs owned database during PR checks -- Tests can be run locally by configuring a `test.env` file with appropriate `ENV` variables: - ```shell - cp test.env.example test.env - $EDITOR test.env - ``` +Some changes require a change in `dbt-common` and `dbt-adapters`. +In that case, the dependency on `dbt-common` must be updated to point to the development branch. For example: -> **_WARNING:_** The parameters in `test.env` must link to a valid database. -> `test.env` is git-ignored, but be _extra_ careful to never check in credentials -> or other sensitive information when developing. +```toml +[tool.hatch.envs.default] +dependencies = [ + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@my-dev-branch", + ..., +] +``` -Functional tests can be run locally with a valid database connection configured in `test.env`: +This will install `dbt-common` as a snapshot. In other words, if `my-dev-branch` is updated on GitHub, those updates will not be reflected locally. +In order to pick up those updates, the `hatch` environment(s) will need to be rebuilt: ```shell -# Note: replace $strings with valid names - -# run all functional tests in a directory -python -m pytest tests/functional/$test_directory -# run all functional tests in a module -python -m pytest tests/functional/$test_dir_and_filename.py -# run all functional tests in a class -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name -# run a specific functional test -python -m pytest tests/functional/$test_dir_and_filename.py::$test_class_name::$test__method_name +exit +hatch env prune +hatch shell ``` - ## Documentation ### User documentation diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index 20342078..73bce49e 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -32,14 +32,6 @@ dependencies = [ "dbt-adapters", "pyyaml", ] - -[project.optional-dependencies] -build = [ - "wheel", - "twine", - "check-wheel-contents", -] - [project.urls] Homepage = "https://github.com/dbt-labs/dbt-adapters" Documentation = "https://docs.getdbt.com" @@ -62,7 +54,11 @@ include = ["dbt/tests", "dbt/__init__.py"] [tool.hatch.envs.build] detached = true -features = ["build"] +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] [tool.hatch.envs.build.scripts] check-all = [ "- check-wheel", diff --git a/pyproject.toml b/pyproject.toml index e50aa63a..55529f34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ maintainers = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] classifiers = [ - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", @@ -43,9 +43,6 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.version] path = "dbt/adapters/__about__.py" @@ -60,20 +57,14 @@ dependencies = [ "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', -] -[tool.hatch.envs.default.scripts] -dev = "pre-commit install" -code-quality = "pre-commit run --all-files" - -[tool.hatch.envs.unit-tests] -dependencies = [ - "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", "pytest", "pytest-dotenv", "pytest-xdist", ] -[tool.hatch.envs.unit-tests.scripts] -all = "python -m pytest {args:tests/unit}" +[tool.hatch.envs.default.scripts] +setup = "pre-commit install" +code-quality = "pre-commit run --all-files" +unit-tests = "python -m pytest {args:tests/unit}" [tool.hatch.envs.build] detached = true From ebf8f37705823c8210e6b23b84946075d0e29a7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:20:41 -0400 Subject: [PATCH 24/38] Bump actions/checkout from 3 to 4 (#170) --- .github/workflows/changelog-existence.yml | 3 --- .github/workflows/release_prep_hatch.yml | 10 +++++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml index d778f565..8732177f 100644 --- a/.github/workflows/changelog-existence.yml +++ b/.github/workflows/changelog-existence.yml @@ -19,9 +19,6 @@ name: Check Changelog Entry on: pull_request_target: types: [opened, reopened, labeled, unlabeled, synchronize] - paths-ignore: ['.changes/**', '.github/**', 'tests/**', 'third-party-stubs/**', '**.md', '**.yml'] - - workflow_dispatch: defaults: run: diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index ec029ad5..a13d45ce 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -288,7 +288,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} - name: Setup `hatch` @@ -392,7 +392,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} - name: "Setup `hatch`" @@ -407,7 +407,7 @@ jobs: steps: - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.create-temp-branch.outputs.branch_name }} @@ -467,7 +467,7 @@ jobs: echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: "Merge Changes Into ${{ inputs.target_branch }}" uses: everlytic/branch-merge@1.1.5 @@ -524,7 +524,7 @@ jobs: message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ steps.resolve_branch.outputs.target_branch }} From f76375b333615e05b2ee5f1bb2909e9b16da1dcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:22:17 -0400 Subject: [PATCH 25/38] Bump actions/download-artifact from 3 to 4 (#168) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/github-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index 1c2f41b5..ad0cc2d8 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -208,7 +208,7 @@ jobs: ref: ${{ inputs.sha }} - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive_name }} path: dist/ From 4b3806818f3831bbd0206a6b22fdcb4fe2a505c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 21:25:32 +0000 Subject: [PATCH 26/38] Bump dbt-labs/actions from 1.1.0 to 1.1.1 (#169) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/release_prep_hatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml index a13d45ce..a6105786 100644 --- a/.github/workflows/release_prep_hatch.yml +++ b/.github/workflows/release_prep_hatch.yml @@ -131,7 +131,7 @@ jobs: - name: "Audit Version And Parse Into Parts" id: semver - uses: dbt-labs/actions/parse-semver@v1.1.0 + uses: dbt-labs/actions/parse-semver@v1.1.1 with: version: ${{ inputs.version_number }} From e3465c2a7baaea66f71b1da94647a193c281a992 Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:12:07 -0700 Subject: [PATCH 27/38] Correct CHANGELOG.md (#264) Co-authored-by: Mike Alfare <mike.alfare@dbtlabs.com> Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .changes/0.1.0/Docs-20240109-131629.yaml | 6 -- .changes/0.1.0/Docs-20240109-131736.yaml | 6 -- .changes/0.1.0/Docs-20240109-131858.yaml | 6 -- .changes/0.1.0/Docs-20240109-131917.yaml | 6 -- .changes/0.1.0/Features-20240212-123544.yaml | 6 -- .changes/0.1.0/Fixes-20240215-141545.yaml | 6 -- .changes/0.1.0/Fixes-20240216-135420.yaml | 6 -- .../0.1.0/Under the Hood-20240109-131958.yaml | 6 -- .../0.1.0/Under the Hood-20240112-230236.yaml | 6 -- .../0.1.0/Under the Hood-20240123-121220.yaml | 6 -- .../0.1.0/Under the Hood-20240220-164223.yaml | 6 -- .changes/1.0.0.md | 27 +++++- .changes/1.1.0.md | 26 +++--- .changes/1.1.1.md | 2 +- .changes/1.2.1.md | 11 ++- .changes/1.3.0.md | 2 +- .changes/1.3.2.md | 4 +- .changes/1.3.3.md | 4 +- .changes/1.8.0.md | 9 -- .changes/1.9.0.md | 1 - .changes/1.9.1.md | 1 - .changes/1.9.2.md | 1 - .changie.yaml | 61 ++++++++++-- CHANGELOG.md | 92 ++++++++++--------- 24 files changed, 150 insertions(+), 157 deletions(-) delete mode 100644 .changes/0.1.0/Docs-20240109-131629.yaml delete mode 100644 .changes/0.1.0/Docs-20240109-131736.yaml delete mode 100644 .changes/0.1.0/Docs-20240109-131858.yaml delete mode 100644 .changes/0.1.0/Docs-20240109-131917.yaml delete mode 100644 .changes/0.1.0/Features-20240212-123544.yaml delete mode 100644 .changes/0.1.0/Fixes-20240215-141545.yaml delete mode 100644 .changes/0.1.0/Fixes-20240216-135420.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240109-131958.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240112-230236.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240123-121220.yaml delete mode 100644 .changes/0.1.0/Under the Hood-20240220-164223.yaml delete mode 100644 .changes/1.8.0.md delete mode 100644 .changes/1.9.0.md delete mode 100644 .changes/1.9.1.md delete mode 100644 .changes/1.9.2.md diff --git a/.changes/0.1.0/Docs-20240109-131629.yaml b/.changes/0.1.0/Docs-20240109-131629.yaml deleted file mode 100644 index 22b2ad3f..00000000 --- a/.changes/0.1.0/Docs-20240109-131629.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Configure `changie` -time: 2024-01-09T13:16:29.763021-05:00 -custom: - Author: mikealfare - Issue: 16 diff --git a/.changes/0.1.0/Docs-20240109-131736.yaml b/.changes/0.1.0/Docs-20240109-131736.yaml deleted file mode 100644 index 43186903..00000000 --- a/.changes/0.1.0/Docs-20240109-131736.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Setup ADR tracking framework -time: 2024-01-09T13:17:36.094147-05:00 -custom: - Author: mikealfare - Issue: "11" diff --git a/.changes/0.1.0/Docs-20240109-131858.yaml b/.changes/0.1.0/Docs-20240109-131858.yaml deleted file mode 100644 index decef9a7..00000000 --- a/.changes/0.1.0/Docs-20240109-131858.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Create issue templates -time: 2024-01-09T13:18:58.11819-05:00 -custom: - Author: mikealfare - Issue: "12" diff --git a/.changes/0.1.0/Docs-20240109-131917.yaml b/.changes/0.1.0/Docs-20240109-131917.yaml deleted file mode 100644 index 3c531060..00000000 --- a/.changes/0.1.0/Docs-20240109-131917.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Docs -body: Create PR template -time: 2024-01-09T13:19:17.749914-05:00 -custom: - Author: mikealfare - Issue: "13" diff --git a/.changes/0.1.0/Features-20240212-123544.yaml b/.changes/0.1.0/Features-20240212-123544.yaml deleted file mode 100644 index 239ad59f..00000000 --- a/.changes/0.1.0/Features-20240212-123544.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Update RelationConfig to capture all fields used by adapters -time: 2024-02-12T12:35:44.653555-08:00 -custom: - Author: colin-rogers-dbt - Issue: "30" diff --git a/.changes/0.1.0/Fixes-20240215-141545.yaml b/.changes/0.1.0/Fixes-20240215-141545.yaml deleted file mode 100644 index ced62f25..00000000 --- a/.changes/0.1.0/Fixes-20240215-141545.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Ignore adapter-level support warnings for 'custom' constraints -time: 2024-02-15T14:15:45.764145+01:00 -custom: - Author: jtcohen6 - Issue: "90" diff --git a/.changes/0.1.0/Fixes-20240216-135420.yaml b/.changes/0.1.0/Fixes-20240216-135420.yaml deleted file mode 100644 index a04cd26b..00000000 --- a/.changes/0.1.0/Fixes-20240216-135420.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Make all adapter zone tests importable by removing "Test" prefix -time: 2024-02-16T13:54:20.411864-05:00 -custom: - Author: mikealfare - Issue: "93" diff --git a/.changes/0.1.0/Under the Hood-20240109-131958.yaml b/.changes/0.1.0/Under the Hood-20240109-131958.yaml deleted file mode 100644 index a062a299..00000000 --- a/.changes/0.1.0/Under the Hood-20240109-131958.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Configure `dependabot` -time: 2024-01-09T13:19:58.060742-05:00 -custom: - Author: mikealfare - Issue: "14" diff --git a/.changes/0.1.0/Under the Hood-20240112-230236.yaml b/.changes/0.1.0/Under the Hood-20240112-230236.yaml deleted file mode 100644 index 1470ac6e..00000000 --- a/.changes/0.1.0/Under the Hood-20240112-230236.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Implement unit testing in CI -time: 2024-01-12T23:02:36.630106-05:00 -custom: - Author: mikealfare - Issue: "10" diff --git a/.changes/0.1.0/Under the Hood-20240123-121220.yaml b/.changes/0.1.0/Under the Hood-20240123-121220.yaml deleted file mode 100644 index 8d01f256..00000000 --- a/.changes/0.1.0/Under the Hood-20240123-121220.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Allow version to be specified in either __version__.py or __about__.py -time: 2024-01-23T12:12:20.529147-05:00 -custom: - Author: mikealfare - Issue: "44" diff --git a/.changes/0.1.0/Under the Hood-20240220-164223.yaml b/.changes/0.1.0/Under the Hood-20240220-164223.yaml deleted file mode 100644 index eefa441e..00000000 --- a/.changes/0.1.0/Under the Hood-20240220-164223.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove __init__.py file from dbt.tests -time: 2024-02-20T16:42:23.706-05:00 -custom: - Author: gshank - Issue: "96" diff --git a/.changes/1.0.0.md b/.changes/1.0.0.md index d381d635..c46c8148 100644 --- a/.changes/1.0.0.md +++ b/.changes/1.0.0.md @@ -1,15 +1,32 @@ ## dbt-adapters 1.0.0 - April 01, 2024 +### Features + +* Update RelationConfig to capture all fields used by adapters ([#30](https://github.com/dbt-labs/dbt-adapters/issues/30)) + ### Fixes -* Add field wrapper to BaseRelation members that were missing it. -* Add "description" and "meta" fields to RelationConfig protocol +* Add field wrapper to BaseRelation members that were missing it. ([#108](https://github.com/dbt-labs/dbt-adapters/issues/108)) +* Add "description" and "meta" fields to RelationConfig protocol ([#119](https://github.com/dbt-labs/dbt-adapters/issues/119)) +* Ignore adapter-level support warnings for 'custom' constraints ([#90](https://github.com/dbt-labs/dbt-adapters/issues/90)) +* Make all adapter zone tests importable by removing "Test" prefix ([#93](https://github.com/dbt-labs/dbt-adapters/issues/93)) + +### Docs + +* Configure `changie` ([#16](https://github.com/dbt-labs/dbt-adapters/issues/16)) +* Setup ADR tracking framework ([#11](https://github.com/dbt-labs/dbt-adapters/issues/11)) +* Create issue templates ([#12](https://github.com/dbt-labs/dbt-adapters/issues/12)) +* Create PR template ([#13](https://github.com/dbt-labs/dbt-adapters/issues/13)) ### Under the Hood -* Lazy load agate to improve dbt-core performance -* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS +* Lazy load agate to improve dbt-core performance ([#125](https://github.com/dbt-labs/dbt-adapters/issues/125)) +* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS ([#131](https://github.com/dbt-labs/dbt-adapters/issues/131)) +* Configure `dependabot` ([#14](https://github.com/dbt-labs/dbt-adapters/issues/14)) +* Implement unit testing in CI ([#22](https://github.com/dbt-labs/dbt-adapters/issues/22)) +* Allow version to be specified in either __version__.py or __about__.py ([#44](https://github.com/dbt-labs/dbt-adapters/issues/44)) +* Remove __init__.py file from dbt.tests ([#96](https://github.com/dbt-labs/dbt-adapters/issues/96)) ### Security -* Pin `black>=24.3` in `pyproject.toml` +* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) diff --git a/.changes/1.1.0.md b/.changes/1.1.0.md index 9e7db78e..224d8e85 100644 --- a/.changes/1.1.0.md +++ b/.changes/1.1.0.md @@ -2,28 +2,28 @@ ### Features -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing -* Cross-database `cast` macro -* Allow adapters to opt out of aliasing the subquery generated by render_limited -* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing +* Debug log when `type_code` fails to convert to a `data_type` ([#135](https://github.com/dbt-labs/dbt-adapters/issues/135)) +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch ([#127](https://github.com/dbt-labs/dbt-adapters/issues/127)) +* Support for sql fixtures in unit testing ([#146](https://github.com/dbt-labs/dbt-adapters/issues/146)) +* Cross-database `cast` macro ([#173](https://github.com/dbt-labs/dbt-adapters/issues/173)) +* Allow adapters to opt out of aliasing the subquery generated by render_limited ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) ### Fixes -* Fix adapter-specific cast handling for constraint enforcement +* Fix adapter-specific cast handling for constraint enforcement ([#165](https://github.com/dbt-labs/dbt-adapters/issues/165)) ### Docs -* Use `dbt-adapters` throughout the contributing guide +* Use `dbt-adapters` throughout the contributing guide ([#137](https://github.com/dbt-labs/dbt-adapters/issues/137)) ### Under the Hood -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA -* Validate that dbt-core and dbt-adapters remain de-coupled -* remove dbt_version from query comment test fixture +* Add the option to set the log level of the AdapterRegistered event ([#141](https://github.com/dbt-labs/dbt-adapters/issues/141)) +* Update dependabot config to cover GHA ([#161](https://github.com/dbt-labs/dbt-adapters/issues/161)) +* Validate that dbt-core and dbt-adapters remain de-coupled ([#174](https://github.com/dbt-labs/dbt-adapters/issues/174)) +* remove dbt_version from query comment test fixture ([#184](https://github.com/dbt-labs/dbt-adapters/issues/184)) ### Dependencies -* add support for py3.12 +* add support for py3.12 ([#185](https://github.com/dbt-labs/dbt-adapters/issues/185)) diff --git a/.changes/1.1.1.md b/.changes/1.1.1.md index a1c38c88..9e590f94 100644 --- a/.changes/1.1.1.md +++ b/.changes/1.1.1.md @@ -2,4 +2,4 @@ ### Features -* Enable serialization contexts +* Enable serialization contexts ([#197](https://github.com/dbt-labs/dbt-adapters/issues/197)) diff --git a/.changes/1.2.1.md b/.changes/1.2.1.md index 2aac5674..e554b90b 100644 --- a/.changes/1.2.1.md +++ b/.changes/1.2.1.md @@ -2,9 +2,14 @@ ### Features -* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist ([#203](https://github.com/dbt-labs/dbt-adapters/issues/203)) +* Cross-database `date` macro ([#191](https://github.com/dbt-labs/dbt-adapters/issues/191)) + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes ([#194](https://github.com/dbt-labs/dbt-adapters/issues/194)) ### Under the Hood -* Add query recording for adapters which use SQLConnectionManager -* Improve memory efficiency of process_results() +* Add query recording for adapters which use SQLConnectionManager ([#195](https://github.com/dbt-labs/dbt-adapters/issues/195)) +* Improve memory efficiency of process_results() ([#217](https://github.com/dbt-labs/dbt-adapters/issues/217)) diff --git a/.changes/1.3.0.md b/.changes/1.3.0.md index dae1f819..6a23c3ba 100644 --- a/.changes/1.3.0.md +++ b/.changes/1.3.0.md @@ -2,4 +2,4 @@ ### Features -* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation ([#231](https://github.com/dbt-labs/dbt-adapters/issues/231)) diff --git a/.changes/1.3.2.md b/.changes/1.3.2.md index bdc7f593..6963a4c3 100644 --- a/.changes/1.3.2.md +++ b/.changes/1.3.2.md @@ -2,5 +2,5 @@ ### Under the Hood -* Fix query timer resolution -* Add optional release_connection parameter to connection_named method +* Fix query timer resolution ([#246](https://github.com/dbt-labs/dbt-adapters/issues/246)) +* Add optional release_connection parameter to connection_named method ([#247](https://github.com/dbt-labs/dbt-adapters/issues/247)) diff --git a/.changes/1.3.3.md b/.changes/1.3.3.md index 22ab18e3..c62a0562 100644 --- a/.changes/1.3.3.md +++ b/.changes/1.3.3.md @@ -2,8 +2,8 @@ ### Fixes -* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#213](https://github.com/dbt-labs/dbt-adapters/issues/213)) ### Under the Hood -* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. ([#207](https://github.com/dbt-labs/dbt-adapters/issues/207)) diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md deleted file mode 100644 index e781a985..00000000 --- a/.changes/1.8.0.md +++ /dev/null @@ -1,9 +0,0 @@ -## dbt-adapters 1.8.0 - May 09, 2024 - -### Features - -* Cross-database `date` macro - -### Fixes - -* Update Clone test to reflect core change removing `deferred` attribute from nodes diff --git a/.changes/1.9.0.md b/.changes/1.9.0.md deleted file mode 100644 index 2f016563..00000000 --- a/.changes/1.9.0.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapters 1.9.0 - June 18, 2024 diff --git a/.changes/1.9.1.md b/.changes/1.9.1.md deleted file mode 100644 index 900e6b75..00000000 --- a/.changes/1.9.1.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapters 1.9.1 - June 20, 2024 diff --git a/.changes/1.9.2.md b/.changes/1.9.2.md deleted file mode 100644 index 26505f20..00000000 --- a/.changes/1.9.2.md +++ /dev/null @@ -1 +0,0 @@ -## dbt-adapters 1.9.2 - July 09, 2024 diff --git a/.changie.yaml b/.changie.yaml index afbafb22..1198cfe0 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -1,27 +1,70 @@ changesDir: .changes unreleasedDir: unreleased headerPath: header.tpl.md +versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md envPrefix: CHANGIE_ versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' -changeFormat: '* {{.Body}}' +changeFormat: |- + {{- $IssueList := list }} + {{- $changes := splitList " " $.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) kinds: - - label: Breaking Changes - - label: Features - - label: Fixes - - label: Docs - - label: Under the Hood - - label: Dependencies - - label: Security +- label: Breaking Changes +- label: Features +- label: Fixes +- label: Docs +- label: Under the Hood +- label: Dependencies + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 +- label: Security + changeFormat: |- + {{- $PRList := list }} + {{- $changes := splitList " " $.Custom.PR }} + {{- range $pullrequest := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $pullrequest }} + {{- $PRList = append $PRList $changeLink }} + {{- end -}} + - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) + skipGlobalChoices: true + additionalChoices: + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 newlines: afterChangelogHeader: 1 afterKind: 1 afterChangelogVersion: 1 beforeKind: 1 endOfVersion: 1 - custom: - key: Author label: GitHub Username(s) (separated by a single space if multiple) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a2f1833..8bd48b5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,38 +5,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). -## dbt-adapters 1.9.2 - July 09, 2024 - -## dbt-adapters 1.9.1 - June 20, 2024 - -## dbt-adapters 1.9.0 - June 18, 2024 - -## dbt-adapters 1.8.0 - May 09, 2024 - -### Features - -* Cross-database `date` macro - -### Fixes - -* Update Clone test to reflect core change removing `deferred` attribute from nodes - ## dbt-adapters 1.3.3 - July 09, 2024 ### Fixes -* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses +* Fix scenario where using the `--empty` flag causes metadata queries to contain limit clauses ([#213](https://github.com/dbt-labs/dbt-adapters/issues/213)) ### Under the Hood -* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. +* --limit flag no longer subshells the query. This resolves the dbt Cloud experience issue where limit prevents ordering elements.. ([#207](https://github.com/dbt-labs/dbt-adapters/issues/207)) ## dbt-adapters 1.3.2 - July 02, 2024 ### Under the Hood -* Fix query timer resolution -* Add optional release_connection parameter to connection_named method +* Fix query timer resolution ([#246](https://github.com/dbt-labs/dbt-adapters/issues/246)) +* Add optional release_connection parameter to connection_named method ([#247](https://github.com/dbt-labs/dbt-adapters/issues/247)) ## dbt-adapters 1.3.1 - June 20, 2024 @@ -44,67 +28,89 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ### Features -* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation +* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation ([#231](https://github.com/dbt-labs/dbt-adapters/issues/231)) ## dbt-adapters 1.2.1 - May 21, 2024 ### Features -* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist +* Improvement of the compile error message in the get_fixture-sql.sql when the relation or the model not exist ([#203](https://github.com/dbt-labs/dbt-adapters/issues/203)) +* Cross-database `date` macro ([#191](https://github.com/dbt-labs/dbt-adapters/issues/191)) + +### Fixes + +* Update Clone test to reflect core change removing `deferred` attribute from nodes ([#194](https://github.com/dbt-labs/dbt-adapters/issues/194)) ### Under the Hood -* Add query recording for adapters which use SQLConnectionManager -* Improve memory efficiency of process_results() +* Add query recording for adapters which use SQLConnectionManager ([#195](https://github.com/dbt-labs/dbt-adapters/issues/195)) +* Improve memory efficiency of process_results() ([#217](https://github.com/dbt-labs/dbt-adapters/issues/217)) ## dbt-adapters 1.1.1 - May 07, 2024 ### Features -* Enable serialization contexts +* Enable serialization contexts ([#197](https://github.com/dbt-labs/dbt-adapters/issues/197)) ## dbt-adapters 1.1.0 - May 01, 2024 ### Features -* Debug log when `type_code` fails to convert to a `data_type` -* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch -* Support for sql fixtures in unit testing -* Cross-database `cast` macro -* Allow adapters to opt out of aliasing the subquery generated by render_limited -* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing +* Debug log when `type_code` fails to convert to a `data_type` ([#135](https://github.com/dbt-labs/dbt-adapters/issues/135)) +* Introduce TableLastModifiedMetadataBatch and implement BaseAdapter.calculate_freshness_from_metadata_batch ([#127](https://github.com/dbt-labs/dbt-adapters/issues/127)) +* Support for sql fixtures in unit testing ([#146](https://github.com/dbt-labs/dbt-adapters/issues/146)) +* Cross-database `cast` macro ([#173](https://github.com/dbt-labs/dbt-adapters/issues/173)) +* Allow adapters to opt out of aliasing the subquery generated by render_limited ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) +* subquery alias generated by render_limited now includes the relation name to mitigate duplicate aliasing ([#179](https://github.com/dbt-labs/dbt-adapters/issues/179)) ### Fixes -* Fix adapter-specific cast handling for constraint enforcement +* Fix adapter-specific cast handling for constraint enforcement ([#165](https://github.com/dbt-labs/dbt-adapters/issues/165)) ### Docs -* Use `dbt-adapters` throughout the contributing guide +* Use `dbt-adapters` throughout the contributing guide ([#137](https://github.com/dbt-labs/dbt-adapters/issues/137)) ### Under the Hood -* Add the option to set the log level of the AdapterRegistered event -* Update dependabot config to cover GHA -* Validate that dbt-core and dbt-adapters remain de-coupled -* remove dbt_version from query comment test fixture +* Add the option to set the log level of the AdapterRegistered event ([#141](https://github.com/dbt-labs/dbt-adapters/issues/141)) +* Update dependabot config to cover GHA ([#161](https://github.com/dbt-labs/dbt-adapters/issues/161)) +* Validate that dbt-core and dbt-adapters remain de-coupled ([#174](https://github.com/dbt-labs/dbt-adapters/issues/174)) +* remove dbt_version from query comment test fixture ([#184](https://github.com/dbt-labs/dbt-adapters/issues/184)) ### Dependencies -* add support for py3.12 +* add support for py3.12 ([#185](https://github.com/dbt-labs/dbt-adapters/issues/185)) ## dbt-adapters 1.0.0 - April 01, 2024 +### Features + +* Update RelationConfig to capture all fields used by adapters ([#30](https://github.com/dbt-labs/dbt-adapters/issues/30)) + ### Fixes -* Add field wrapper to BaseRelation members that were missing it. -* Add "description" and "meta" fields to RelationConfig protocol +* Add field wrapper to BaseRelation members that were missing it. ([#108](https://github.com/dbt-labs/dbt-adapters/issues/108)) +* Add "description" and "meta" fields to RelationConfig protocol ([#119](https://github.com/dbt-labs/dbt-adapters/issues/119)) +* Ignore adapter-level support warnings for 'custom' constraints ([#90](https://github.com/dbt-labs/dbt-adapters/issues/90)) +* Make all adapter zone tests importable by removing "Test" prefix ([#93](https://github.com/dbt-labs/dbt-adapters/issues/93)) + +### Docs + +* Configure `changie` ([#16](https://github.com/dbt-labs/dbt-adapters/issues/16)) +* Setup ADR tracking framework ([#11](https://github.com/dbt-labs/dbt-adapters/issues/11)) +* Create issue templates ([#12](https://github.com/dbt-labs/dbt-adapters/issues/12)) +* Create PR template ([#13](https://github.com/dbt-labs/dbt-adapters/issues/13)) ### Under the Hood -* Lazy load agate to improve dbt-core performance -* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS +* Lazy load agate to improve dbt-core performance ([#125](https://github.com/dbt-labs/dbt-adapters/issues/125)) +* add BaseAdapater.MAX_SCHEMA_METADATA_RELATIONS ([#131](https://github.com/dbt-labs/dbt-adapters/issues/131)) +* Configure `dependabot` ([#14](https://github.com/dbt-labs/dbt-adapters/issues/14)) +* Implement unit testing in CI ([#22](https://github.com/dbt-labs/dbt-adapters/issues/22)) +* Allow version to be specified in either __version__.py or __about__.py ([#44](https://github.com/dbt-labs/dbt-adapters/issues/44)) +* Remove __init__.py file from dbt.tests ([#96](https://github.com/dbt-labs/dbt-adapters/issues/96)) ### Security -* Pin `black>=24.3` in `pyproject.toml` +* Pin `black>=24.3` in `pyproject.toml` ([#140](https://github.com/dbt-labs/dbt-adapters/issues/140)) From b088e518477ad7023c014eec5d9fad36a273ef86 Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:02:25 -0700 Subject: [PATCH 28/38] Adap 821/support for redshift 821 (#270) Co-authored-by: Mila Page <versusfacit@users.noreply.github.com> --- .../macros/unit_test_sql/get_fixture_sql.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql index c25a87f7..a3a8173b 100644 --- a/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql +++ b/dbt/include/global_project/macros/unit_test_sql/get_fixture_sql.sql @@ -22,6 +22,7 @@ {%- do default_row.update({column_name: (safe_cast("null", column_type) | trim )}) -%} {%- endfor -%} +{{ validate_fixture_rows(rows, row_number) }} {%- for row in rows -%} {%- set formatted_row = format_row(row, column_name_to_data_types) -%} @@ -93,3 +94,11 @@ union all {%- endfor -%} {{ return(formatted_row) }} {%- endmacro -%} + +{%- macro validate_fixture_rows(rows, row_number) -%} + {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }} +{%- endmacro -%} + +{%- macro default__validate_fixture_rows(rows, row_number) -%} + {# This is an abstract method for adapter overrides as needed #} +{%- endmacro -%} From d23b9e8dc9fc236ce9eae5519c39b19cf758f276 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:11:56 -0400 Subject: [PATCH 29/38] Make `dependabot` ignore patch updates (#267) --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 02f010c7..907926a3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,13 +5,25 @@ updates: schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "pip" directory: "/dbt-tests-adapter" schedule: interval: "daily" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" rebase-strategy: "disabled" + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-patch From fbfa095c2f3c567b636de601b099f792021d0144 Mon Sep 17 00:00:00 2001 From: Michelle Ark <MichelleArk@users.noreply.github.com> Date: Thu, 25 Jul 2024 10:56:28 -0400 Subject: [PATCH 30/38] render to and to_columns on column and model level fk constraints (#260) --- .../unreleased/Features-20240723-104221.yaml | 6 ++++ .../dbt/tests/adapter/constraints/fixtures.py | 3 +- dbt/adapters/base/impl.py | 34 +++++++++++++------ tests/unit/test_base_adapter.py | 32 +++++++++++++++++ 4 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 .changes/unreleased/Features-20240723-104221.yaml diff --git a/.changes/unreleased/Features-20240723-104221.yaml b/.changes/unreleased/Features-20240723-104221.yaml new file mode 100644 index 00000000..cc038fd4 --- /dev/null +++ b/.changes/unreleased/Features-20240723-104221.yaml @@ -0,0 +1,6 @@ +kind: Features +body: render 'to' and 'to_columns' fields on foreign key constraints +time: 2024-07-23T10:42:21.222203-04:00 +custom: + Author: michelleark + Issue: "271" diff --git a/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py b/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py index cfbd5379..2a4f089b 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py +++ b/dbt-tests-adapter/dbt/tests/adapter/constraints/fixtures.py @@ -363,7 +363,8 @@ - type: check expression: id >= 1 - type: foreign_key - expression: {schema}.foreign_key_model (id) + to: ref('foreign_key_model') + to_columns: ["id"] - type: unique data_tests: - unique diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index ec590137..e0627c47 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -1602,8 +1602,13 @@ def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional rendered_column_constraint = f"unique {constraint_expression}" elif constraint.type == ConstraintType.primary_key: rendered_column_constraint = f"primary key {constraint_expression}" - elif constraint.type == ConstraintType.foreign_key and constraint_expression: - rendered_column_constraint = f"references {constraint_expression}" + elif constraint.type == ConstraintType.foreign_key: + if constraint.to and constraint.to_columns: + rendered_column_constraint = ( + f"references {constraint.to} ({', '.join(constraint.to_columns)})" + ) + elif constraint_expression: + rendered_column_constraint = f"references {constraint_expression}" elif constraint.type == ConstraintType.custom and constraint_expression: rendered_column_constraint = constraint_expression @@ -1682,20 +1687,29 @@ def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[s rendering.""" constraint_prefix = f"constraint {constraint.name} " if constraint.name else "" column_list = ", ".join(constraint.columns) + rendered_model_constraint = None + if constraint.type == ConstraintType.check and constraint.expression: - return f"{constraint_prefix}check ({constraint.expression})" + rendered_model_constraint = f"{constraint_prefix}check ({constraint.expression})" elif constraint.type == ConstraintType.unique: constraint_expression = f" {constraint.expression}" if constraint.expression else "" - return f"{constraint_prefix}unique{constraint_expression} ({column_list})" + rendered_model_constraint = ( + f"{constraint_prefix}unique{constraint_expression} ({column_list})" + ) elif constraint.type == ConstraintType.primary_key: constraint_expression = f" {constraint.expression}" if constraint.expression else "" - return f"{constraint_prefix}primary key{constraint_expression} ({column_list})" - elif constraint.type == ConstraintType.foreign_key and constraint.expression: - return f"{constraint_prefix}foreign key ({column_list}) references {constraint.expression}" + rendered_model_constraint = ( + f"{constraint_prefix}primary key{constraint_expression} ({column_list})" + ) + elif constraint.type == ConstraintType.foreign_key: + if constraint.to and constraint.to_columns: + rendered_model_constraint = f"{constraint_prefix}foreign key ({column_list}) references {constraint.to} ({', '.join(constraint.to_columns)})" + elif constraint.expression: + rendered_model_constraint = f"{constraint_prefix}foreign key ({column_list}) references {constraint.expression}" elif constraint.type == ConstraintType.custom and constraint.expression: - return f"{constraint_prefix}{constraint.expression}" - else: - return None + rendered_model_constraint = f"{constraint_prefix}{constraint.expression}" + + return rendered_model_constraint @classmethod def capabilities(cls) -> CapabilityDict: diff --git a/tests/unit/test_base_adapter.py b/tests/unit/test_base_adapter.py index 95fe5ae2..5fa109b7 100644 --- a/tests/unit/test_base_adapter.py +++ b/tests/unit/test_base_adapter.py @@ -39,6 +39,14 @@ def connection_manager(self): [{"type": "foreign_key", "expression": "other_table (c1)"}], ["column_name integer references other_table (c1)"], ), + ( + [{"type": "foreign_key", "to": "other_table", "to_columns": ["c1"]}], + ["column_name integer references other_table (c1)"], + ), + ( + [{"type": "foreign_key", "to": "other_table", "to_columns": ["c1", "c2"]}], + ["column_name integer references other_table (c1, c2)"], + ), ([{"type": "check"}, {"type": "unique"}], ["column_name integer unique"]), ([{"type": "custom", "expression": "-- noop"}], ["column_name integer -- noop"]), ] @@ -176,6 +184,30 @@ def test_render_raw_columns_constraints_unsupported( ], ["constraint test_name foreign key (c1, c2) references other_table (c1)"], ), + ( + [ + { + "type": "foreign_key", + "columns": ["c1", "c2"], + "to": "other_table", + "to_columns": ["c1"], + "name": "test_name", + } + ], + ["constraint test_name foreign key (c1, c2) references other_table (c1)"], + ), + ( + [ + { + "type": "foreign_key", + "columns": ["c1", "c2"], + "to": "other_table", + "to_columns": ["c1", "c2"], + "name": "test_name", + } + ], + ["constraint test_name foreign key (c1, c2) references other_table (c1, c2)"], + ), ] @pytest.mark.parametrize("constraints,expected_rendered_constraints", model_constraints) From 4b2755dd129c54f04c3d467cd7e83a46d0afc01a Mon Sep 17 00:00:00 2001 From: Michelle Ark <MichelleArk@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:36:29 -0400 Subject: [PATCH 31/38] bump dbt-common min to 1.6 (#272) --- .changes/unreleased/Features-20240723-104221.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changes/unreleased/Features-20240723-104221.yaml b/.changes/unreleased/Features-20240723-104221.yaml index cc038fd4..45499971 100644 --- a/.changes/unreleased/Features-20240723-104221.yaml +++ b/.changes/unreleased/Features-20240723-104221.yaml @@ -1,5 +1,5 @@ kind: Features -body: render 'to' and 'to_columns' fields on foreign key constraints +body: render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 time: 2024-07-23T10:42:21.222203-04:00 custom: Author: michelleark diff --git a/pyproject.toml b/pyproject.toml index 55529f34..e794781c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common>=1.3,<2.0", + "dbt-common>=1.6,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", From b64d3a5efd83b8a3ab0612145f6bdeaa8cb0f2ba Mon Sep 17 00:00:00 2001 From: Gerda Shank <gerda.shank@gmail.com> Date: Mon, 29 Jul 2024 16:29:59 -0400 Subject: [PATCH 32/38] When using incremental tables, the column data types are invalidly updated (#269) --- .changes/unreleased/Fixes-20240728-221030.yaml | 6 ++++++ dbt/adapters/base/column.py | 3 --- .../materializations/models/incremental/incremental.sql | 9 ++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240728-221030.yaml diff --git a/.changes/unreleased/Fixes-20240728-221030.yaml b/.changes/unreleased/Fixes-20240728-221030.yaml new file mode 100644 index 00000000..a1cddb24 --- /dev/null +++ b/.changes/unreleased/Fixes-20240728-221030.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Incremental table varchar column definition changed +time: 2024-07-28T22:10:30.673107-04:00 +custom: + Author: gshank + Issue: "276" diff --git a/dbt/adapters/base/column.py b/dbt/adapters/base/column.py index e2e6e1e0..195684a4 100644 --- a/dbt/adapters/base/column.py +++ b/dbt/adapters/base/column.py @@ -123,9 +123,6 @@ def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str: else: return "{}({},{})".format(dtype, precision, scale) - def __repr__(self) -> str: - return "<Column {} ({})>".format(self.name, self.data_type) - @classmethod def from_description(cls, name: str, raw_data_type: str) -> "Column": match = re.match(r"([^(]+)(\([^)]+\))?", raw_data_type) diff --git a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql index e8ff5c1e..f932751a 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql @@ -39,9 +39,12 @@ {% set need_swap = true %} {% else %} {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} - {% do adapter.expand_target_column_types( - from_relation=temp_relation, - to_relation=target_relation) %} + {% set contract_config = config.get('contract') %} + {% if not contract_config or not contract_config.enforced %} + {% do adapter.expand_target_column_types( + from_relation=temp_relation, + to_relation=target_relation) %} + {% endif %} {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} {% if not dest_columns %} From 1799db9bf49414be995abd000e2bca3e9bfd8064 Mon Sep 17 00:00:00 2001 From: Github Build Bot <buildbot@fishtownanalytics.com> Date: Tue, 30 Jul 2024 16:29:05 +0000 Subject: [PATCH 33/38] Bumping version to 1.4.0 and generate changelog --- .changes/1.4.0.md | 13 +++++++++++++ .changes/unreleased/Features-20240723-104221.yaml | 6 ------ .changes/unreleased/Fixes-20240728-221030.yaml | 6 ------ .../unreleased/Under the Hood-20240625-110833.yaml | 6 ------ CHANGELOG.md | 14 ++++++++++++++ dbt/adapters/__about__.py | 2 +- 6 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 .changes/1.4.0.md delete mode 100644 .changes/unreleased/Features-20240723-104221.yaml delete mode 100644 .changes/unreleased/Fixes-20240728-221030.yaml delete mode 100644 .changes/unreleased/Under the Hood-20240625-110833.yaml diff --git a/.changes/1.4.0.md b/.changes/1.4.0.md new file mode 100644 index 00000000..fc6279db --- /dev/null +++ b/.changes/1.4.0.md @@ -0,0 +1,13 @@ +## dbt-adapters 1.4.0 - July 30, 2024 + +### Features + +- render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 ([#271](https://github.com/dbt-labs/dbt-adapters/issues/271)) + +### Fixes + +- Incremental table varchar column definition changed ([#276](https://github.com/dbt-labs/dbt-adapters/issues/276)) + +### Under the Hood + +- Rework record/replay to record at the database connection level. ([#244](https://github.com/dbt-labs/dbt-adapters/issues/244)) diff --git a/.changes/unreleased/Features-20240723-104221.yaml b/.changes/unreleased/Features-20240723-104221.yaml deleted file mode 100644 index 45499971..00000000 --- a/.changes/unreleased/Features-20240723-104221.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 -time: 2024-07-23T10:42:21.222203-04:00 -custom: - Author: michelleark - Issue: "271" diff --git a/.changes/unreleased/Fixes-20240728-221030.yaml b/.changes/unreleased/Fixes-20240728-221030.yaml deleted file mode 100644 index a1cddb24..00000000 --- a/.changes/unreleased/Fixes-20240728-221030.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Incremental table varchar column definition changed -time: 2024-07-28T22:10:30.673107-04:00 -custom: - Author: gshank - Issue: "276" diff --git a/.changes/unreleased/Under the Hood-20240625-110833.yaml b/.changes/unreleased/Under the Hood-20240625-110833.yaml deleted file mode 100644 index c4486a8f..00000000 --- a/.changes/unreleased/Under the Hood-20240625-110833.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Rework record/replay to record at the database connection level. -time: 2024-06-25T11:08:33.264457-04:00 -custom: - Author: peteralllenwebb - Issue: "244" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bd48b5d..4146e95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.4.0 - July 30, 2024 + +### Features + +- render 'to' and 'to_columns' fields on foreign key constraints, and bump dbt-common lower bound to 1.6 ([#271](https://github.com/dbt-labs/dbt-adapters/issues/271)) + +### Fixes + +- Incremental table varchar column definition changed ([#276](https://github.com/dbt-labs/dbt-adapters/issues/276)) + +### Under the Hood + +- Rework record/replay to record at the database connection level. ([#244](https://github.com/dbt-labs/dbt-adapters/issues/244)) + ## dbt-adapters 1.3.3 - July 09, 2024 ### Fixes diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index cfa2db68..d619c757 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.3.3" +version = "1.4.0" From 2c4065b10e7aab67c342d6a561411465f9ba6beb Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Tue, 30 Jul 2024 13:43:46 -0500 Subject: [PATCH 34/38] upgrade upload action to v4 in all reference points (#277) --- .github/actions/build-hatch/action.yml | 2 +- .github/actions/publish-results/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-hatch/action.yml b/.github/actions/build-hatch/action.yml index f2da5935..6d81339a 100644 --- a/.github/actions/build-hatch/action.yml +++ b/.github/actions/build-hatch/action.yml @@ -31,7 +31,7 @@ runs: - name: Upload artifacts if: ${{ inputs.archive-name != '' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ inputs.archive-name }} path: ${{ inputs.working-dir }}dist/ diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml index d863d659..0d5cb7e6 100644 --- a/.github/actions/publish-results/action.yml +++ b/.github/actions/publish-results/action.yml @@ -19,7 +19,7 @@ runs: run: echo "ts=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts shell: bash - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.file-name }}_python-${{ inputs.python-version }}_${{ steps.timestamp.outputs.ts }}.csv path: ${{ inputs.source-file }} From ced2f98847c7f53f05f0f0486c51329efd64c73a Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Wed, 31 Jul 2024 21:37:27 -0400 Subject: [PATCH 35/38] Update `setup-python` action to `v5` to move off of a deprecated version of Node (`node16`) (#278) --- .github/actions/setup-hatch/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-hatch/action.yml b/.github/actions/setup-hatch/action.yml index 6b15cdbf..6bf8ea10 100644 --- a/.github/actions/setup-hatch/action.yml +++ b/.github/actions/setup-hatch/action.yml @@ -13,7 +13,7 @@ runs: using: composite steps: - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} From 7935b1380555002c03dbd92bf79475523eef55c6 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 1 Aug 2024 00:40:14 -0400 Subject: [PATCH 36/38] Update `download-artifact` to `v4` (#279) --- .github/actions/publish-pypi/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/publish-pypi/action.yml b/.github/actions/publish-pypi/action.yml index deffc6e3..25bc3a8d 100644 --- a/.github/actions/publish-pypi/action.yml +++ b/.github/actions/publish-pypi/action.yml @@ -14,7 +14,7 @@ runs: steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ inputs.archive-name }} path: dist/ From 2bdf580ba269cffb8a3331f626c86f7b0116e51c Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:55:06 -0400 Subject: [PATCH 37/38] Updating changie.yaml to add contributors and PR links (#280) --- .../Under the Hood-20240801-220551.yaml | 6 ++ .changie.yaml | 92 +++++++++++++++---- 2 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240801-220551.yaml diff --git a/.changes/unreleased/Under the Hood-20240801-220551.yaml b/.changes/unreleased/Under the Hood-20240801-220551.yaml new file mode 100644 index 00000000..25b54a65 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240801-220551.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Updating changie.yaml to add contributors and PR links +time: 2024-08-01T22:05:51.327652-04:00 +custom: + Author: leahwicz + Issue: "219" diff --git a/.changie.yaml b/.changie.yaml index 1198cfe0..8f1d8615 100644 --- a/.changie.yaml +++ b/.changie.yaml @@ -4,7 +4,7 @@ headerPath: header.tpl.md versionHeaderPath: "" changelogPath: CHANGELOG.md versionExt: md -envPrefix: CHANGIE_ +envPrefix: "CHANGIE_" versionFormat: '## dbt-adapters {{.Version}} - {{.Time.Format "January 02, 2006"}}' kindFormat: '### {{.Kind}}' changeFormat: |- @@ -15,11 +15,11 @@ changeFormat: |- {{- $IssueList = append $IssueList $changeLink }} {{- end -}} - {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}}) + kinds: - label: Breaking Changes - label: Features - label: Fixes -- label: Docs - label: Under the Hood - label: Dependencies changeFormat: |- @@ -32,14 +32,14 @@ kinds: - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) skipGlobalChoices: true additionalChoices: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: PR - label: GitHub Pull Request Number (separated by a single space if multiple) - type: string - minLength: 1 + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 - label: Security changeFormat: |- {{- $PRList := list }} @@ -51,20 +51,22 @@ kinds: - {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}}) skipGlobalChoices: true additionalChoices: - - key: Author - label: GitHub Username(s) (separated by a single space if multiple) - type: string - minLength: 3 - - key: PR - label: GitHub Pull Request Number (separated by a single space if multiple) - type: string - minLength: 1 + - key: Author + label: GitHub Username(s) (separated by a single space if multiple) + type: string + minLength: 3 + - key: PR + label: GitHub Pull Request Number (separated by a single space if multiple) + type: string + minLength: 1 + newlines: afterChangelogHeader: 1 afterKind: 1 afterChangelogVersion: 1 beforeKind: 1 endOfVersion: 1 + custom: - key: Author label: GitHub Username(s) (separated by a single space if multiple) @@ -74,3 +76,57 @@ custom: label: GitHub Issue Number (separated by a single space if multiple) type: string minLength: 1 + + +footerFormat: | + {{- $contributorDict := dict }} + {{- /* ensure all names in this list are all lowercase for later matching purposes */}} + {{- $core_team := splitList " " .Env.CORE_TEAM }} + {{- /* ensure we always skip snyk and dependabot in addition to the core team */}} + {{- $maintainers := list "dependabot[bot]" "snyk-bot"}} + {{- range $team_member := $core_team }} + {{- $team_member_lower := lower $team_member }} + {{- $maintainers = append $maintainers $team_member_lower }} + {{- end }} + {{- range $change := .Changes }} + {{- $authorList := splitList " " $change.Custom.Author }} + {{- /* loop through all authors for a single changelog */}} + {{- range $author := $authorList }} + {{- $authorLower := lower $author }} + {{- /* we only want to include non-core team contributors */}} + {{- if not (has $authorLower $maintainers)}} + {{- $changeList := splitList " " $change.Custom.Author }} + {{- $IssueList := list }} + {{- $changeLink := $change.Kind }} + {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }} + {{- $changes := splitList " " $change.Custom.PR }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/pull/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- else }} + {{- $changes := splitList " " $change.Custom.Issue }} + {{- range $issueNbr := $changes }} + {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-adapters/issues/nbr)" | replace "nbr" $issueNbr }} + {{- $IssueList = append $IssueList $changeLink }} + {{- end -}} + {{- end }} + {{- /* check if this contributor has other changes associated with them already */}} + {{- if hasKey $contributorDict $author }} + {{- $contributionList := get $contributorDict $author }} + {{- $contributionList = concat $contributionList $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- else }} + {{- $contributionList := $IssueList }} + {{- $contributorDict := set $contributorDict $author $contributionList }} + {{- end }} + {{- end}} + {{- end}} + {{- end }} + {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} + {{- if $contributorDict}} + ### Contributors + {{- range $k,$v := $contributorDict }} + - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) + {{- end }} + {{- end }} From 313d4a506984b5c024d51a110eb9beaeef58fcfa Mon Sep 17 00:00:00 2001 From: Jean Cochrane <jeancochrane@users.noreply.github.com> Date: Thu, 8 Aug 2024 19:13:08 -0500 Subject: [PATCH 38/38] Use model alias for the CTE identifier generated during ephemeral materialization (#236) --- .changes/unreleased/Fixes-20240610-195300.yaml | 6 ++++++ dbt/adapters/base/relation.py | 7 +++++-- tests/unit/test_relation.py | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240610-195300.yaml diff --git a/.changes/unreleased/Fixes-20240610-195300.yaml b/.changes/unreleased/Fixes-20240610-195300.yaml new file mode 100644 index 00000000..1f8cd5a5 --- /dev/null +++ b/.changes/unreleased/Fixes-20240610-195300.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use model alias for the CTE identifier generated during ephemeral materialization +time: 2024-06-10T19:53:00.086488231Z +custom: + Author: jeancochrane + Issue: "5273" diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 210a2dcd..1aab7b2f 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -241,8 +241,11 @@ def create_ephemeral_from( relation_config: RelationConfig, limit: Optional[int] = None, ) -> Self: - # Note that ephemeral models are based on the name. - identifier = cls.add_ephemeral_prefix(relation_config.name) + # Note that ephemeral models are based on the identifier, which will + # point to the model's alias if one exists and otherwise fall back to + # the filename. This is intended to give the user more control over + # the way that the CTE name is constructed + identifier = cls.add_ephemeral_prefix(relation_config.identifier) return cls.create( type=cls.CTE, identifier=identifier, diff --git a/tests/unit/test_relation.py b/tests/unit/test_relation.py index a1c01c5c..97d56419 100644 --- a/tests/unit/test_relation.py +++ b/tests/unit/test_relation.py @@ -1,4 +1,4 @@ -from dataclasses import replace +from dataclasses import dataclass, replace import pytest @@ -79,3 +79,16 @@ def test_render_limited(limit, require_alias, expected_result): actual_result = my_relation.render_limited() assert actual_result == expected_result assert str(my_relation) == expected_result + + +def test_create_ephemeral_from_uses_identifier(): + @dataclass + class Node: + """Dummy implementation of RelationConfig protocol""" + + name: str + identifier: str + + node = Node(name="name_should_not_be_used", identifier="test") + ephemeral_relation = BaseRelation.create_ephemeral_from(node) + assert str(ephemeral_relation) == "__dbt__cte__test"