Merge branch 'main' into dbeatty/test-snapshot-configs

dbt-labs · Dec 19, 2024 · 4f5cc63 · 4f5cc63
2 parents 0b03c5d + 048cb2b
commit 4f5cc63
Show file tree

Hide file tree

Showing 26 changed files with 513 additions and 106 deletions.
diff --git a/.changes/1.10.4.md b/.changes/1.10.4.md
@@ -1 +1,12 @@
 ## dbt-adapters 1.10.4 - November 11, 2024
+
+### Features
+
+- Use a behavior flag to gate microbatch functionality (instead of an environment variable) ([#327](https://github.com/dbt-labs/dbt-adapters/issues/327))
+
+### Under the Hood
+
+- Add `query_id` to SQLQueryStatus ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
+
+### Contributors
+- [@cmcarthur](https://github.com/cmcarthur) ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
diff --git a/.changes/1.11.0.md b/.changes/1.11.0.md
@@ -1,12 +1,10 @@
-## dbt-adapters 1.11.0 - November 11, 2024
+## dbt-adapters 1.11.0 - December 17, 2024
 
 ### Features
 
-- Use a behavior flag to gate microbatch functionality (instead of an environment variable) ([#327](https://github.com/dbt-labs/dbt-adapters/issues/327))
+- Add new hard_deletes="new_record" mode for snapshots. ([#317](https://github.com/dbt-labs/dbt-adapters/issues/317))
+- Introduce new Capability for MicrobatchConcurrency support ([#359](https://github.com/dbt-labs/dbt-adapters/issues/359))
 
 ### Under the Hood
 
-- Add `query_id` to SQLQueryStatus ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
-
-### Contributors
-- [@cmcarthur](https://github.com/cmcarthur) ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
+- Add retry logic for retryable exceptions. ([#368](https://github.com/dbt-labs/dbt-adapters/issues/368))
diff --git a/.changes/1.12.0.md b/.changes/1.12.0.md
@@ -0,0 +1 @@
+## dbt-adapters 1.12.0 - December 18, 2024
diff --git a/.changes/unreleased/Features-20241104-120653.yaml b/.changes/unreleased/Features-20241104-120653.yaml
diff --git a/.changes/unreleased/Features-20241120-112806.yaml b/.changes/unreleased/Features-20241120-112806.yaml
diff --git a/.changes/unreleased/Features-20241216-172047.yaml b/.changes/unreleased/Features-20241216-172047.yaml
@@ -0,0 +1,6 @@
+kind: Features
+body: Add function to run custom sql for getting freshness info
+time: 2024-12-16T17:20:47.065611-08:00
+custom:
+  Author: ChenyuLInx
+  Issue: "8797"
diff --git a/.changes/unreleased/Under the Hood-20241217-110536.yaml b/.changes/unreleased/Under the Hood-20241217-110536.yaml
@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Added new equals macro that handles null value checks in sql
+time: 2024-12-17T11:05:36.363421+02:00
+custom:
+    Author: adrianburusdbt
+    Issue: "159"
diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml
@@ -198,6 +198,9 @@ jobs:
         if: ${{ needs.temp-branch.outputs.name != '' && inputs.merge }}
         runs-on: ${{ vars.DEFAULT_RUNNER }}
         steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ needs.temp-branch.outputs.name }}
         -   uses: everlytic/[email protected]
             with:
                 source_ref: ${{ needs.temp-branch.outputs.name }}

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -58,11 +58,45 @@ jobs:
             branch: ${{ needs.generate-changelog.outputs.branch-name }}
         secrets: inherit
 
-    publish-pypi:
+    package:
         if: ${{ inputs.pypi-public == true }}
-        needs: generate-changelog
-        uses: ./.github/workflows/_publish-pypi.yml
+        uses: ./.github/workflows/_package-directory.yml
         with:
             package: ${{ inputs.package }}
-            deploy-to: ${{ inputs.deploy-to }}
-            branch: ${{ needs.generate-changelog.outputs.branch-name }}
+
+    publish-pypi:
+        if: ${{ inputs.pypi-public == true }}
+        needs: [package, generate-changelog]
+        runs-on: ${{ vars.DEFAULT_RUNNER }}
+        environment:
+            name: ${{ inputs.deploy-to }}
+            url: ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}
+        permissions:
+            # this permission is required for trusted publishing
+            # see https://github.com/marketplace/actions/pypi-publish
+            id-token: write
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ needs.generate-changelog.outputs.branch-name }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ vars.DEFAULT_PYTHON_VERSION }}
+        -   uses: pypa/hatch@install
+            # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test
+            # this is done via environment variables in the test environment in GitHub
+        -   run: hatch build && hatch run build:check-all
+            working-directory: ./${{ needs.package.outputs.directory }}
+        -   uses: pypa/gh-action-pypi-publish@release/v1
+            with:
+                repository-url: ${{ vars.PYPI_REPOSITORY_URL }}
+                packages-dir: ./${{ needs.package.outputs.directory }}dist/
+        -   id: version
+            run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT
+            working-directory: ./${{ needs.package.outputs.directory }}
+        -   uses: nick-fields/retry@v3
+            with:
+                timeout_seconds: 10
+                retry_wait_seconds: 10
+                max_attempts: 15  # 5 minutes: (10s timeout + 10s delay) * 15 attempts
+                command: wget ${{ vars.PYPI_PROJECT_URL }}/${{ steps.version.outputs.version }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
-## dbt-adapters 1.11.0 - November 11, 2024
+## dbt-adapters 1.12.0 - December 18, 2024
+
+
+
+## dbt-adapters 1.11.0 - December 17, 2024
+
+### Features
+
+- Add new hard_deletes="new_record" mode for snapshots. ([#317](https://github.com/dbt-labs/dbt-adapters/issues/317))
+- Introduce new Capability for MicrobatchConcurrency support ([#359](https://github.com/dbt-labs/dbt-adapters/issues/359))
+
+### Under the Hood
+
+- Add retry logic for retryable exceptions. ([#368](https://github.com/dbt-labs/dbt-adapters/issues/368))
+
+## dbt-adapters 1.10.4 - November 11, 2024
 
 ### Features
 
@@ -18,8 +33,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
 ### Contributors
 - [@cmcarthur](https://github.com/cmcarthur) ([#342](https://github.com/dbt-labs/dbt-adapters/issues/342))
 
-## dbt-adapters 1.10.4 - November 11, 2024
-
 ## dbt-adapters 1.10.3 - October 29, 2024
 
 ## dbt-adapters 1.10.2 - October 01, 2024
@@ -39,8 +52,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
 - Negate the check for microbatch behavior flag in determining builtins ([#349](https://github.com/dbt-labs/dbt-adapters/issues/349))
 - Move require_batched_execution_for_custom_microbatch_strategy flag to global ([#351](https://github.com/dbt-labs/dbt-adapters/issues/351))
 
-
-
 ## dbt-adapters 1.8.0 - October 29, 2024
 
 ### Fixes

diff --git a/dbt-tests-adapter/dbt/tests/adapter/concurrency/test_concurrency.py b/dbt-tests-adapter/dbt/tests/adapter/concurrency/test_concurrency.py
@@ -1,11 +1,13 @@
+from collections import Counter
+
 import pytest
 
+from dbt.artifacts.schemas.results import RunStatus
 from dbt.tests.util import (
     check_relations_equal,
     check_table_does_not_exist,
     rm_file,
     run_dbt,
-    run_dbt_and_capture,
     write_file,
 )
 
@@ -317,16 +319,22 @@ def test_concurrency(self, project):
         rm_file(project.project_root, "seeds", "seed.csv")
         write_file(seeds__update_csv, project.project_root, "seeds", "seed.csv")
 
-        results, output = run_dbt_and_capture(["run"], expect_pass=False)
-        assert len(results) == 7
+        results = run_dbt(["run"], expect_pass=False)
+
         check_relations_equal(project.adapter, ["seed", "view_model"])
         check_relations_equal(project.adapter, ["seed", "dep"])
         check_relations_equal(project.adapter, ["seed", "table_a"])
         check_relations_equal(project.adapter, ["seed", "table_b"])
         check_table_does_not_exist(project.adapter, "invalid")
         check_table_does_not_exist(project.adapter, "skip")
 
-        assert "PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7" in output
+        result_statuses = Counter([result.status for result in results])
+        expected_statuses = {
+            RunStatus.Success: 5,
+            RunStatus.Error: 1,
+            RunStatus.Skipped: 1,
+        }
+        assert result_statuses == expected_statuses
 
 
 class TestConcurenncy(BaseConcurrency):

diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py
@@ -240,6 +240,8 @@
 select 'NY','New York','Manhattan','2021-04-01'
 union all
 select 'PA','Philadelphia','Philadelphia','2021-05-21'
+union all
+select 'CO','Denver',null,'2021-06-18'
 
 """
 
@@ -265,6 +267,8 @@
 select 'NY','New York','Manhattan','2021-04-01'
 union all
 select 'PA','Philadelphia','Philadelphia','2021-05-21'
+union all
+select 'CO','Denver',null,'2021-06-18'
 
 """
 
@@ -288,6 +292,7 @@
 NY,Kings,Brooklyn,2021-04-02
 NY,New York,Manhattan,2021-04-01
 PA,Philadelphia,Philadelphia,2021-05-21
+CO,Denver,,2021-06-18
 """
 
 seeds__add_new_rows_sql = """
@@ -439,7 +444,7 @@ def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name):
     def test__no_unique_keys(self, project):
         """with no unique keys, seed and model should match"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project, seed="seed", incremental_model="no_unique_key", update_sql_file="add_new_rows"
         )
@@ -449,7 +454,7 @@ def test__no_unique_keys(self, project):
     def test__empty_str_unique_key(self, project):
         """with empty string for unique key, seed and model should match"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project,
             seed="seed",
@@ -462,7 +467,7 @@ def test__one_unique_key(self, project):
         """with one unique key, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="one_str__overwrite", seed_rows=7, opt_model_count=1
+            relation="one_str__overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -487,7 +492,7 @@ def test__bad_unique_key(self, project):
     def test__empty_unique_key_list(self, project):
         """with no unique keys, seed and model should match"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project,
             seed="seed",
@@ -500,7 +505,7 @@ def test__unary_unique_key_list(self, project):
         """with one unique key, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1
+            relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -515,7 +520,7 @@ def test__duplicated_unary_unique_key_list(self, project):
         """with two of the same unique key, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1
+            relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -530,7 +535,7 @@ def test__trinary_unique_key_list(self, project):
         """with three unique keys, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1
+            relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -545,7 +550,7 @@ def test__trinary_unique_key_list_no_update(self, project):
         """even with three unique keys, adding distinct rows to seed does not
         cause seed and model to diverge"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project,
             seed="seed",

diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py
@@ -1,16 +1,6 @@
 import pytest
 from dbt.tests.util import run_dbt
 
-
-macros__equals_sql = """
-{% macro equals(expr1, expr2) -%}
-case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null))
-    then 0
-    else 1
-end = 0
-{% endmacro %}
-"""
-
 macros__test_assert_equal_sql = """
 {% test assert_equal(model, actual, expected) %}
 select * from {{ model }}
@@ -33,7 +23,6 @@ class BaseUtils:
     @pytest.fixture(scope="class")
     def macros(self):
         return {
-            "equals.sql": macros__equals_sql,
             "test_assert_equal.sql": macros__test_assert_equal_sql,
             "replace_empty.sql": macros__replace_empty_sql,
         }

diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py
@@ -1,16 +1,10 @@
 import pytest
 
-from dbt.tests.adapter.utils import base_utils, fixture_equals
+from dbt.tests.adapter.utils import fixture_equals
 from dbt.tests.util import relation_from_name, run_dbt
 
 
 class BaseEquals:
-    @pytest.fixture(scope="class")
-    def macros(self):
-        return {
-            "equals.sql": base_utils.macros__equals_sql,
-        }
-
     @pytest.fixture(scope="class")
     def seeds(self):
         return {