Merge branch 'main' into paw/fix-timer-resolution

.changes/1.3.0.md

@@ -0,0 +1,5 @@
+## dbt-adapters 1.3.0 - June 18, 2024
+
+### Features
+
+* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation

.changes/1.3.1.md

@@ -0,0 +1 @@
+## dbt-adapters 1.3.1 - June 20, 2024

.changes/1.9.0.md

@@ -0,0 +1 @@
+## dbt-adapters 1.9.0 - June 18, 2024

.changes/1.9.1.md

@@ -0,0 +1 @@
+## dbt-adapters 1.9.1 - June 20, 2024

.github/workflows/docs-issue.yml

@@ -0,0 +1,41 @@
+# **what?**
+# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed
+
+# **why?**
+# To reduce barriers for keeping docs up to date
+
+# **when?**
+# When an issue is labeled `user docs` and is closed as completed.  Can be labeled before or after the issue is closed.
+
+
+name: Open issues in docs.getdbt.com repo when an issue is labeled
+run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}"
+
+on:
+  issues:
+    types: [labeled, closed]
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+    issues: write # comments on issues
+
+jobs:
+  open_issues:
+    # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned.
+    # If this logic does not exist in this workflow, it runs the
+    # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having
+    # generating the comment before the other runs.  This lives here instead of the shared workflow because this is where we
+    # decide if it should run or not.
+    if: |
+      (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && (
+      (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) ||
+      (github.event.action == 'labeled' && github.event.label.name == 'user docs'))
+    uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main
+    with:
+        issue_repository: "dbt-labs/docs.getdbt.com"
+        issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}"
+        issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated."
+    secrets: inherit

CHANGELOG.md

@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-adapters 1.9.1 - June 20, 2024
+
+## dbt-adapters 1.9.0 - June 18, 2024
+
 ## dbt-adapters 1.8.0 - May 09, 2024
 
 ### Features
@@ -15,6 +19,14 @@ and is generated by [Changie](https://github.com/miniscruff/changie).
 
 * Update Clone test to reflect core change removing `deferred` attribute from nodes
 
+## dbt-adapters 1.3.1 - June 20, 2024
+
+## dbt-adapters 1.3.0 - June 18, 2024
+
+### Features
+
+* Add get_catalog_for_single_relation macro and capability to enable adapters to optimize catalog generation
+
 ## dbt-adapters 1.2.1 - May 21, 2024
 
 ### Features

dbt-tests-adapter/dbt/tests/__about__.py

@@ -1 +1 @@
-version = "1.8.0"
+version = "1.9.1"

dbt-tests-adapter/dbt/tests/adapter/basic/test_get_catalog_for_single_relation.py

@@ -0,0 +1,87 @@
+import pytest
+
+from dbt.tests.util import run_dbt, get_connection
+
+models__my_table_model_sql = """
+select * from {{ ref('my_seed') }}
+"""
+
+
+models__my_view_model_sql = """
+{{
+    config(
+        materialized='view',
+    )
+}}
+
+select * from {{ ref('my_seed') }}
+"""
+
+seed__my_seed_csv = """id,first_name,email,ip_address,updated_at
+1,Larry,[email protected],69.135.206.194,2008-09-12 19:08:31
+"""
+
+
+class BaseGetCatalogForSingleRelation:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"name": "get_catalog_for_single_relation"}
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "my_seed.csv": seed__my_seed_csv,
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_view_model.sql": models__my_view_model_sql,
+            "my_table_model.sql": models__my_table_model_sql,
+        }
+
+    @pytest.fixture(scope="class")
+    def expected_catalog_my_seed(self, project):
+        raise NotImplementedError(
+            "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`."
+        )
+
+    @pytest.fixture(scope="class")
+    def expected_catalog_my_model(self, project):
+        raise NotImplementedError(
+            "To use this test, please implement `get_catalog_for_single_relation`, inherited from `SQLAdapter`."
+        )
+
+    def get_relation_for_identifier(self, project, identifier):
+        return project.adapter.get_relation(
+            database=project.database,
+            schema=project.test_schema,
+            identifier=identifier,
+        )
+
+    def test_get_catalog_for_single_relation(
+        self, project, expected_catalog_my_seed, expected_catalog_my_view_model
+    ):
+        results = run_dbt(["seed"])
+        assert len(results) == 1
+
+        my_seed_relation = self.get_relation_for_identifier(project, "my_seed")
+
+        with get_connection(project.adapter):
+            actual_catalog_my_seed = project.adapter.get_catalog_for_single_relation(
+                my_seed_relation
+            )
+
+        assert actual_catalog_my_seed == expected_catalog_my_seed
+
+        results = run_dbt(["run"])
+        assert len(results) == 2
+
+        my_view_model_relation = self.get_relation_for_identifier(project, "my_view_model")
+
+        with get_connection(project.adapter):
+            actual_catalog_my_view_model = project.adapter.get_catalog_for_single_relation(
+                my_view_model_relation
+            )
+
+        assert actual_catalog_my_view_model == expected_catalog_my_view_model

dbt/adapters/__about__.py

@@ -1 +1 @@
-version = "1.2.1"
+version = "1.3.1"

dbt/adapters/base/impl.py

@@ -1,10 +1,10 @@
 import abc
+import time
 from concurrent.futures import as_completed, Future
 from contextlib import contextmanager
 from datetime import datetime
 from enum import Enum
 from multiprocessing.context import SpawnContext
-import time
 from typing import (
     Any,
     Callable,
@@ -23,12 +23,15 @@
     TYPE_CHECKING,
 )
 
+import pytz
 from dbt_common.clients.jinja import CallableMacroGenerator
 from dbt_common.contracts.constraints import (
     ColumnLevelConstraint,
     ConstraintType,
     ModelLevelConstraint,
 )
+from dbt_common.contracts.metadata import CatalogTable
+from dbt_common.events.functions import fire_event, warn_or_error
 from dbt_common.exceptions import (
     DbtInternalError,
     DbtRuntimeError,
@@ -38,14 +41,12 @@
     NotImplementedError,
     UnexpectedNullError,
 )
-from dbt_common.events.functions import fire_event, warn_or_error
 from dbt_common.utils import (
     AttrDict,
     cast_to_str,
     executor,
     filter_null_values,
 )
-import pytz
 
 from dbt.adapters.base.column import Column as BaseColumn
 from dbt.adapters.base.connections import (
@@ -222,6 +223,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         - truncate_relation
         - rename_relation
         - get_columns_in_relation
+        - get_catalog_for_single_relation
         - get_column_schema_from_query
         - expand_column_types
         - list_relations_without_caching
@@ -627,6 +629,12 @@ def get_columns_in_relation(self, relation: BaseRelation) -> List[BaseColumn]:
         """Get a list of the columns in the given Relation."""
         raise NotImplementedError("`get_columns_in_relation` is not implemented for this adapter!")
 
+    def get_catalog_for_single_relation(self, relation: BaseRelation) -> Optional[CatalogTable]:
+        """Get catalog information including table-level and column-level metadata for a single relation."""
+        raise NotImplementedError(
+            "`get_catalog_for_single_relation` is not implemented for this adapter!"
+        )
+
     @available.deprecated("get_columns_in_relation", lambda *a, **k: [])
     def get_columns_in_table(self, schema: str, identifier: str) -> List[BaseColumn]:
         """DEPRECATED: Get a list of the columns in the given table."""

dbt/adapters/capability.py

@@ -14,7 +14,12 @@ class Capability(str, Enum):
     """Indicates support for determining the time of the last table modification by querying database metadata."""
 
     TableLastModifiedMetadataBatch = "TableLastModifiedMetadataBatch"
-    """Indicates support for performantly determining the time of the last table modification by querying database metadata in batch."""
+    """Indicates support for performantly determining the time of the last table modification by querying database
+    metadata in batch."""
+
+    GetCatalogForSingleRelation = "GetCatalogForSingleRelation"
+    """Indicates support for getting catalog information including table-level and column-level metadata for a single
+    relation."""
 
 
 class Support(str, Enum):

dbt/adapters/sql/impl.py

@@ -9,7 +9,6 @@
 from dbt.adapters.exceptions import RelationTypeNullError
 from dbt.adapters.sql.connections import SQLConnectionManager
 
-
 LIST_RELATIONS_MACRO_NAME = "list_relations_without_caching"
 GET_COLUMNS_IN_RELATION_MACRO_NAME = "get_columns_in_relation"
 LIST_SCHEMAS_MACRO_NAME = "list_schemas"
@@ -41,6 +40,7 @@ class SQLAdapter(BaseAdapter):
         - get_catalog
         - list_relations_without_caching
         - get_columns_in_relation
+        - get_catalog_for_single_relation
     """
 
     ConnectionManager: Type[SQLConnectionManager]

dbt/include/global_project/macros/adapters/metadata.sql

@@ -77,6 +77,15 @@
     'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}
 {% endmacro %}
 
+{% macro get_catalog_for_single_relation(relation) %}
+  {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}
+{% endmacro %}
+
+{% macro default__get_catalog_for_single_relation(relation) %}
+  {{ exceptions.raise_not_implemented(
+    'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}
+{% endmacro %}
+
 {% macro get_relations() %}
   {{ return(adapter.dispatch('get_relations', 'dbt')()) }}
 {% endmacro %}

pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "dbt-common<2.0",
+    "dbt-common>=1.3,<2.0",
     "pytz>=2015.7",
     # installed via dbt-common but used directly
     "agate>=1.0,<2.0",
@@ -43,6 +43,9 @@ Changelog = "https://github.com/dbt-labs/dbt-adapters/blob/main/CHANGELOG.md"
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.hatch.version]
 path = "dbt/adapters/__about__.py"