From 47fbbfb14f70143369ca5c8b5e9f3e1f47b78d6e Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sat, 13 May 2023 12:47:15 +0200 Subject: [PATCH 1/2] ignore owner when testing docs in azure --- tests/conftest.py | 27 +++++++++----- tests/functional/adapter/test_docs.py | 53 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d0b4c6d4..0d644cb8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os +from typing import Any import pytest from _pytest.fixtures import FixtureRequest @@ -13,7 +14,7 @@ def pytest_addoption(parser): @pytest.fixture(scope="class") -def dbt_profile_target(request: FixtureRequest): +def dbt_profile_target(request: FixtureRequest) -> dict[str, Any]: profile = request.config.getoption("--profile") if profile == "ci_sql_server": @@ -34,7 +35,13 @@ def dbt_profile_target(request: FixtureRequest): raise ValueError(f"Unknown profile: {profile}") -def _all_profiles_base(): +@pytest.fixture(scope="class") +def is_azure(request: FixtureRequest) -> bool: + profile = request.config.getoption("--profile") + return "azure" in profile + + +def _all_profiles_base() -> dict[str, Any]: return { "type": "sqlserver", "driver": os.getenv("SQLSERVER_TEST_DRIVER", "ODBC Driver 18 for SQL Server"), @@ -43,7 +50,7 @@ def _all_profiles_base(): } -def _profile_ci_azure_base(): +def _profile_ci_azure_base() -> dict[str, Any]: return { **_all_profiles_base(), **{ @@ -55,7 +62,7 @@ def _profile_ci_azure_base(): } -def _profile_ci_azure_basic(): +def _profile_ci_azure_basic() -> dict[str, Any]: return { **_profile_ci_azure_base(), **{ @@ -65,7 +72,7 @@ def _profile_ci_azure_basic(): } -def _profile_ci_azure_cli(): +def _profile_ci_azure_cli() -> dict[str, Any]: return { **_profile_ci_azure_base(), **{ @@ -74,7 +81,7 @@ def _profile_ci_azure_cli(): } -def _profile_ci_azure_auto(): +def _profile_ci_azure_auto() -> dict[str, Any]: return { **_profile_ci_azure_base(), **{ @@ -83,7 +90,7 @@ def _profile_ci_azure_auto(): } -def _profile_ci_azure_environment(): +def _profile_ci_azure_environment() -> dict[str, Any]: return { **_profile_ci_azure_base(), **{ @@ -92,7 +99,7 @@ def _profile_ci_azure_environment(): } -def _profile_ci_sql_server(): +def _profile_ci_sql_server() -> dict[str, Any]: return { **_all_profiles_base(), **{ @@ -106,7 +113,7 @@ def _profile_ci_sql_server(): } -def _profile_user(): +def _profile_user() -> dict[str, Any]: profile = { **_all_profiles_base(), **{ @@ -121,7 +128,7 @@ def _profile_user(): return profile -def _profile_user_azure(): +def _profile_user_azure() -> dict[str, Any]: profile = { **_all_profiles_base(), **{ diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py index 941c48dc..408b856c 100644 --- a/tests/functional/adapter/test_docs.py +++ b/tests/functional/adapter/test_docs.py @@ -1,3 +1,5 @@ +import os + import pytest from dbt.tests.adapter.basic.expected_catalog import ( base_expected_catalog, @@ -7,13 +9,44 @@ from dbt.tests.adapter.basic.test_docs_generate import ( BaseDocsGenerate, BaseDocsGenReferences, + get_artifact, ref_models__docs_md, ref_models__ephemeral_copy_sql, ref_models__schema_yml, ref_sources__schema_yml, + run_and_generate, + verify_metadata, ) +def verify_catalog(project, expected_catalog, start_time, ignore_owner): + # get the catalog.json + catalog_path = os.path.join(project.project_root, "target", "catalog.json") + assert os.path.exists(catalog_path) + catalog = get_artifact(catalog_path) + + # verify the catalog + assert set(catalog) == {"errors", "metadata", "nodes", "sources"} + verify_metadata( + catalog["metadata"], + "https://schemas.getdbt.com/dbt/catalog/v1.json", + start_time, + ) + assert not catalog["errors"] + for key in "nodes", "sources": + for unique_id, expected_node in expected_catalog[key].items(): + found_node = catalog[key][unique_id] + for node_key in expected_node: + assert node_key in found_node + + if node_key == "metadata" and ignore_owner: + expected_node[node_key]["owner"] = found_node[node_key]["owner"] + + assert ( + found_node[node_key] == expected_node[node_key] + ), f"Key '{node_key}' in '{unique_id}' did not match" + + class TestDocsGenerateSQLServer(BaseDocsGenerate): @pytest.fixture(scope="class") def expected_catalog(self, project): @@ -28,6 +61,22 @@ def expected_catalog(self, project): model_stats=no_stats(), ) + # Test "--no-compile" flag works and produces no manifest.json + def test_run_and_generate_no_compile(self, project, expected_catalog, is_azure: bool): + start_time = run_and_generate(project, ["--no-compile"]) + assert not os.path.exists(os.path.join(project.project_root, "target", "manifest.json")) + verify_catalog(project, expected_catalog, start_time, is_azure) + + # Test generic "docs generate" command + def test_run_and_generate(self, project, expected_catalog, is_azure: bool): + start_time = run_and_generate(project) + verify_catalog(project, expected_catalog, start_time, is_azure) + + # Check that assets have been copied to the target directory for use in the docs html page + assert os.path.exists(os.path.join(".", "target", "assets")) + assert os.path.exists(os.path.join(".", "target", "assets", "lorem-ipsum.txt")) + assert not os.path.exists(os.path.join(".", "target", "non-existent-assets")) + class TestDocsGenReferencesSQLServer(BaseDocsGenReferences): @pytest.fixture(scope="class") @@ -77,3 +126,7 @@ def models(self): "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, "docs.md": ref_models__docs_md, } + + def test_references(self, project, expected_catalog, is_azure: bool): + start_time = run_and_generate(project) + verify_catalog(project, expected_catalog, start_time, is_azure) From facff2c81b7250907996aaea1c4c6cd40a20fd32 Mon Sep 17 00:00:00 2001 From: Sam Debruyn Date: Sat, 13 May 2023 12:52:04 +0200 Subject: [PATCH 2/2] remove type hints --- tests/conftest.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0d644cb8..87b43b62 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ import os -from typing import Any import pytest from _pytest.fixtures import FixtureRequest @@ -14,7 +13,7 @@ def pytest_addoption(parser): @pytest.fixture(scope="class") -def dbt_profile_target(request: FixtureRequest) -> dict[str, Any]: +def dbt_profile_target(request: FixtureRequest): profile = request.config.getoption("--profile") if profile == "ci_sql_server": @@ -41,7 +40,7 @@ def is_azure(request: FixtureRequest) -> bool: return "azure" in profile -def _all_profiles_base() -> dict[str, Any]: +def _all_profiles_base(): return { "type": "sqlserver", "driver": os.getenv("SQLSERVER_TEST_DRIVER", "ODBC Driver 18 for SQL Server"), @@ -50,7 +49,7 @@ def _all_profiles_base() -> dict[str, Any]: } -def _profile_ci_azure_base() -> dict[str, Any]: +def _profile_ci_azure_base(): return { **_all_profiles_base(), **{ @@ -62,7 +61,7 @@ def _profile_ci_azure_base() -> dict[str, Any]: } -def _profile_ci_azure_basic() -> dict[str, Any]: +def _profile_ci_azure_basic(): return { **_profile_ci_azure_base(), **{ @@ -72,7 +71,7 @@ def _profile_ci_azure_basic() -> dict[str, Any]: } -def _profile_ci_azure_cli() -> dict[str, Any]: +def _profile_ci_azure_cli(): return { **_profile_ci_azure_base(), **{ @@ -81,7 +80,7 @@ def _profile_ci_azure_cli() -> dict[str, Any]: } -def _profile_ci_azure_auto() -> dict[str, Any]: +def _profile_ci_azure_auto(): return { **_profile_ci_azure_base(), **{ @@ -90,7 +89,7 @@ def _profile_ci_azure_auto() -> dict[str, Any]: } -def _profile_ci_azure_environment() -> dict[str, Any]: +def _profile_ci_azure_environment(): return { **_profile_ci_azure_base(), **{ @@ -99,7 +98,7 @@ def _profile_ci_azure_environment() -> dict[str, Any]: } -def _profile_ci_sql_server() -> dict[str, Any]: +def _profile_ci_sql_server(): return { **_all_profiles_base(), **{ @@ -113,7 +112,7 @@ def _profile_ci_sql_server() -> dict[str, Any]: } -def _profile_user() -> dict[str, Any]: +def _profile_user(): profile = { **_all_profiles_base(), **{ @@ -128,7 +127,7 @@ def _profile_user() -> dict[str, Any]: return profile -def _profile_user_azure() -> dict[str, Any]: +def _profile_user_azure(): profile = { **_all_profiles_base(), **{