From 385fd618dfe24b072f6c6f71670b0ae65378ba5c Mon Sep 17 00:00:00 2001 From: ArgusLi Date: Thu, 3 Nov 2022 16:06:41 -0700 Subject: [PATCH 1/8] Implement TestBaseIncrementalNotSchemaChange --- .../adapter/basic/test_incremental.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/basic/test_incremental.py b/tests/functional/adapter/basic/test_incremental.py index 02c0b80..4d92179 100644 --- a/tests/functional/adapter/basic/test_incremental.py +++ b/tests/functional/adapter/basic/test_incremental.py @@ -1,5 +1,5 @@ import pytest -from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_incremental import BaseIncremental, BaseIncrementalNotSchemaChange from tests.functional.adapter.utils.test_utils import DATALAKE @@ -33,3 +33,34 @@ def dbt_profile_data( if profiles_config_update: profile.update(profiles_config_update) return profile + +class TestBaseIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile \ No newline at end of file From 5689870875db81fedbecd16e2d7925c27386721d Mon Sep 17 00:00:00 2001 From: ArgusLi Date: Thu, 3 Nov 2022 16:45:28 -0700 Subject: [PATCH 2/8] Implement TestTypeBoolean --- .../adapter/basic/test_data_types.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/functional/adapter/basic/test_data_types.py diff --git a/tests/functional/adapter/basic/test_data_types.py b/tests/functional/adapter/basic/test_data_types.py new file mode 100644 index 0000000..7ec309c --- /dev/null +++ b/tests/functional/adapter/basic/test_data_types.py @@ -0,0 +1,34 @@ +import pytest +from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean +from tests.functional.adapter.utils.test_utils import DATALAKE + +class TestTypeBoolean(BaseTypeBoolean): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile \ No newline at end of file From 9b14ab1bfd06b13aa4e717bb44c1c3d2bb88b2f6 Mon Sep 17 00:00:00 2001 From: ArgusLi Date: Fri, 4 Nov 2022 10:12:00 -0700 Subject: [PATCH 3/8] Rearrange test_data_types and implement test_current_timestamp. --- .../adapter/{basic => }/test_data_types.py | 0 .../adapter/utils/test_current_timestamp.py | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+) rename tests/functional/adapter/{basic => }/test_data_types.py (100%) create mode 100644 tests/functional/adapter/utils/test_current_timestamp.py diff --git a/tests/functional/adapter/basic/test_data_types.py b/tests/functional/adapter/test_data_types.py similarity index 100% rename from tests/functional/adapter/basic/test_data_types.py rename to tests/functional/adapter/test_data_types.py diff --git a/tests/functional/adapter/utils/test_current_timestamp.py b/tests/functional/adapter/utils/test_current_timestamp.py new file mode 100644 index 0000000..7401f7d --- /dev/null +++ b/tests/functional/adapter/utils/test_current_timestamp.py @@ -0,0 +1,36 @@ +import pytest +from dbt.tests.adapter.utils.test_current_timestamp import ( + BaseCurrentTimestampNaive, +) +from tests.functional.adapter.utils.test_utils import DATALAKE + +class TestCurrentTimestampNaive(BaseCurrentTimestampNaive): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile \ No newline at end of file From 9bb43e2ff7438c4ae2aaded4acab99c49cee4a06 Mon Sep 17 00:00:00 2001 From: ArgusLi Date: Fri, 4 Nov 2022 17:05:23 -0700 Subject: [PATCH 4/8] Implement remaining tests. Also move fixtures into profiles.py and modify tests that use generic profile data to just import fixtures. Of the tests implemented, the following still need work to make work: - test_array_append.py - test_array_concat.py - test_array_construct.py - test_concurrency.py - test_ephemeral.py (BaseEphemeralMulti) --- tests/fixtures/profiles.py | 34 ++++++++++ .../adapter/basic/test_adapter_methods.py | 32 +-------- .../functional/adapter/basic/test_base_mat.py | 32 +-------- .../adapter/basic/test_docs_generate.py | 63 +----------------- .../adapter/basic/test_ephemeral.py | 61 +++++++++-------- .../adapter/basic/test_generic_tests.py | 32 +-------- .../adapter/basic/test_incremental.py | 66 ++----------------- .../adapter/basic/test_singular_ephemeral.py | 33 +--------- .../adapter/basic/test_snapshots.py | 40 +---------- tests/functional/adapter/test_concurrency.py | 40 +++++++++++ tests/functional/adapter/test_data_types.py | 34 +--------- .../adapter/utils/test_array_append.py | 8 +++ .../adapter/utils/test_array_concat.py | 8 +++ .../adapter/utils/test_array_construct.py | 8 +++ .../adapter/utils/test_current_timestamp.py | 34 +--------- 15 files changed, 153 insertions(+), 372 deletions(-) create mode 100644 tests/fixtures/profiles.py create mode 100644 tests/functional/adapter/test_concurrency.py create mode 100644 tests/functional/adapter/utils/test_array_append.py create mode 100644 tests/functional/adapter/utils/test_array_concat.py create mode 100644 tests/functional/adapter/utils/test_array_construct.py diff --git a/tests/fixtures/profiles.py b/tests/fixtures/profiles.py new file mode 100644 index 0000000..eb2b44c --- /dev/null +++ b/tests/fixtures/profiles.py @@ -0,0 +1,34 @@ +import pytest +from tests.functional.adapter.utils.test_utils import DATALAKE + +# Override this fixture to prepend our schema with DATALAKE +# This ensures the schema works with our datalake +@pytest.fixture(scope="class") +def unique_schema(request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + + +# Override this fixture to set root_path=schema +@pytest.fixture(scope="class") +def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile diff --git a/tests/functional/adapter/basic/test_adapter_methods.py b/tests/functional/adapter/basic/test_adapter_methods.py index 89a47f7..f13f36a 100644 --- a/tests/functional/adapter/basic/test_adapter_methods.py +++ b/tests/functional/adapter/basic/test_adapter_methods.py @@ -1,7 +1,7 @@ import pytest from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod from dbt.tests.adapter.basic.test_adapter_methods import models__upstream_sql -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data models__my_model_sql = """ @@ -52,36 +52,6 @@ def models(self): "model_view.sql": models__my_model_sql, } - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - @pytest.fixture(scope="class") def equal_tables(self): return ["model_view", "expected_view"] diff --git a/tests/functional/adapter/basic/test_base_mat.py b/tests/functional/adapter/basic/test_base_mat.py index d25e924..ce56e11 100644 --- a/tests/functional/adapter/basic/test_base_mat.py +++ b/tests/functional/adapter/basic/test_base_mat.py @@ -14,7 +14,7 @@ run_dbt, check_result_nodes_by_name, ) -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data # Unable to insert variable into docstring, so "rav-test" is hardcoded schema_base_yml = """ @@ -50,36 +50,6 @@ def project_config_update(self): "vars": {"dremio:reflections": "false"}, } - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - def test_base(self, project): # seed command diff --git a/tests/functional/adapter/basic/test_docs_generate.py b/tests/functional/adapter/basic/test_docs_generate.py index 6a12a55..612345c 100644 --- a/tests/functional/adapter/basic/test_docs_generate.py +++ b/tests/functional/adapter/basic/test_docs_generate.py @@ -3,7 +3,6 @@ from tests.functional.adapter.utils.test_utils import ( base_expected_catalog, expected_references_catalog, - DATALAKE, ) from dbt.tests.adapter.basic.test_docs_generate import ( BaseDocsGenerate, @@ -16,6 +15,7 @@ get_artifact, ) from dbt.tests.adapter.basic.expected_catalog import no_stats +from tests.fixtures.profiles import unique_schema, dbt_profile_data # required to explicitly use alternate_schema # otherwise will use unique_schema under profiles fixture @@ -66,15 +66,6 @@ def models(self): "model.sql": models__model_sql, } - # Override this fixture to prepend our schema with DATALAKE - # This ensures the schema works with our datalake - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - # Override this fixture to prevent (twin_strategy) creating a view for seeds @pytest.fixture(scope="class") def project_config_update(self, unique_schema): @@ -91,29 +82,6 @@ def project_config_update(self, unique_schema): }, } - # Override this fixture to set root_path=schema - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - # Override this fixture to change expected types to Dremio types @pytest.fixture(scope="class") def expected_catalog(self, project): @@ -148,12 +116,6 @@ def test_run_and_generate(self, project, expected_catalog): class TestBaseDocsGenReferencesDremio(BaseDocsGenReferences): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema # Override this fixture to allow (twin_strategy) to create a view for seeds # The creation of some models looks for the seed under the database/schema @@ -171,29 +133,6 @@ def project_config_update(self, unique_schema): }, } - # Override this fixture to set root_path=schema - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - # Override this fixture to change expected types to Dremio types @pytest.fixture(scope="class") def expected_catalog(self, project): diff --git a/tests/functional/adapter/basic/test_ephemeral.py b/tests/functional/adapter/basic/test_ephemeral.py index a33eaa9..ee95c88 100644 --- a/tests/functional/adapter/basic/test_ephemeral.py +++ b/tests/functional/adapter/basic/test_ephemeral.py @@ -1,35 +1,42 @@ import pytest from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti from tests.functional.adapter.utils.test_utils import DATALAKE +from dbt.tests.util import run_dbt +from tests.fixtures.profiles import unique_schema, dbt_profile_data class TestEphemeralDremio(BaseEphemeral): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema + pass - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - if profiles_config_update: - profile.update(profiles_config_update) - return profile +class TestEphemeralMulti(BaseEphemeralMulti): + def test_ephemeral_multi(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 3 + + check_relations_equal(project.adapter, ["seed", "dependent"]) + check_relations_equal(project.adapter, ["seed", "double_dependent"]) + check_relations_equal(project.adapter, ["seed", "super_dependent"]) + assert os.path.exists("./target/run/test/models/double_dependent.sql") + with open("./target/run/test/models/double_dependent.sql", "r") as fp: + sql_file = fp.read() + + sql_file = re.sub(r"\d+", "", sql_file) + expected_sql = ( + 'create view "dbt"."test_test_ephemeral"."double_dependent__dbt_tmp" as (' + "with __dbt__cte__base as (" + "select * from test_test_ephemeral.seed" + "), __dbt__cte__base_copy as (" + "select * from __dbt__cte__base" + ")-- base_copy just pulls from base. Make sure the listed" + "-- graph of CTEs all share the same dbt_cte__base cte" + "select * from __dbt__cte__base where gender = 'Male'" + "union all" + "select * from __dbt__cte__base_copy where gender = 'Female'" + ");" + ) + sql_file = "".join(sql_file.split()) + expected_sql = "".join(expected_sql.split()) + assert sql_file == expected_sql diff --git a/tests/functional/adapter/basic/test_generic_tests.py b/tests/functional/adapter/basic/test_generic_tests.py index 3af01f4..e53516d 100644 --- a/tests/functional/adapter/basic/test_generic_tests.py +++ b/tests/functional/adapter/basic/test_generic_tests.py @@ -1,35 +1,7 @@ import pytest from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data class TestGenericTestsDremio(BaseGenericTests): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile + pass diff --git a/tests/functional/adapter/basic/test_incremental.py b/tests/functional/adapter/basic/test_incremental.py index 4d92179..71a387f 100644 --- a/tests/functional/adapter/basic/test_incremental.py +++ b/tests/functional/adapter/basic/test_incremental.py @@ -1,66 +1,14 @@ import pytest -from dbt.tests.adapter.basic.test_incremental import BaseIncremental, BaseIncrementalNotSchemaChange -from tests.functional.adapter.utils.test_utils import DATALAKE +from dbt.tests.adapter.basic.test_incremental import ( + BaseIncremental, + BaseIncrementalNotSchemaChange, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data class TestIncrementalDremio(BaseIncremental): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema + pass - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile class TestBaseIncrementalNotSchemaChange(BaseIncrementalNotSchemaChange): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile \ No newline at end of file + pass diff --git a/tests/functional/adapter/basic/test_singular_ephemeral.py b/tests/functional/adapter/basic/test_singular_ephemeral.py index 4ddd805..efb95db 100644 --- a/tests/functional/adapter/basic/test_singular_ephemeral.py +++ b/tests/functional/adapter/basic/test_singular_ephemeral.py @@ -1,37 +1,8 @@ -import pytest from dbt.tests.adapter.basic.test_singular_tests_ephemeral import ( BaseSingularTestsEphemeral, ) -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data class TestSingularTestsEphemeralDremio(BaseSingularTestsEphemeral): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile + pass diff --git a/tests/functional/adapter/basic/test_snapshots.py b/tests/functional/adapter/basic/test_snapshots.py index 77cc63e..6cd59f9 100644 --- a/tests/functional/adapter/basic/test_snapshots.py +++ b/tests/functional/adapter/basic/test_snapshots.py @@ -2,41 +2,11 @@ from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema @pytest.mark.skip(reason="https://github.com/dremio/dbt-dremio/issues/20") class TestSnapshotCheckColsDremio(BaseSnapshotCheckCols): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - target["database"] = target["datalake"] - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - @pytest.fixture(scope="class") def project_config_update(self): return { @@ -46,14 +16,6 @@ def project_config_update(self): class TestSnapshotTimestampDremio(BaseSnapshotTimestamp): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - @pytest.fixture(scope="class") def dbt_profile_data( self, unique_schema, dbt_profile_target, profiles_config_update diff --git a/tests/functional/adapter/test_concurrency.py b/tests/functional/adapter/test_concurrency.py new file mode 100644 index 0000000..fdc764a --- /dev/null +++ b/tests/functional/adapter/test_concurrency.py @@ -0,0 +1,40 @@ +import pytest +from dbt.tests.adapter.concurrency.test_concurrency import ( + BaseConcurrency, +) +from dbt.tests.util import ( + check_relations_equal, + check_table_does_not_exist, + rm_file, + run_dbt, + run_dbt_and_capture, + write_file, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestConcurrency(BaseConcurrency): + def test_concurrency(self, project): + run_dbt(["seed", "--select", "seed"]) + results = run_dbt(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal(project.adapter, ["seed", "view_model"]) + check_relations_equal(project.adapter, ["seed", "dep"]) + check_relations_equal(project.adapter, ["seed", "table_a"]) + check_relations_equal(project.adapter, ["seed", "table_b"]) + check_table_does_not_exist(project.adapter, "invalid") + check_table_does_not_exist(project.adapter, "skip") + + rm_file(project.project_root, "seeds", "seed.csv") + write_file(seeds__update_csv, project.project_root, "seeds", "seed.csv") + + results, output = run_dbt_and_capture(["run"], expect_pass=False) + assert len(results) == 7 + check_relations_equal(project.adapter, ["seed", "view_model"]) + check_relations_equal(project.adapter, ["seed", "dep"]) + check_relations_equal(project.adapter, ["seed", "table_a"]) + check_relations_equal(project.adapter, ["seed", "table_b"]) + check_table_does_not_exist(project.adapter, "invalid") + check_table_does_not_exist(project.adapter, "skip") + + assert "PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7" in output diff --git a/tests/functional/adapter/test_data_types.py b/tests/functional/adapter/test_data_types.py index 7ec309c..a841298 100644 --- a/tests/functional/adapter/test_data_types.py +++ b/tests/functional/adapter/test_data_types.py @@ -1,34 +1,6 @@ -import pytest from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data -class TestTypeBoolean(BaseTypeBoolean): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - if profiles_config_update: - profile.update(profiles_config_update) - return profile \ No newline at end of file +class TestTypeBoolean(BaseTypeBoolean): + pass diff --git a/tests/functional/adapter/utils/test_array_append.py b/tests/functional/adapter/utils/test_array_append.py new file mode 100644 index 0000000..a83f7c3 --- /dev/null +++ b/tests/functional/adapter/utils/test_array_append.py @@ -0,0 +1,8 @@ +from dbt.tests.adapter.utils.test_array_append import ( + BaseArrayAppend, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestArrayAppend(BaseArrayAppend): + pass diff --git a/tests/functional/adapter/utils/test_array_concat.py b/tests/functional/adapter/utils/test_array_concat.py new file mode 100644 index 0000000..ddb0c51 --- /dev/null +++ b/tests/functional/adapter/utils/test_array_concat.py @@ -0,0 +1,8 @@ +from dbt.tests.adapter.utils.test_array_concat import ( + BaseArrayConcat, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestArrayConcat(BaseArrayConcat): + pass diff --git a/tests/functional/adapter/utils/test_array_construct.py b/tests/functional/adapter/utils/test_array_construct.py new file mode 100644 index 0000000..358e0f7 --- /dev/null +++ b/tests/functional/adapter/utils/test_array_construct.py @@ -0,0 +1,8 @@ +from dbt.tests.adapter.utils.test_array_construct import ( + BaseArrayConstruct, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestArrayConstruct(BaseArrayConstruct): + pass diff --git a/tests/functional/adapter/utils/test_current_timestamp.py b/tests/functional/adapter/utils/test_current_timestamp.py index 7401f7d..5c99509 100644 --- a/tests/functional/adapter/utils/test_current_timestamp.py +++ b/tests/functional/adapter/utils/test_current_timestamp.py @@ -1,36 +1,8 @@ -import pytest from dbt.tests.adapter.utils.test_current_timestamp import ( BaseCurrentTimestampNaive, ) -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data -class TestCurrentTimestampNaive(BaseCurrentTimestampNaive): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - if profiles_config_update: - profile.update(profiles_config_update) - return profile \ No newline at end of file +class TestCurrentTimestampNaive(BaseCurrentTimestampNaive): + pass From c15536a0e0ef07eeac2a7b91e1c4438bb1d0a1ad Mon Sep 17 00:00:00 2001 From: Jared LaRue Date: Tue, 8 Nov 2022 21:02:31 -0800 Subject: [PATCH 5/8] Updates require to release dbt-dremio 1.3.0 - README - versioning - copyright headers - ... --- .github/dependabot.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f942a69 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +# Copyright (C) 2022 Dremio Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + rebase-strategy: "disabled" From f6bcfe435c2c06b6122150e438a14eebe58d59ac Mon Sep 17 00:00:00 2001 From: Jared LaRue Date: Tue, 8 Nov 2022 21:02:31 -0800 Subject: [PATCH 6/8] Updates require to release dbt-dremio 1.3.0 - README - versioning - copyright headers - ... --- .github/dependabot.yml | 21 +++++++++++++++++++++ README.md | 22 +++++++++++++--------- dbt/adapters/dremio/__version__.py | 13 ++++++++++++- dev_requirements.txt | 6 +++++- setup.py | 2 +- tests/conftest.py | 11 +++++++++++ tests/fixtures/profiles.py | 11 +++++++++++ tests/unit/test_connection.py | 11 +++++++++++ 8 files changed, 85 insertions(+), 12 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f942a69 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +# Copyright (C) 2022 Dremio Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + rebase-strategy: "disabled" diff --git a/README.md b/README.md index 408cd76..6eabfe9 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. -## dbt-dremio preview version 1.1.0b ## - -The `dbt-dremio` package contains all of the code enabling dbt to work with Dremio. For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). +## dbt-dremio version 1.3.0 ## +--- +The `dbt-dremio` package contains all of the code enabling dbt to work with [Dremio](https://www.dremio.com/). For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later). @@ -15,18 +15,22 @@ Installing the dbt-dremio package will install or update dbt-core to version 1.3 > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b. ## Getting started - +--- - [Install dbt](https://docs.getdbt.com/docs/installation) - Note that dbt-dremio requires dbt-core 1.3.0. - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community - +--- - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) - Read more on the [dbt Community Discourse](https://discourse.getdbt.com) ## Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know by opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) - - +--- +- Open bugs and feature requests can be found at [dbt-dremio's GitHub issues](https://github.com/dremio/dbt-dremio/issues). +- Want to report a bug or request a feature? Let us know by on [Slack]([Slack](http://community.getdbt.com/)), or opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) +- Want to help us build dbt-dremio? Check out the [Contributing Guide](https://github.com/dremio/dbt-dremio/blob/main/CONTRIBUTING.md). + +## Code of Conduct +--- +Everyone interacting in the dbt-dremio project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt-dremio Code of Conduct](https://github.com/dremio/dbt-dremio/blob/main/CODE_OF_CONDUCT.md). diff --git a/dbt/adapters/dremio/__version__.py b/dbt/adapters/dremio/__version__.py index b2b60a5..61a16c2 100644 --- a/dbt/adapters/dremio/__version__.py +++ b/dbt/adapters/dremio/__version__.py @@ -1 +1,12 @@ -version = "1.1.0" +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version = "1.3.0" diff --git a/dev_requirements.txt b/dev_requirements.txt index 59d8f4b..ce2eb5a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -17,4 +17,8 @@ pytest-xdist pytz tox>=3.13 twine -wheel \ No newline at end of file +wheel +agate==1.6.3 +requests==2.28.1 +setuptools==63.2.0 +third-party-license-file-generator diff --git a/setup.py b/setup.py index d58bf30..64b037d 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from setuptools import find_namespace_packages, setup package_name = "dbt-dremio" -package_version = "1.1.0" +package_version = "1.3.0" description = """The Dremio adapter plugin for dbt""" setup( diff --git a/tests/conftest.py b/tests/conftest.py index 0725695..a8a6864 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import pytest from dotenv import load_dotenv diff --git a/tests/fixtures/profiles.py b/tests/fixtures/profiles.py index eb2b44c..4e6370f 100644 --- a/tests/fixtures/profiles.py +++ b/tests/fixtures/profiles.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest from tests.functional.adapter.utils.test_utils import DATALAKE diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index a117ccf..e979f32 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest from unittest.mock import patch from dbt.exceptions import FailedToConnectException From aa9873adfb2e26050d4f63395167d7af95c7939c Mon Sep 17 00:00:00 2001 From: Jared LaRue Date: Tue, 8 Nov 2022 21:35:12 -0800 Subject: [PATCH 7/8] Address README and dev_requirments review comments --- README.md | 2 +- dev_requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6eabfe9..bd10bee 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Installing the dbt-dremio package will install or update dbt-core to version 1.3 ## Reporting bugs and contributing code --- - Open bugs and feature requests can be found at [dbt-dremio's GitHub issues](https://github.com/dremio/dbt-dremio/issues). -- Want to report a bug or request a feature? Let us know by on [Slack]([Slack](http://community.getdbt.com/)), or opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) +- Want to report a bug or request a feature? Let us know by on [Slack](https://getdbt.slack.com/archives/C049G61TKBK), or opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) - Want to help us build dbt-dremio? Check out the [Contributing Guide](https://github.com/dremio/dbt-dremio/blob/main/CONTRIBUTING.md). ## Code of Conduct diff --git a/dev_requirements.txt b/dev_requirements.txt index ce2eb5a..bd39be7 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -21,4 +21,3 @@ wheel agate==1.6.3 requests==2.28.1 setuptools==63.2.0 -third-party-license-file-generator From 644ee59a95a0bf5ddd639be544390ca7c5f4e8e7 Mon Sep 17 00:00:00 2001 From: ArgusLi Date: Wed, 9 Nov 2022 10:35:04 -0800 Subject: [PATCH 8/8] Add license to profiles.py. --- tests/fixtures/profiles.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/fixtures/profiles.py b/tests/fixtures/profiles.py index d05a505..b6a66c9 100644 --- a/tests/fixtures/profiles.py +++ b/tests/fixtures/profiles.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest from tests.functional.adapter.utils.test_utils import DATALAKE