diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index f942a69..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2022 Dremio Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 2 -updates: - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "daily" - rebase-strategy: "disabled" diff --git a/README.md b/README.md index bd10bee..7ac4b07 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. -## dbt-dremio version 1.3.0 ## ---- -The `dbt-dremio` package contains all of the code enabling dbt to work with [Dremio](https://www.dremio.com/). For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). +## dbt-dremio version 1.3.0b beta ## + +The `dbt-dremio` package contains all of the code enabling dbt to work with Dremio. For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later). @@ -15,22 +15,18 @@ Installing the dbt-dremio package will install or update dbt-core to version 1.3 > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b. ## Getting started ---- + - [Install dbt](https://docs.getdbt.com/docs/installation) - Note that dbt-dremio requires dbt-core 1.3.0. - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community ---- + - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) - Read more on the [dbt Community Discourse](https://discourse.getdbt.com) ## Reporting bugs and contributing code ---- -- Open bugs and feature requests can be found at [dbt-dremio's GitHub issues](https://github.com/dremio/dbt-dremio/issues). -- Want to report a bug or request a feature? Let us know by on [Slack](https://getdbt.slack.com/archives/C049G61TKBK), or opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) -- Want to help us build dbt-dremio? Check out the [Contributing Guide](https://github.com/dremio/dbt-dremio/blob/main/CONTRIBUTING.md). - -## Code of Conduct ---- -Everyone interacting in the dbt-dremio project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt-dremio Code of Conduct](https://github.com/dremio/dbt-dremio/blob/main/CODE_OF_CONDUCT.md). + +- Want to report a bug or request a feature? Let us know by opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) + + diff --git a/dbt/adapters/dremio/__version__.py b/dbt/adapters/dremio/__version__.py index 61a16c2..1a4228d 100644 --- a/dbt/adapters/dremio/__version__.py +++ b/dbt/adapters/dremio/__version__.py @@ -1,12 +1 @@ -# Copyright (C) 2022 Dremio Corporation -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version = "1.3.0" +version = "1.3.0b" diff --git a/dev_requirements.txt b/dev_requirements.txt index bd39be7..59d8f4b 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -17,7 +17,4 @@ pytest-xdist pytz tox>=3.13 twine -wheel -agate==1.6.3 -requests==2.28.1 -setuptools==63.2.0 +wheel \ No newline at end of file diff --git a/setup.py b/setup.py index 64b037d..ad75c87 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from setuptools import find_namespace_packages, setup package_name = "dbt-dremio" -package_version = "1.3.0" +package_version = "1.3.0b" description = """The Dremio adapter plugin for dbt""" setup( diff --git a/tests/conftest.py b/tests/conftest.py index 768f4c0..bd6a9e1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,3 @@ -# Copyright (C) 2022 Dremio Corporation -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import os import pytest from dotenv import load_dotenv diff --git a/tests/fixtures/profiles.py b/tests/fixtures/profiles.py index b6a66c9..d05a505 100644 --- a/tests/fixtures/profiles.py +++ b/tests/fixtures/profiles.py @@ -1,14 +1,3 @@ -# Copyright (C) 2022 Dremio Corporation -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import pytest from tests.functional.adapter.utils.test_utils import DATALAKE diff --git a/tests/functional/adapter/basic/test_base_mat.py b/tests/functional/adapter/basic/test_base_mat.py index f6c5381..8d11184 100644 --- a/tests/functional/adapter/basic/test_base_mat.py +++ b/tests/functional/adapter/basic/test_base_mat.py @@ -29,7 +29,7 @@ run_dbt, check_result_nodes_by_name, ) -from tests.fixtures.profiles import unique_schema, dbt_profile_data +from tests.functional.adapter.utils.test_utils import DATALAKE # Unable to insert variable into docstring, so "dbt_test_source" is hardcoded schema_base_yml = """ @@ -65,6 +65,36 @@ def project_config_update(self): "vars": {"dremio:reflections": "false"}, } + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + def test_base(self, project): # seed command diff --git a/tests/functional/adapter/basic/test_docs_generate.py b/tests/functional/adapter/basic/test_docs_generate.py index 2e465af..e68385b 100644 --- a/tests/functional/adapter/basic/test_docs_generate.py +++ b/tests/functional/adapter/basic/test_docs_generate.py @@ -17,6 +17,7 @@ from tests.functional.adapter.utils.test_utils import ( base_expected_catalog, expected_references_catalog, + DATALAKE, ) from dbt.tests.adapter.basic.test_docs_generate import ( BaseDocsGenerate, @@ -29,7 +30,6 @@ get_artifact, ) from dbt.tests.adapter.basic.expected_catalog import no_stats -from tests.fixtures.profiles import unique_schema, dbt_profile_data # required to explicitly use alternate_schema # otherwise will use unique_schema under profiles fixture @@ -79,6 +79,15 @@ def models(self): "model.sql": models__model_sql, } + # Override this fixture to prepend our schema with DATALAKE + # This ensures the schema works with our datalake + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + # Override this fixture to prevent (twin_strategy) creating a view for seeds @pytest.fixture(scope="class") def project_config_update(self, unique_schema): @@ -95,6 +104,29 @@ def project_config_update(self, unique_schema): }, } + # Override this fixture to set root_path=schema + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + # Override this fixture to change expected types to Dremio types @pytest.fixture(scope="class") def expected_catalog(self, project): @@ -129,6 +161,12 @@ def test_run_and_generate(self, project, expected_catalog): class TestBaseDocsGenReferencesDremio(BaseDocsGenReferences): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema # Override this fixture to allow (twin_strategy) to create a view for seeds # The creation of some models looks for the seed under the database/schema @@ -146,6 +184,29 @@ def project_config_update(self, unique_schema): }, } + # Override this fixture to set root_path=schema + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + # Override this fixture to change expected types to Dremio types @pytest.fixture(scope="class") def expected_catalog(self, project): diff --git a/tests/functional/adapter/basic/test_snapshots.py b/tests/functional/adapter/basic/test_snapshots.py index 9f7d468..cff387a 100644 --- a/tests/functional/adapter/basic/test_snapshots.py +++ b/tests/functional/adapter/basic/test_snapshots.py @@ -16,11 +16,41 @@ from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from tests.functional.adapter.utils.test_utils import DATALAKE -from tests.fixtures.profiles import unique_schema @pytest.mark.skip(reason="https://github.com/dremio/dbt-dremio/issues/20") class TestSnapshotCheckColsDremio(BaseSnapshotCheckCols): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "config": {"send_anonymous_usage_stats": False}, + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = unique_schema + target["database"] = target["datalake"] + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + @pytest.fixture(scope="class") def project_config_update(self): return { @@ -30,6 +60,14 @@ def project_config_update(self): class TestSnapshotTimestampDremio(BaseSnapshotTimestamp): + @pytest.fixture(scope="class") + def unique_schema(self, request, prefix) -> str: + test_file = request.module.__name__ + # We only want the last part of the name + test_file = test_file.split(".")[-1] + unique_schema = f"{DATALAKE}.{prefix}_{test_file}" + return unique_schema + @pytest.fixture(scope="class") def dbt_profile_data( self, unique_schema, dbt_profile_target, profiles_config_update diff --git a/tests/functional/adapter/utils/test_array_append.py b/tests/functional/adapter/utils/test_array_append.py deleted file mode 100644 index a83f7c3..0000000 --- a/tests/functional/adapter/utils/test_array_append.py +++ /dev/null @@ -1,8 +0,0 @@ -from dbt.tests.adapter.utils.test_array_append import ( - BaseArrayAppend, -) -from tests.fixtures.profiles import unique_schema, dbt_profile_data - - -class TestArrayAppend(BaseArrayAppend): - pass diff --git a/tests/functional/adapter/utils/test_array_concat.py b/tests/functional/adapter/utils/test_array_concat.py deleted file mode 100644 index ddb0c51..0000000 --- a/tests/functional/adapter/utils/test_array_concat.py +++ /dev/null @@ -1,8 +0,0 @@ -from dbt.tests.adapter.utils.test_array_concat import ( - BaseArrayConcat, -) -from tests.fixtures.profiles import unique_schema, dbt_profile_data - - -class TestArrayConcat(BaseArrayConcat): - pass diff --git a/tests/functional/adapter/utils/test_array_construct.py b/tests/functional/adapter/utils/test_array_construct.py deleted file mode 100644 index 358e0f7..0000000 --- a/tests/functional/adapter/utils/test_array_construct.py +++ /dev/null @@ -1,8 +0,0 @@ -from dbt.tests.adapter.utils.test_array_construct import ( - BaseArrayConstruct, -) -from tests.fixtures.profiles import unique_schema, dbt_profile_data - - -class TestArrayConstruct(BaseArrayConstruct): - pass diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index e979f32..a117ccf 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -1,14 +1,3 @@ -# Copyright (C) 2022 Dremio Corporation -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - import pytest from unittest.mock import patch from dbt.exceptions import FailedToConnectException