From 94f0da8b82152d0be31bb6135cef142e654d9139 Mon Sep 17 00:00:00 2001 From: Jared <97905507+jlarue26@users.noreply.github.com> Date: Wed, 9 Nov 2022 18:59:34 +0000 Subject: [PATCH] dbt-dremio 1.3.0 Release Readiness (#66) ### Summary dbt-dremio 1.3.0 Release Readiness ### Description Update: - README - versioning - dev_requirements - THIRD_PARTY_LICENCES ### Related Issue None ### Additional Reviewers @ArgusLi @ravjotbrar Co-authored-by: ArgusLi --- .github/dependabot.yml | 21 +++++++ README.md | 22 ++++--- dbt/adapters/dremio/__version__.py | 13 +++- dev_requirements.txt | 5 +- setup.py | 2 +- tests/conftest.py | 11 ++++ tests/fixtures/profiles.py | 11 ++++ .../functional/adapter/basic/test_base_mat.py | 32 +--------- .../adapter/basic/test_docs_generate.py | 63 +------------------ .../adapter/basic/test_snapshots.py | 40 +----------- .../adapter/utils/test_array_append.py | 8 +++ .../adapter/utils/test_array_concat.py | 8 +++ .../adapter/utils/test_array_construct.py | 8 +++ tests/unit/test_connection.py | 11 ++++ 14 files changed, 111 insertions(+), 144 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 tests/functional/adapter/utils/test_array_append.py create mode 100644 tests/functional/adapter/utils/test_array_concat.py create mode 100644 tests/functional/adapter/utils/test_array_construct.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f942a69 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +# Copyright (C) 2022 Dremio Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + rebase-strategy: "disabled" diff --git a/README.md b/README.md index 7ac4b07..bd10bee 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. -## dbt-dremio version 1.3.0b beta ## - -The `dbt-dremio` package contains all of the code enabling dbt to work with Dremio. For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). +## dbt-dremio version 1.3.0 ## +--- +The `dbt-dremio` package contains all of the code enabling dbt to work with [Dremio](https://www.dremio.com/). For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile). The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later). @@ -15,18 +15,22 @@ Installing the dbt-dremio package will install or update dbt-core to version 1.3 > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b. ## Getting started - +--- - [Install dbt](https://docs.getdbt.com/docs/installation) - Note that dbt-dremio requires dbt-core 1.3.0. - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community - +--- - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) - Read more on the [dbt Community Discourse](https://discourse.getdbt.com) ## Reporting bugs and contributing code - -- Want to report a bug or request a feature? Let us know by opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) - - +--- +- Open bugs and feature requests can be found at [dbt-dremio's GitHub issues](https://github.com/dremio/dbt-dremio/issues). +- Want to report a bug or request a feature? Let us know by on [Slack](https://getdbt.slack.com/archives/C049G61TKBK), or opening [an issue](https://github.com/dremio/dbt-dremio/issues/new) +- Want to help us build dbt-dremio? Check out the [Contributing Guide](https://github.com/dremio/dbt-dremio/blob/main/CONTRIBUTING.md). + +## Code of Conduct +--- +Everyone interacting in the dbt-dremio project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt-dremio Code of Conduct](https://github.com/dremio/dbt-dremio/blob/main/CODE_OF_CONDUCT.md). diff --git a/dbt/adapters/dremio/__version__.py b/dbt/adapters/dremio/__version__.py index 1a4228d..61a16c2 100644 --- a/dbt/adapters/dremio/__version__.py +++ b/dbt/adapters/dremio/__version__.py @@ -1 +1,12 @@ -version = "1.3.0b" +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version = "1.3.0" diff --git a/dev_requirements.txt b/dev_requirements.txt index 59d8f4b..bd39be7 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -17,4 +17,7 @@ pytest-xdist pytz tox>=3.13 twine -wheel \ No newline at end of file +wheel +agate==1.6.3 +requests==2.28.1 +setuptools==63.2.0 diff --git a/setup.py b/setup.py index ad75c87..64b037d 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from setuptools import find_namespace_packages, setup package_name = "dbt-dremio" -package_version = "1.3.0b" +package_version = "1.3.0" description = """The Dremio adapter plugin for dbt""" setup( diff --git a/tests/conftest.py b/tests/conftest.py index bd6a9e1..768f4c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import pytest from dotenv import load_dotenv diff --git a/tests/fixtures/profiles.py b/tests/fixtures/profiles.py index d05a505..b6a66c9 100644 --- a/tests/fixtures/profiles.py +++ b/tests/fixtures/profiles.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest from tests.functional.adapter.utils.test_utils import DATALAKE diff --git a/tests/functional/adapter/basic/test_base_mat.py b/tests/functional/adapter/basic/test_base_mat.py index 8d11184..f6c5381 100644 --- a/tests/functional/adapter/basic/test_base_mat.py +++ b/tests/functional/adapter/basic/test_base_mat.py @@ -29,7 +29,7 @@ run_dbt, check_result_nodes_by_name, ) -from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema, dbt_profile_data # Unable to insert variable into docstring, so "dbt_test_source" is hardcoded schema_base_yml = """ @@ -65,36 +65,6 @@ def project_config_update(self): "vars": {"dremio:reflections": "false"}, } - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - def test_base(self, project): # seed command diff --git a/tests/functional/adapter/basic/test_docs_generate.py b/tests/functional/adapter/basic/test_docs_generate.py index e68385b..2e465af 100644 --- a/tests/functional/adapter/basic/test_docs_generate.py +++ b/tests/functional/adapter/basic/test_docs_generate.py @@ -17,7 +17,6 @@ from tests.functional.adapter.utils.test_utils import ( base_expected_catalog, expected_references_catalog, - DATALAKE, ) from dbt.tests.adapter.basic.test_docs_generate import ( BaseDocsGenerate, @@ -30,6 +29,7 @@ get_artifact, ) from dbt.tests.adapter.basic.expected_catalog import no_stats +from tests.fixtures.profiles import unique_schema, dbt_profile_data # required to explicitly use alternate_schema # otherwise will use unique_schema under profiles fixture @@ -79,15 +79,6 @@ def models(self): "model.sql": models__model_sql, } - # Override this fixture to prepend our schema with DATALAKE - # This ensures the schema works with our datalake - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - # Override this fixture to prevent (twin_strategy) creating a view for seeds @pytest.fixture(scope="class") def project_config_update(self, unique_schema): @@ -104,29 +95,6 @@ def project_config_update(self, unique_schema): }, } - # Override this fixture to set root_path=schema - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - # Override this fixture to change expected types to Dremio types @pytest.fixture(scope="class") def expected_catalog(self, project): @@ -161,12 +129,6 @@ def test_run_and_generate(self, project, expected_catalog): class TestBaseDocsGenReferencesDremio(BaseDocsGenReferences): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema # Override this fixture to allow (twin_strategy) to create a view for seeds # The creation of some models looks for the seed under the database/schema @@ -184,29 +146,6 @@ def project_config_update(self, unique_schema): }, } - # Override this fixture to set root_path=schema - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - # Override this fixture to change expected types to Dremio types @pytest.fixture(scope="class") def expected_catalog(self, project): diff --git a/tests/functional/adapter/basic/test_snapshots.py b/tests/functional/adapter/basic/test_snapshots.py index cff387a..9f7d468 100644 --- a/tests/functional/adapter/basic/test_snapshots.py +++ b/tests/functional/adapter/basic/test_snapshots.py @@ -16,41 +16,11 @@ from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp from tests.functional.adapter.utils.test_utils import DATALAKE +from tests.fixtures.profiles import unique_schema @pytest.mark.skip(reason="https://github.com/dremio/dbt-dremio/issues/20") class TestSnapshotCheckColsDremio(BaseSnapshotCheckCols): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - - @pytest.fixture(scope="class") - def dbt_profile_data( - self, unique_schema, dbt_profile_target, profiles_config_update - ): - profile = { - "config": {"send_anonymous_usage_stats": False}, - "test": { - "outputs": { - "default": {}, - }, - "target": "default", - }, - } - target = dbt_profile_target - target["schema"] = unique_schema - target["root_path"] = unique_schema - target["database"] = target["datalake"] - profile["test"]["outputs"]["default"] = target - - if profiles_config_update: - profile.update(profiles_config_update) - return profile - @pytest.fixture(scope="class") def project_config_update(self): return { @@ -60,14 +30,6 @@ def project_config_update(self): class TestSnapshotTimestampDremio(BaseSnapshotTimestamp): - @pytest.fixture(scope="class") - def unique_schema(self, request, prefix) -> str: - test_file = request.module.__name__ - # We only want the last part of the name - test_file = test_file.split(".")[-1] - unique_schema = f"{DATALAKE}.{prefix}_{test_file}" - return unique_schema - @pytest.fixture(scope="class") def dbt_profile_data( self, unique_schema, dbt_profile_target, profiles_config_update diff --git a/tests/functional/adapter/utils/test_array_append.py b/tests/functional/adapter/utils/test_array_append.py new file mode 100644 index 0000000..a83f7c3 --- /dev/null +++ b/tests/functional/adapter/utils/test_array_append.py @@ -0,0 +1,8 @@ +from dbt.tests.adapter.utils.test_array_append import ( + BaseArrayAppend, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestArrayAppend(BaseArrayAppend): + pass diff --git a/tests/functional/adapter/utils/test_array_concat.py b/tests/functional/adapter/utils/test_array_concat.py new file mode 100644 index 0000000..ddb0c51 --- /dev/null +++ b/tests/functional/adapter/utils/test_array_concat.py @@ -0,0 +1,8 @@ +from dbt.tests.adapter.utils.test_array_concat import ( + BaseArrayConcat, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestArrayConcat(BaseArrayConcat): + pass diff --git a/tests/functional/adapter/utils/test_array_construct.py b/tests/functional/adapter/utils/test_array_construct.py new file mode 100644 index 0000000..358e0f7 --- /dev/null +++ b/tests/functional/adapter/utils/test_array_construct.py @@ -0,0 +1,8 @@ +from dbt.tests.adapter.utils.test_array_construct import ( + BaseArrayConstruct, +) +from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +class TestArrayConstruct(BaseArrayConstruct): + pass diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index a117ccf..e979f32 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -1,3 +1,14 @@ +# Copyright (C) 2022 Dremio Corporation +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest from unittest.mock import patch from dbt.exceptions import FailedToConnectException