From 8dc2c119da3c7c7aa97c9b3968b8fbf1bdf80010 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 1 May 2023 18:10:27 -0700 Subject: [PATCH 1/4] Support profiles that are simply 'type: duckdb' by correctly configuring the in-memory database in dbt-duckdb --- dbt/adapters/duckdb/credentials.py | 33 ++++++++++++++++-------------- tests/conftest.py | 6 +++--- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/dbt/adapters/duckdb/credentials.py b/dbt/adapters/duckdb/credentials.py index ea9853f0..05d97411 100644 --- a/dbt/adapters/duckdb/credentials.py +++ b/dbt/adapters/duckdb/credentials.py @@ -118,22 +118,25 @@ class DuckDBCredentials(Credentials): @classmethod def __pre_deserialize__(cls, data: Dict[Any, Any]) -> Dict[Any, Any]: - data = super().__pre_deserialize__(data) - path = data.get("path") - if path and "database" not in data and duckdb.__version__ >= "0.7.0": - if path == ":memory:": - data["database"] = "memory" - else: - parsed = urlparse(path) - base_file = os.path.basename(parsed.path) - db = os.path.splitext(base_file)[0] - if db: - data["database"] = db + if duckdb.__version__ >= "0.7.0": + data = super().__pre_deserialize__(data) + if "database" not in data: + # if no database is specified in the profile, figure out + # the database value to use from the path argument + path = data.get("path") + if path is None or path == ":memory:": + data["database"] = "memory" else: - raise dbt.exceptions.DbtRuntimeError( - "Unable to determine database name from path" - " and no database was specified in profile" - ) + parsed = urlparse(path) + base_file = os.path.basename(parsed.path) + db = os.path.splitext(base_file)[0] + if db: + data["database"] = db + else: + raise dbt.exceptions.DbtRuntimeError( + "Unable to determine database name from path" + " and no database was specified in profile" + ) return data @property diff --git a/tests/conftest.py b/tests/conftest.py index a870f471..46cab077 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,10 +50,10 @@ def dbt_profile_target(profile_type, bv_server_process, tmp_path_factory): "port": 5433, "user": "test", } - elif profile_type == "memory": - profile["path"] = ":memory:" elif profile_type == "file": profile["path"] = str(tmp_path_factory.getbasetemp() / "tmp.db") + elif profile_type == "memory": + pass # use the default path-less profile else: raise ValueError(f"Invalid profile type '{profile_type}'") @@ -71,4 +71,4 @@ def skip_by_profile_type(profile_type, request): @pytest.fixture(scope="session") def test_data_path(): test_dir = os.path.dirname(os.path.abspath(__file__)) - return os.path.join(test_dir, "data") \ No newline at end of file + return os.path.join(test_dir, "data") From 1e4a55d2c40c14c7269506e99a9e83b4e516609d Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 1 May 2023 20:55:57 -0700 Subject: [PATCH 2/4] update the attach test --- tests/functional/adapter/test_attach.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/test_attach.py b/tests/functional/adapter/test_attach.py index 82b7e485..9f69e45c 100644 --- a/tests/functional/adapter/test_attach.py +++ b/tests/functional/adapter/test_attach.py @@ -31,6 +31,7 @@ SELECT * FROM {{ ref('source_model') }} """ + @pytest.mark.skip_profile("buenavista") class TestAttachedDatabase: @pytest.fixture(scope="class") @@ -45,14 +46,12 @@ def attach_test_db(self): @pytest.fixture(scope="class") def profiles_config_update(self, dbt_profile_target, attach_test_db): - if "path" not in dbt_profile_target: - return {} return { "test": { "outputs": { "dev": { "type": "duckdb", - "path": dbt_profile_target["path"], + "path": dbt_profile_target.get("path", ":memory:"), "attach": [{"path": attach_test_db}], } }, From 9f0acd3678a56d5f6e64738380402bee29e6638d Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 1 May 2023 21:17:13 -0700 Subject: [PATCH 3/4] more tests that need updating --- tests/functional/fsspec/test_filesystems.py | 7 +++++-- tests/functional/plugins/test_plugins.py | 5 +---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/functional/fsspec/test_filesystems.py b/tests/functional/fsspec/test_filesystems.py index c17da46b..c563893e 100644 --- a/tests/functional/fsspec/test_filesystems.py +++ b/tests/functional/fsspec/test_filesystems.py @@ -9,6 +9,7 @@ WHERE conf = 'West' """ + @pytest.mark.skip_profile("buenavista") class TestFilesystems: @pytest.fixture(scope="class") @@ -17,8 +18,10 @@ def dbt_profile_target(self, dbt_profile_target): return dbt_profile_target return { "type": "duckdb", - "path": dbt_profile_target["path"], - "filesystems": [{"fs": "github", "org": "jwills", "repo": "nba_monte_carlo"}], + "path": dbt_profile_target.get("path", ":memory:"), + "filesystems": [ + {"fs": "github", "org": "jwills", "repo": "nba_monte_carlo"} + ], } @pytest.fixture(scope="class") diff --git a/tests/functional/plugins/test_plugins.py b/tests/functional/plugins/test_plugins.py index ea5b6e34..09e31178 100644 --- a/tests/functional/plugins/test_plugins.py +++ b/tests/functional/plugins/test_plugins.py @@ -73,9 +73,6 @@ def sqlite_test_db(self): @pytest.fixture(scope="class") def profiles_config_update(self, dbt_profile_target, sqlite_test_db): - if "path" not in dbt_profile_target: - return {} - config = {"connection_url": f"sqlite:///{sqlite_test_db}"} plugins = [ {"name": "excel", "impl": "excel"}, @@ -87,7 +84,7 @@ def profiles_config_update(self, dbt_profile_target, sqlite_test_db): "outputs": { "dev": { "type": "duckdb", - "path": dbt_profile_target["path"], + "path": dbt_profile_target.get("path", ":memory:"), "plugins": plugins, } }, From 9a4159a4a4a6d5c89208c216d09e94cd956c69f4 Mon Sep 17 00:00:00 2001 From: Josh Wills Date: Mon, 1 May 2023 21:31:14 -0700 Subject: [PATCH 4/4] oh yep that also --- tests/functional/fsspec/test_filesystems.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/functional/fsspec/test_filesystems.py b/tests/functional/fsspec/test_filesystems.py index c563893e..b7815980 100644 --- a/tests/functional/fsspec/test_filesystems.py +++ b/tests/functional/fsspec/test_filesystems.py @@ -14,8 +14,6 @@ class TestFilesystems: @pytest.fixture(scope="class") def dbt_profile_target(self, dbt_profile_target): - if "path" not in dbt_profile_target: - return dbt_profile_target return { "type": "duckdb", "path": dbt_profile_target.get("path", ":memory:"),