Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Require duckdb >= 0.7.0 for dbt-duckdb going forward and remove some code... #212

Merged
merged 1 commit into from
Jul 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 27 additions & 30 deletions dbt/adapters/duckdb/credentials.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
from typing import Tuple
from urllib.parse import urlparse

import duckdb

import dbt.exceptions
from dbt.adapters.base import Credentials
from dbt.dataclass_schema import dbtClassMixin
Expand Down Expand Up @@ -124,36 +122,35 @@ class DuckDBCredentials(Credentials):

@classmethod
def __pre_deserialize__(cls, data: Dict[Any, Any]) -> Dict[Any, Any]:
if duckdb.__version__ >= "0.7.0":
data = super().__pre_deserialize__(data)
path = data.get("path")
path_db = None
if path is None or path == ":memory:":
path_db = "memory"
else:
parsed = urlparse(path)
base_file = os.path.basename(parsed.path)
path_db = os.path.splitext(base_file)[0]
# For MotherDuck, turn on disable_transactions unless
# it's explicitly set already by the user
if parsed.scheme in {"md", "motherduck"}:
if "disable_transactions" not in data:
data["disable_transactions"] = True
if path_db == "":
path_db = "my_db"

if path_db and "database" not in data:
data["database"] = path_db
elif path_db and data["database"] != path_db:
if not data.get("remote"):
raise dbt.exceptions.DbtRuntimeError(
"Inconsistency detected between 'path' and 'database' fields in profile; "
f"the 'database' property must be set to '{path_db}' to match the 'path'"
)
elif not path_db:
data = super().__pre_deserialize__(data)
path = data.get("path")
path_db = None
if path is None or path == ":memory:":
path_db = "memory"
else:
parsed = urlparse(path)
base_file = os.path.basename(parsed.path)
path_db = os.path.splitext(base_file)[0]
# For MotherDuck, turn on disable_transactions unless
# it's explicitly set already by the user
if parsed.scheme in {"md", "motherduck"}:
if "disable_transactions" not in data:
data["disable_transactions"] = True
if path_db == "":
path_db = "my_db"

if path_db and "database" not in data:
data["database"] = path_db
elif path_db and data["database"] != path_db:
if not data.get("remote"):
raise dbt.exceptions.DbtRuntimeError(
"Unable to determine target database name from 'path' field in profile"
"Inconsistency detected between 'path' and 'database' fields in profile; "
f"the 'database' property must be set to '{path_db}' to match the 'path'"
)
elif not path_db:
raise dbt.exceptions.DbtRuntimeError(
"Unable to determine target database name from 'path' field in profile"
)
return data

@property
Expand Down
5 changes: 0 additions & 5 deletions dbt/adapters/duckdb/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from typing import Sequence

import agate
import duckdb

from dbt.adapters.base import BaseRelation
from dbt.adapters.base.column import Column
Expand Down Expand Up @@ -96,10 +95,6 @@ def store_relation(
def external_root(self) -> str:
return self.config.credentials.external_root

@available
def use_database(self) -> bool:
return duckdb.__version__ >= "0.7.0"

@available
def get_binding_char(self):
return DuckDBConnectionManager.env().get_binding_char()
Expand Down
24 changes: 7 additions & 17 deletions dbt/include/duckdb/macros/adapters.sql
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@

{% macro duckdb__create_schema(relation) -%}
{%- call statement('create_schema') -%}
create schema if not exists {{ relation.without_identifier().include(database=adapter.use_database()) }}
create schema if not exists {{ relation.without_identifier() }}
{%- endcall -%}
{% endmacro %}

{% macro duckdb__drop_schema(relation) -%}
{%- call statement('drop_schema') -%}
drop schema if exists {{ relation.without_identifier().include(database=adapter.use_database()) }} cascade
drop schema if exists {{ relation.without_identifier() }} cascade
{%- endcall -%}
{% endmacro %}

Expand All @@ -27,9 +27,7 @@
select count(*)
from system.information_schema.schemata
where schema_name = '{{ schema }}'
{% if adapter.use_database() %}
and catalog_name = '{{ information_schema.database }}'
{% endif %}
{%- endset %}
{{ return(run_query(sql)) }}
{% endmacro %}
Expand Down Expand Up @@ -57,7 +55,7 @@
{{ sql_header if sql_header is not none }}

create {% if temporary: -%}temporary{%- endif %} table
{{ relation.include(database=(not temporary and adapter.use_database()), schema=(not temporary)) }}
{{ relation.include(database=(not temporary), schema=(not temporary)) }}
{% if contract_config.enforced and not temporary %}
{#-- DuckDB doesnt support constraints on temp tables --#}
{{ get_table_columns_and_constraints() }} ;
Expand Down Expand Up @@ -89,7 +87,7 @@ def materialize(df, con):
if pyarrow_available and isinstance(df, pyarrow.Table):
# https://github.com/duckdb/duckdb/issues/6584
import pyarrow.dataset
con.execute('create table {{ relation.include(database=adapter.use_database()) }} as select * from df')
con.execute('create table {{ relation }} as select * from df')
{% endmacro %}

{% macro duckdb__create_view_as(relation, sql) -%}
Expand All @@ -100,7 +98,7 @@ def materialize(df, con):
{%- set sql_header = config.get('sql_header', none) -%}

{{ sql_header if sql_header is not none }}
create view {{ relation.include(database=adapter.use_database()) }} as (
create view {{ relation }} as (
{{ sql }}
);
{% endmacro %}
Expand All @@ -119,7 +117,7 @@ def materialize(df, con):
{% if relation.schema %}
and table_schema = '{{ relation.schema }}'
{% endif %}
{% if adapter.use_database() and relation.database %}
{% if relation.database %}
and table_catalog = '{{ relation.database }}'
{% endif %}
order by ordinal_position
Expand All @@ -142,22 +140,14 @@ def materialize(df, con):
END as type
from system.information_schema.tables
where table_schema = '{{ schema_relation.schema }}'
{% if adapter.use_database() %}
and table_catalog = '{{ schema_relation.database }}'
{% endif %}
{% endcall %}
{{ return(load_result('list_relations_without_caching').table) }}
{% endmacro %}

{% macro duckdb__drop_relation(relation) -%}
{% call statement('drop_relation', auto_begin=False) -%}
drop {{ relation.type }} if exists {{ relation.include(database=adapter.use_database()) }} cascade
{%- endcall %}
{% endmacro %}

{% macro duckdb__truncate_relation(relation) -%}
{% call statement('truncate_relation') -%}
DELETE FROM {{ relation.include(database=adapter.use_database()) }} WHERE 1=1
drop {{ relation.type }} if exists {{ relation }} cascade
{%- endcall %}
{% endmacro %}

Expand Down
2 changes: 1 addition & 1 deletion dbt/include/duckdb/macros/materializations/external.sql
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
{{ write_to_file(temp_relation, location, write_options) }}
-- create a view on top of the location
{% call statement('main', language='sql') -%}
create or replace view {{ intermediate_relation.include(database=adapter.use_database()) }} as (
create or replace view {{ intermediate_relation }} as (
select * from '{{ read_location }}'
);
{%- endcall %}
Expand Down
2 changes: 1 addition & 1 deletion dbt/include/duckdb/macros/utils/upstream.sql
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
{% do upstream_schemas.update({upstream_rel.schema: None}) %}
{% endif %}
{% call statement('main', language='sql') -%}
create or replace view {{ upstream_rel.include(database=adapter.use_database()) }} as (
create or replace view {{ upstream_rel }} as (
select * from '{{ upstream_location }}'
);
{%- endcall %}
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def _dbt_duckdb_version():
include_package_data=True,
install_requires=[
"dbt-core~=1.5.0",
"duckdb>=0.5.0",
"duckdb>=0.7.0",
],
extras_require={
"glue": ["boto3", "mypy-boto3-glue"],
Expand Down