Skip to content
This repository has been archived by the owner on Apr 8, 2024. It is now read-only.

refactor: move some hacks location #742

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test_integration_cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ jobs:
dbt: "1.1.*"
- profile: "fal"
dbt: "1.2.*"
# TODO: use dbt-athena-community instead
# dbt-athena-adapter only supports dbt-core==1.0.* for now
- profile: "athena"
dbt: "1.1.*"
Expand Down
9 changes: 9 additions & 0 deletions adapter/src/dbt/adapters/fal/wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,15 @@ def __getattr__(self, name):
else:
getattr(super(), name)

def get_relation(self, database: str, schema: str, identifier: str):
# HACK: When compiling Python models, we get an all-False quoting policy
# This does not happen in 1.4
if self._db_adapter.type() == "athena":
# and dbt-athena-community breaks for that case
self.config.quoting = {"database": True, "schema": True, "identifier": True}

return self._db_adapter.get_relation(database, schema, identifier)

Comment on lines +97 to +105
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: Does it matter if we put it in a wrapper or in impl.py? I think impl.py is better because it's more visible there. This feels more like putting something under a rug.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The way I saw this change was taking it from the "general" soluton of the Adapter Mixin and putting it in the wrapper that is actually being used, since this is 1.3-specific and will not be replicated for 1.4

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

merging and you can make a decision in your PR


def find_funcs_in_stack(funcs: Set[str]) -> bool:
import inspect
Expand Down
9 changes: 1 addition & 8 deletions adapter/src/dbt/adapters/fal_experimental/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from contextlib import contextmanager
from typing import Iterator

from dbt.adapters.base.impl import BaseAdapter, Optional
from dbt.adapters.base.impl import BaseAdapter
from dbt.adapters.base.meta import AdapterMeta, available
from dbt.adapters.base.relation import BaseRelation
from dbt.contracts.connection import AdapterResponse
Expand Down Expand Up @@ -61,13 +61,6 @@ def manifest(self) -> Manifest:
def macro_manifest(self) -> MacroManifest:
return self._db_adapter.load_macro_manifest()

def get_relation(self, database: str, schema: str, identifier: str) -> Optional[BaseRelation]:
# HACK: When compiling Python models, dbt-athena-community adapter loses quoting policy
# So we reset it manually for AthenaAdapter
if type(self._db_adapter).__name__ == "AthenaAdapter":
self.config.quoting = {'database': True, 'schema': True, 'identifier': True}
return BaseAdapter.get_relation(self, database, schema, identifier)

@telemetry.log_call("experimental_submit_python_job", config=True)
def submit_python_job(
self, parsed_model: dict, compiled_code: str
Expand Down
15 changes: 9 additions & 6 deletions adapter/src/dbt/adapters/fal_experimental/utils/environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,18 +225,21 @@ def _parse_remote_config(config: Dict[str, Any], parsed_config: Dict[str, Any])
"target_environments": [env_definition]
}

def _get_package_from_type(adapter_type: str):
SPECIAL_ADAPTERS = {
# Documented in dbt website
"athena": "dbt-athena-community",
}
return SPECIAL_ADAPTERS.get(adapter_type, f"dbt-{adapter_type}")


def _get_dbt_packages(
adapter_type: str,
is_teleport: bool = False,
is_remote: bool = False
) -> Iterator[Tuple[str, Optional[str]]]:
dbt_adapter = f"dbt-{adapter_type}"
for dbt_plugin_name in ['dbt-core', dbt_adapter]:
if dbt_plugin_name == "dbt-athena":
# dbt-athena from PyPI doesn't support dbt 1.3, but dbt-athena-community does:
# https://github.com/dbt-athena/dbt-athena
dbt_plugin_name = "dbt-athena-community"
dbt_adapter = _get_package_from_type(adapter_type)
for dbt_plugin_name in ["dbt-core", dbt_adapter]:
distribution = importlib_metadata.distribution(dbt_plugin_name)

yield dbt_plugin_name, distribution.version
Expand Down