diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 3124384975a..ff9a78dc612 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -12,7 +12,7 @@ Policy, Path, ) -from dbt.exceptions import InternalException +from dbt.exceptions import InternalException, ApproximateMatch from dbt.node_types import NodeType from dbt.utils import filter_null_values, deep_merge, classproperty @@ -99,7 +99,7 @@ def matches( if approximate_match and not exact_match: target = self.create(database=database, schema=schema, identifier=identifier) - dbt.exceptions.approximate_relation_match(target, self) + raise ApproximateMatch(target, self) return exact_match diff --git a/core/dbt/jinja_exceptions.py b/core/dbt/context/exceptions_jinja.py similarity index 99% rename from core/dbt/jinja_exceptions.py rename to core/dbt/context/exceptions_jinja.py index 67c1fbc30c0..7b1f08d33a9 100644 --- a/core/dbt/jinja_exceptions.py +++ b/core/dbt/context/exceptions_jinja.py @@ -4,6 +4,7 @@ from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning + from dbt.exceptions import ( RuntimeException, MissingConfig, diff --git a/core/dbt/context/macro_resolver.py b/core/dbt/context/macro_resolver.py index 2766dc4130c..2caf97102f0 100644 --- a/core/dbt/context/macro_resolver.py +++ b/core/dbt/context/macro_resolver.py @@ -1,6 +1,6 @@ from typing import Dict, MutableMapping, Optional from dbt.contracts.graph.parsed import ParsedMacro -from dbt.exceptions import raise_duplicate_macro_name, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, raise_compiler_error from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME from dbt.clients.jinja import MacroGenerator @@ -86,7 +86,7 @@ def _add_macro_to( package_namespaces[macro.package_name] = namespace if macro.name in namespace: - raise_duplicate_macro_name(macro, macro, macro.package_name) + raise DuplicateMacroName(macro, macro, macro.package_name) package_namespaces[macro.package_name][macro.name] = macro def add_macro(self, macro: ParsedMacro): diff --git a/core/dbt/context/macros.py b/core/dbt/context/macros.py index dccd376b876..2f6906130ec 100644 --- a/core/dbt/context/macros.py +++ b/core/dbt/context/macros.py @@ -3,7 +3,7 @@ from dbt.clients.jinja import MacroGenerator, MacroStack from dbt.contracts.graph.parsed import ParsedMacro from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME -from dbt.exceptions import raise_duplicate_macro_name, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, raise_compiler_error FlatNamespace = Dict[str, MacroGenerator] @@ -122,7 +122,7 @@ def _add_macro_to( hierarchy[macro.package_name] = namespace if macro.name in namespace: - raise_duplicate_macro_name(macro_func.macro, macro, macro.package_name) + raise DuplicateMacroName(macro_func.macro, macro, macro.package_name) hierarchy[macro.package_name][macro.name] = macro_func def add_macro(self, macro: ParsedMacro, ctx: Dict[str, Any]): diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 6a4f7125635..a5369edd17f 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -19,13 +19,14 @@ from dbt.clients import agate_helper from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack from dbt.config import RuntimeConfig, Project -from .base import contextmember, contextproperty, Var -from .configured import FQNLookup -from .context_config import ContextConfig from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER +from dbt.context.base import contextmember, contextproperty, Var +from dbt.context.configured import FQNLookup +from dbt.context.context_config import ContextConfig +from dbt.context.exceptions_jinja import wrapped_exports from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace -from .macros import MacroNamespaceBuilder, MacroNamespace -from .manifest import ManifestContext +from dbt.context.macros import MacroNamespaceBuilder, MacroNamespace +from dbt.context.manifest import ManifestContext from dbt.contracts.connection import AdapterResponse from dbt.contracts.graph.manifest import Manifest, Disabled from dbt.contracts.graph.compiled import ( @@ -58,7 +59,6 @@ raise_parsing_error, disallow_secret_env_var, ) -from dbt.jinja_exceptions import wrapped_exports from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 21052d88c3b..f2a5ba08072 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,29 +1,13 @@ import builtins from typing import NoReturn, Optional, Mapping, Any +from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning from dbt.events.contextvars import get_node_info + from dbt.node_types import NodeType -from dbt.jinja_exceptions import ( - warn, - missing_config, - missing_materialization, - missing_relation, - raise_ambiguous_alias, - raise_ambiguous_catalog_match, - raise_cache_inconsistent, - raise_dataclass_not_dict, - # raise_compiler_error, - raise_database_error, - raise_dep_not_found, - raise_dependency_error, - raise_duplicate_patch_name, - raise_duplicate_resource_name, - raise_invalid_property_yml_version, - raise_not_implemented, - relation_wrong_type, -) + import dbt.dataclass_schema @@ -38,6 +22,16 @@ def validator_error_message(exc): return "at path {}: {}".format(path, exc.message) +def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: + if path_1 == path_2: + return f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" + else: + return ( + f"remove the {type_name} entry for {name} in one of these files:\n" + f" - {path_1!s}\n{path_2!s}" + ) + + class Exception(builtins.Exception): CODE = -32000 MESSAGE = "Server Error" @@ -715,155 +709,113 @@ def get_relation_returned_multiple_results(kwargs, matches): multiple_matching_relations(kwargs, matches) -def approximate_relation_match(target, relation): - raise_compiler_error( - "When searching for a relation, dbt found an approximate match. " - "Instead of guessing \nwhich relation to use, dbt will move on. " - "Please delete {relation}, or rename it to be less ambiguous." - "\nSearched for: {target}\nFound: {relation}".format(target=target, relation=relation) - ) - +# context level exceptions +class DuplicateMacroName(CompilationException): + def __init__(self, node_1, node_2, namespace): + self.node_1 = node_1 + self.node_2 = node_2 + self.namespace = namespace + super().__init__(self.get_message()) -def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: - duped_name = node_1.name - if node_1.package_name != node_2.package_name: - extra = ' ("{}" and "{}" are both in the "{}" namespace)'.format( - node_1.package_name, node_2.package_name, namespace - ) - else: - extra = "" + def get_message(self) -> str: + duped_name = self.node_1.name + if self.node_1.package_name != self.node_2.package_name: + extra = ' ("{}" and "{}" are both in the "{}" namespace)'.format( + self.node_1.package_name, self.node_2.package_name, self.namespace + ) + else: + extra = "" - raise_compiler_error( - 'dbt found two macros with the name "{}" in the namespace "{}"{}. ' - "Since these macros have the same name and exist in the same " - "namespace, dbt will be unable to decide which to call. To fix this, " - "change the name of one of these macros:\n- {} ({})\n- {} ({})".format( - duped_name, - namespace, - extra, - node_1.unique_id, - node_1.original_file_path, - node_2.unique_id, - node_2.original_file_path, + msg = ( + f'dbt found two macros with the name "{duped_name}" in the namespace "{self.namespace}"{extra}. ' + "Since these macros have the same name and exist in the same " + "namespace, dbt will be unable to decide which to call. To fix this, " + f"change the name of one of these macros:\n- {self.node_1.unique_id} " + f"({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" ) - ) + return msg -def raise_patch_targets_not_found(patches): - patch_list = "\n\t".join( - "model {} (referenced in path {})".format(p.name, p.original_file_path) - for p in patches.values() - ) - raise_compiler_error( - "dbt could not find models for the following patches:\n\t{}".format(patch_list) - ) +# parser level exceptions +class DuplicateSourcePatchName(CompilationException): + def __init__(self, patch_1, patch_2): + self.patch_1 = patch_1 + self.patch_2 = patch_2 + super().__init__(self.get_message()) -def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: - if path_1 == path_2: - return f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" - else: - return ( - f"remove the {type_name} entry for {name} in one of these files:\n" - f" - {path_1!s}\n{path_2!s}" + def get_message(self) -> str: + name = f"{self.patch_1.overrides}.{self.patch_1.name}" + fix = _fix_dupe_msg( + self.patch_1.path, + self.patch_2.path, + name, + "sources", ) + msg = ( + f"dbt found two schema.yml entries for the same source named " + f"{self.patch_1.name} in package {self.patch_1.overrides}. Sources may only be " + f"overridden a single time. To fix this, {fix}" + ) + return msg -def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): - package_name = patch_1.package_name - name = patch_1.name - fix = _fix_dupe_msg(patch_1.original_file_path, existing_patch_path, name, "macros") - raise_compiler_error( - f"dbt found two schema.yml entries for the same macro in package " - f"{package_name} named {name}. Macros may only be described a single " - f"time. To fix this, {fix}" - ) +class DuplicateMacroPatchName(CompilationException): + def __init__(self, patch_1, existing_patch_path): + self.patch_1 = patch_1 + self.existing_patch_path = existing_patch_path + super().__init__(self.get_message()) + def get_message(self) -> str: + package_name = self.patch_1.package_name + name = self.patch_1.name + fix = _fix_dupe_msg( + self.patch_1.original_file_path, self.existing_patch_path, name, "macros" + ) + msg = ( + f"dbt found two schema.yml entries for the same macro in package " + f"{package_name} named {name}. Macros may only be described a single " + f"time. To fix this, {fix}" + ) + return msg -def raise_duplicate_source_patch_name(patch_1, patch_2): - name = f"{patch_1.overrides}.{patch_1.name}" - fix = _fix_dupe_msg( - patch_1.path, - patch_2.path, - name, - "sources", - ) - raise_compiler_error( - f"dbt found two schema.yml entries for the same source named " - f"{patch_1.name} in package {patch_1.overrides}. Sources may only be " - f"overridden a single time. To fix this, {fix}" - ) +# core level exceptions +class DuplicateAlias(AliasException): + def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str): + self.kwargs = kwargs + self.aliases = aliases + self.canonical_key = canonical_key + super().__init__(self.get_message()) -def raise_unrecognized_credentials_type(typename, supported_types): - raise_compiler_error( - 'Unrecognized credentials type "{}" - supported types are ({})'.format( - typename, ", ".join('"{}"'.format(t) for t in supported_types) + def get_message(self) -> str: + # dupe found: go through the dict so we can have a nice-ish error + key_names = ", ".join( + "{}".format(k) for k in self.kwargs if self.aliases.get(k) == self.canonical_key ) - ) + msg = f'Got duplicate keys: ({key_names}) all map to "{self.canonical_key}"' + return msg -def raise_duplicate_alias( - kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str -) -> NoReturn: - # dupe found: go through the dict so we can have a nice-ish error - key_names = ", ".join("{}".format(k) for k in kwargs if aliases.get(k) == canonical_key) +# adapters exceptions +class ApproximateMatch(CompilationException): + def __init__(self, target, relation): + self.target = target + self.relation = relation + super().__init__(self.get_message()) - raise AliasException(f'Got duplicate keys: ({key_names}) all map to "{canonical_key}"') + def get_message(self) -> str: + msg = ( + "When searching for a relation, dbt found an approximate match. " + "Instead of guessing \nwhich relation to use, dbt will move on. " + f"Please delete {self.relation}, or rename it to be less ambiguous." + f"\nSearched for: {self.target}\nFound: {self.relation}" + ) -def warn(msg, node=None): - dbt.events.functions.warn_or_error( - JinjaLogWarning(msg=msg, node_info=get_node_info()), - node=node, - ) - return "" + return msg -# Update this when a new function should be added to the -# dbt context's `exceptions` key! -CONTEXT_EXPORTS = { - fn.__name__: fn - for fn in [ - warn, - missing_config, - missing_materialization, - missing_relation, - raise_ambiguous_alias, - raise_ambiguous_catalog_match, - raise_cache_inconsistent, - raise_dataclass_not_dict, - raise_compiler_error, - raise_database_error, - raise_dep_not_found, - raise_dependency_error, - raise_duplicate_patch_name, - raise_duplicate_resource_name, - raise_invalid_property_yml_version, - raise_not_implemented, - relation_wrong_type, - ] -} - - -def wrapper(model): - def wrap(func): - @functools.wraps(func) - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - except RuntimeException as exc: - exc.add_node(model) - raise exc - - return inner - - return wrap - - -def wrapped_exports(model): - wrap = wrapper(model) - return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} # adapters exceptions class UnexpectedNull(DatabaseException): def __init__(self, field_name, source): @@ -888,9 +840,6 @@ def __init__(self, field_name, source, dt): super().__init__(msg) -# start new exceptions - - # deps exceptions class MultipleVersionGitDeps(DependencyException): def __init__(self, git, requested): @@ -1234,74 +1183,121 @@ def get_message(self) -> str: return msg -# These are placeholders to not immediately break app adapters utilizing these functions as exceptions. +# These are copies of what's in dbt/context/exceptions_jinja.py to not immediately break adapters +# utilizing these functions as exceptions. These are direct copies to avoid circular imports. # They will be removed in 1 (or 2?) versions. Issue to be created to ensure it happens. + # TODO: add deprecation to functions -def warn(msg, node=None): # type: ignore[no-redef] # noqa - return warn(msg, node) +def warn(msg, node=None): + warn_or_error(JinjaLogWarning(msg=msg, node_info=get_node_info())) + return "" -def missing_config(model, name) -> NoReturn: # type: ignore[no-redef] # noqa - missing_config(model, name) +def missing_config(model, name) -> NoReturn: + raise MissingConfig(unique_id=model.unique_id, name=name) -def missing_materialization(model, adapter_type) -> NoReturn: # type: ignore[no-redef] # noqa - missing_materialization(model, adapter_type) +def missing_materialization(model, adapter_type) -> NoReturn: + raise MissingMaterialization(model=model, adapter_type=adapter_type) -def missing_relation(relation, model=None) -> NoReturn: # type: ignore[no-redef] # noqa - missing_relation(relation, model) +def missing_relation(relation, model=None) -> NoReturn: + raise MissingRelation(relation, model) -def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: # type: ignore[no-redef] # noqa - raise_ambiguous_alias(node_1, node_2, duped_name) +def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: + raise AmbiguousAlias(node_1, node_2, duped_name) -def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: # type: ignore[no-redef] # noqa - raise_ambiguous_catalog_match(unique_id, match_1, match_2) +def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: + raise AmbiguousCatalogMatch(unique_id, match_1, match_2) -def raise_cache_inconsistent(message) -> NoReturn: # type: ignore[no-redef] # noqa - raise_cache_inconsistent(message) +# TODO: this should be improved to not format message here +def raise_cache_inconsistent(message) -> NoReturn: + raise InternalException("Cache inconsistency detected: {}".format(message)) -def raise_dataclass_not_dict(obj) -> NoReturn: # type: ignore[no-redef] # noqa - raise_dataclass_not_dict(obj) +def raise_dataclass_not_dict(obj) -> NoReturn: + raise DataclassNotDict(obj) -# this is already used all over our code so for now can't do this until it's fully -# removed from this file. otherwise casuses recurssion errors. +# TODO: add this is once it's removed above # def raise_compiler_error(msg, node=None) -> NoReturn: -# raise_compiler_error(msg, node=None) +# raise CompilationException(msg, node) + + +def raise_database_error(msg, node=None) -> NoReturn: + raise DatabaseException(msg, node) -def raise_database_error(msg, node=None) -> NoReturn: # type: ignore[no-redef] # noqa - raise_database_error(msg, node) +def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: + raise DependencyNotFound(node, node_description, required_pkg) -def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: # type: ignore[no-redef] # noqa - raise_dep_not_found(node, node_description, required_pkg) +def raise_dependency_error(msg) -> NoReturn: + raise DependencyException(scrub_secrets(msg, env_secrets())) -def raise_dependency_error(msg) -> NoReturn: # type: ignore[no-redef] # noqa - raise_dependency_error(msg) +def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: + raise DuplicatePatchPath(patch_1, existing_patch_path) -def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: # type: ignore[no-redef] # noqa - raise_duplicate_patch_name(patch_1, existing_patch_path) +def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: + raise DuplicateResourceName(node_1, node_2) -def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: # type: ignore[no-redef] # noqa - raise_duplicate_resource_name(node_1, node_2) +def raise_invalid_property_yml_version(path, issue) -> NoReturn: + raise InvalidPropertyYML(path, issue) -def raise_invalid_property_yml_version(path, issue) -> NoReturn: # type: ignore[no-redef] # noqa - raise_invalid_property_yml_version(path, issue) +# TODO: this should be improved to not format message here +def raise_not_implemented(msg) -> NoReturn: + raise NotImplementedException("ERROR: {}".format(msg)) -def raise_not_implemented(msg) -> NoReturn: # type: ignore[no-redef] # noqa - raise_not_implemented(msg) +def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: + raise RelationWrongType(relation, expected_type, model) -def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: # type: ignore[no-redef] # noqa - relation_wrong_type(relation, expected_type, model) +# these were implemented in core so deprecating here by calling the new exception directly +def raise_duplicate_alias( + kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str +) -> NoReturn: + raise DuplicateAlias(kwargs, aliases, canonical_key) + + +def raise_duplicate_source_patch_name(patch_1, patch_2): + raise DuplicateSourcePatchName(patch_1, patch_2) + + +def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): + raise DuplicateMacroPatchName(patch_1, existing_patch_path) + + +def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: + raise DuplicateMacroName(node_1, node_2, namespace) + + +def approximate_relation_match(target, relation): + raise ApproximateMatch(target, relation) + + +# These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework +# TODO: is this valid? Should I create a special exception class for this? +def raise_unrecognized_credentials_type(typename, supported_types): + raise_compiler_error( + 'Unrecognized credentials type "{}" - supported types are ({})'.format( + typename, ", ".join('"{}"'.format(t) for t in supported_types) + ) + ) + + +def raise_patch_targets_not_found(patches): + patch_list = "\n\t".join( + "model {} (referenced in path {})".format(p.name, p.original_file_path) + for p in patches.values() + ) + raise_compiler_error( + "dbt could not find models for the following patches:\n\t{}".format(patch_list) + ) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 046c38fa559..3eb0b75f89f 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -51,7 +51,9 @@ ) from dbt.exceptions import ( CompilationException, + DuplicateMacroPatchName, DuplicatePatchPath, + DuplicateSourcePatchName, JSONValidationException, InternalException, ParsingException, @@ -60,8 +62,6 @@ PropertyYMLVersionNotInt, ValidationException, validator_error_message, - raise_duplicate_macro_patch_name, - raise_duplicate_source_patch_name, ) from dbt.events.functions import warn_or_error from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroPatchNotFound @@ -696,7 +696,7 @@ def parse(self) -> List[TestBlock]: # source patches must be unique key = (patch.overrides, patch.name) if key in self.manifest.source_patches: - raise_duplicate_source_patch_name(patch, self.manifest.source_patches[key]) + raise DuplicateSourcePatchName(patch, self.manifest.source_patches[key]) self.manifest.source_patches[key] = patch source_file.source_patches.append(key) else: @@ -980,7 +980,7 @@ def parse_patch(self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef) return if macro.patch_path: package_name, existing_file_path = macro.patch_path.split("://") - raise_duplicate_macro_patch_name(patch, existing_file_path) + raise DuplicateMacroPatchName(patch, existing_file_path) source_file.macro_patches[patch.name] = unique_id macro.patch(patch) diff --git a/core/dbt/utils.py b/core/dbt/utils.py index b7cc6475319..987371b6b02 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -15,7 +15,7 @@ from pathlib import PosixPath, WindowsPath from contextlib import contextmanager -from dbt.exceptions import ConnectionException +from dbt.exceptions import ConnectionException, DuplicateAlias from dbt.events.functions import fire_event from dbt.events.types import RetryExternalCall, RecordRetryException from dbt import flags @@ -365,7 +365,7 @@ def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]: for key, value in kwargs.items(): canonical_key = self.aliases.get(key, key) if canonical_key in result: - dbt.exceptions.raise_duplicate_alias(kwargs, self.aliases, canonical_key) + raise DuplicateAlias(kwargs, self.aliases, canonical_key) result[canonical_key] = self.translate_value(value) return result