From 3d1d93833f1bb5e4d726c4234e296997c15de786 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 30 Nov 2022 13:43:38 -0600 Subject: [PATCH 01/24] starting to move jinja exceptions --- core/dbt/context/providers.py | 3 +- core/dbt/exceptions.py | 37 ++++++------ core/dbt/jinja_exceptions.py | 110 ++++++++++++++++++++++++++++++++++ core/dbt/task/run.py | 2 +- core/dbt/task/test.py | 3 +- 5 files changed, 132 insertions(+), 23 deletions(-) create mode 100644 core/dbt/jinja_exceptions.py diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 06642810730..53ff901f7e8 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -46,16 +46,15 @@ ValidationException, RuntimeException, macro_invalid_dispatch_arg, - missing_config, raise_compiler_error, ref_invalid_args, metric_invalid_args, target_not_found, ref_bad_context, - wrapped_exports, raise_parsing_error, disallow_secret_env_var, ) +from dbt.jinja_exceptions import missing_config, wrapped_exports from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 32aa8b477a9..40881d2d3a0 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,5 +1,4 @@ import builtins -import functools from typing import NoReturn, Optional, Mapping, Any from dbt.events.helpers import env_secrets, scrub_secrets @@ -661,20 +660,20 @@ def materialization_not_available(model, adapter_type): ) -def missing_materialization(model, adapter_type): - materialization = model.get_materialization() +# def missing_materialization(model, adapter_type): +# materialization = model.get_materialization() - valid_types = "'default'" +# valid_types = "'default'" - if adapter_type != "default": - valid_types = "'default' and '{}'".format(adapter_type) +# if adapter_type != "default": +# valid_types = "'default' and '{}'".format(adapter_type) - raise_compiler_error( - "No materialization '{}' was found for adapter {}! (searched types {})".format( - materialization, adapter_type, valid_types - ), - model, - ) +# raise_compiler_error( +# "No materialization '{}' was found for adapter {}! (searched types {})".format( +# materialization, adapter_type, valid_types +# ), +# model, +# ) def bad_package_spec(repo, spec, error_message): @@ -686,13 +685,13 @@ def raise_cache_inconsistent(message): raise InternalException("Cache inconsistency detected: {}".format(message)) -def missing_config(model, name): - raise_compiler_error( - "Model '{}' does not define a required config parameter '{}'.".format( - model.unique_id, name - ), - model, - ) +# def missing_config(model, name): +# raise_compiler_error( +# "Model '{}' does not define a required config parameter '{}'.".format( +# model.unique_id, name +# ), +# model, +# ) def missing_relation(relation, model=None): diff --git a/core/dbt/jinja_exceptions.py b/core/dbt/jinja_exceptions.py new file mode 100644 index 00000000000..ce3a4ebde46 --- /dev/null +++ b/core/dbt/jinja_exceptions.py @@ -0,0 +1,110 @@ +import functools + +from dbt.events.functions import warn_or_error +from dbt.events.types import JinjaLogWarning +from dbt.exceptions import ( + RuntimeException, + CompilationException, + missing_relation, + raise_ambiguous_alias, + raise_ambiguous_catalog_match, + raise_cache_inconsistent, + raise_dataclass_not_dict, + raise_compiler_error, + raise_database_error, + raise_dep_not_found, + raise_dependency_error, + raise_duplicate_patch_name, + raise_duplicate_resource_name, + raise_invalid_property_yml_version, + raise_not_implemented, + relation_wrong_type, +) + + +def warn(msg, node=None): + warn_or_error(JinjaLogWarning(msg=msg), node=node) + return "" + + +class MissingConfigException(CompilationException): + def __init__(self, unique_id, name): + self.unique_id = unique_id + self.name = name + msg = ( + f"Model '{self.unique_id}' does not define a required config parameter '{self.name}'." + ) + super().__init__(msg) + + +def missing_config(model, name): + raise MissingConfigException(unique_id=model.unique_id, name=name) + + +class MissingMaterializationException(CompilationException): + def __init__(self, model, adapter_type): + self.model = model + self.adapter_type = adapter_type + super().__init__(self.get_message()) + + def get_message(self) -> str: + materialization = self.model.get_materialization() + + valid_types = "'default'" + + if self.adapter_type != "default": + valid_types = f"'default' and '{self.adapter_type}'" + + msg = f"No materialization '{materialization}' was found for adapter {self.adapter_type}! (searched types {valid_types})" + return msg + + +def missing_materialization(model, adapter_type): + raise MissingConfigException(model=model, adapter_type=adapter_type) + + +# Update this when a new function should be added to the +# dbt context's `exceptions` key! +CONTEXT_EXPORTS = { + fn.__name__: fn + for fn in [ + warn, + missing_config, + missing_materialization, + missing_relation, + raise_ambiguous_alias, + raise_ambiguous_catalog_match, + raise_cache_inconsistent, + raise_dataclass_not_dict, + raise_compiler_error, + raise_database_error, + raise_dep_not_found, + raise_dependency_error, + raise_duplicate_patch_name, + raise_duplicate_resource_name, + raise_invalid_property_yml_version, + raise_not_implemented, + relation_wrong_type, + ] +} + + +# wraps context based exceptions in node info +def wrapper(model): + def wrap(func): + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except RuntimeException as exc: + exc.add_node(model) + raise exc + + return inner + + return wrap + + +def wrapped_exports(model): + wrap = wrapper(model) + return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 5b88d039904..ca450101ccc 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -25,8 +25,8 @@ InternalException, RuntimeException, ValidationException, - missing_materialization, ) +from dbt.jinja_exceptions import missing_materialization from dbt.events.functions import fire_event, get_invocation_id, info from dbt.events.types import ( DatabaseErrorRunningHook, diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index e48dc94e4e4..f33e8f12648 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -21,7 +21,8 @@ LogTestResult, LogStartLine, ) -from dbt.exceptions import InternalException, invalid_bool_error, missing_materialization +from dbt.exceptions import InternalException, invalid_bool_error +from dbt.jinja_exceptions import missing_materialization from dbt.graph import ( ResourceTypeSelector, ) From 8797c5dbd6adef955bf83fdab460dbd723344865 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 30 Nov 2022 16:34:15 -0600 Subject: [PATCH 02/24] convert some exceptions --- core/dbt/adapters/base/impl.py | 15 +- core/dbt/adapters/cache.py | 26 +- core/dbt/context/providers.py | 5 +- core/dbt/contracts/graph/manifest.py | 4 +- core/dbt/contracts/relation.py | 6 +- core/dbt/deps/git.py | 7 +- core/dbt/deps/registry.py | 10 +- core/dbt/deps/resolver.py | 24 +- core/dbt/exceptions.py | 575 +++++++++++++++++---------- core/dbt/jinja_exceptions.py | 107 +++-- core/dbt/parser/manifest.py | 8 +- core/dbt/parser/schemas.py | 33 +- core/dbt/task/generate.py | 4 +- core/dbt/task/run.py | 4 +- core/dbt/task/test.py | 5 +- 15 files changed, 498 insertions(+), 335 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index bbac18cb16b..602086bc333 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -22,13 +22,14 @@ import pytz from dbt.exceptions import ( - raise_database_error, raise_compiler_error, invalid_type_error, get_relation_returned_multiple_results, InternalException, NotImplementedException, RuntimeException, + UnexpectedNull, + UnexpectedNonTimestamp, ) from dbt.adapters.protocol import ( @@ -97,18 +98,10 @@ def _utc(dt: Optional[datetime], source: BaseRelation, field_name: str) -> datet assume the datetime is already for UTC and add the timezone. """ if dt is None: - raise raise_database_error( - "Expected a non-null value when querying field '{}' of table " - " {} but received value 'null' instead".format(field_name, source) - ) + raise UnexpectedNull(field_name, source) elif not hasattr(dt, "tzinfo"): - raise raise_database_error( - "Expected a timestamp value when querying field '{}' of table " - "{} but received value of type '{}' instead".format( - field_name, source, type(dt).__name__ - ) - ) + raise UnexpectedNonTimestamp(field_name, source, dt) elif dt.tzinfo: return dt.astimezone(pytz.UTC) diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 6c60039f262..162f7045fe5 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -9,7 +9,7 @@ _make_msg_from_ref_key, _ReferenceKey, ) -import dbt.exceptions +from dbt.exceptions import CacheInconsistency from dbt.events.functions import fire_event, fire_event_if from dbt.events.types import ( AddLink, @@ -150,10 +150,8 @@ def rename_key(self, old_key, new_key): :raises InternalError: If the new key already exists. """ if new_key in self.referenced_by: - dbt.exceptions.raise_cache_inconsistent( - 'in rename of "{}" -> "{}", new name is in the cache already'.format( - old_key, new_key - ) + raise CacheInconsistency( + f'in rename of "{old_key}" -> "{new_key}", new name is in the cache already' ) if old_key not in self.referenced_by: @@ -270,14 +268,14 @@ def _add_link(self, referenced_key, dependent_key): if referenced is None: return if referenced is None: - dbt.exceptions.raise_cache_inconsistent( - "in add_link, referenced link key {} not in cache!".format(referenced_key) + raise CacheInconsistency( + f"in add_link, referenced link key {referenced_key} not in cache!" ) dependent = self.relations.get(dependent_key) if dependent is None: - dbt.exceptions.raise_cache_inconsistent( - "in add_link, dependent link key {} not in cache!".format(dependent_key) + raise CacheInconsistency( + f"in add_link, dependent link key {dependent_key} not in cache!" ) assert dependent is not None # we just raised! @@ -443,10 +441,8 @@ def _check_rename_constraints(self, old_key, new_key): else: message_addendum = "" - dbt.exceptions.raise_cache_inconsistent( - "in rename, new key {} already in cache: {}{}".format( - new_key, list(self.relations.keys()), message_addendum - ) + raise CacheInconsistency( + f"in rename, new key {new_key} already in cache: {list(self.relations.keys())}{message_addendum}" ) if old_key not in self.relations: @@ -505,9 +501,7 @@ def get_relations(self, database: Optional[str], schema: Optional[str]) -> List[ ] if None in results: - dbt.exceptions.raise_cache_inconsistent( - "in get_relations, a None relation was found in the cache!" - ) + raise CacheInconsistency("in get_relations, a None relation was found in the cache!") return results def clear(self): diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 53ff901f7e8..117b38fc5c0 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -45,6 +45,7 @@ InternalException, ValidationException, RuntimeException, + MissingConfig, macro_invalid_dispatch_arg, raise_compiler_error, ref_invalid_args, @@ -54,7 +55,7 @@ raise_parsing_error, disallow_secret_env_var, ) -from dbt.jinja_exceptions import missing_config, wrapped_exports +from dbt.jinja_exceptions import wrapped_exports from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage @@ -377,7 +378,7 @@ def _lookup(self, name, default=_MISSING): else: result = self.model.config.get(name, default) if result is _MISSING: - missing_config(self.model, name) + raise MissingConfig(unique_id=self.model.unique_id, name=name) return result def require(self, name, validator=None): diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index cd1eb561fcc..95acf8438ec 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -41,7 +41,7 @@ from dbt.dataclass_schema import dbtClassMixin from dbt.exceptions import ( CompilationException, - raise_duplicate_resource_name, + DuplicateResourceName, raise_compiler_error, ) from dbt.helper_types import PathSet @@ -1237,7 +1237,7 @@ def __post_serialize__(self, dct): def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]): if value.unique_id in src: - raise_duplicate_resource_name(value, src[value.unique_id]) + raise DuplicateResourceName(value, src[value.unique_id]) K_T = TypeVar("K_T") diff --git a/core/dbt/contracts/relation.py b/core/dbt/contracts/relation.py index fbe18146bb4..e8cba2ad155 100644 --- a/core/dbt/contracts/relation.py +++ b/core/dbt/contracts/relation.py @@ -9,7 +9,7 @@ from dbt.dataclass_schema import dbtClassMixin, StrEnum from dbt.contracts.util import Replaceable -from dbt.exceptions import raise_dataclass_not_dict, CompilationException +from dbt.exceptions import CompilationException, DataclassNotDict from dbt.utils import deep_merge @@ -43,10 +43,10 @@ def __getitem__(self, key): raise KeyError(key) from None def __iter__(self): - raise_dataclass_not_dict(self) + raise DataclassNotDict(self) def __len__(self): - raise_dataclass_not_dict(self) + raise DataclassNotDict(self) def incorporate(self, **kwargs): value = self.to_dict(omit_none=True) diff --git a/core/dbt/deps/git.py b/core/dbt/deps/git.py index e6dcc479a80..5d7a1331c58 100644 --- a/core/dbt/deps/git.py +++ b/core/dbt/deps/git.py @@ -9,7 +9,7 @@ GitPackage, ) from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path -from dbt.exceptions import ExecutableError, raise_dependency_error +from dbt.exceptions import ExecutableError, MultipleVersionGitDeps from dbt.events.functions import fire_event, warn_or_error from dbt.events.types import EnsureGitInstalled, DepsUnpinned @@ -143,10 +143,7 @@ def resolved(self) -> GitPinnedPackage: if len(requested) == 0: requested = {"HEAD"} elif len(requested) > 1: - raise_dependency_error( - "git dependencies should contain exactly one version. " - "{} contains: {}".format(self.git, requested) - ) + raise MultipleVersionGitDeps(self.git, requested) return GitPinnedPackage( git=self.git, diff --git a/core/dbt/deps/registry.py b/core/dbt/deps/registry.py index 9f163d89758..f3398f4b16f 100644 --- a/core/dbt/deps/registry.py +++ b/core/dbt/deps/registry.py @@ -10,10 +10,10 @@ ) from dbt.deps.base import PinnedPackage, UnpinnedPackage from dbt.exceptions import ( - package_version_not_found, - VersionsNotCompatibleException, DependencyException, - package_not_found, + PackageNotFound, + PackageVersionNotFound, + VersionsNotCompatibleException, ) @@ -71,7 +71,7 @@ def __init__( def _check_in_index(self): index = registry.index_cached() if self.package not in index: - package_not_found(self.package) + raise PackageNotFound(self.package) @classmethod def from_contract(cls, contract: RegistryPackage) -> "RegistryUnpinnedPackage": @@ -118,7 +118,7 @@ def resolved(self) -> RegistryPinnedPackage: target = None if not target: # raise an exception if no installable target version is found - package_version_not_found(self.package, range_, installable, should_version_check) + raise PackageVersionNotFound(self.package, range_, installable, should_version_check) latest_compatible = installable[-1] return RegistryPinnedPackage( package=self.package, version=target, version_latest=latest_compatible diff --git a/core/dbt/deps/resolver.py b/core/dbt/deps/resolver.py index e4c1992894c..323e2f562c1 100644 --- a/core/dbt/deps/resolver.py +++ b/core/dbt/deps/resolver.py @@ -1,7 +1,12 @@ from dataclasses import dataclass, field from typing import Dict, List, NoReturn, Union, Type, Iterator, Set -from dbt.exceptions import raise_dependency_error, InternalException +from dbt.exceptions import ( + DuplicateDependencyToRoot, + DuplicateProjectDependency, + MismatchedDependencyTypes, + InternalException, +) from dbt.config import Project, RuntimeConfig from dbt.config.renderer import DbtProjectYamlRenderer @@ -51,10 +56,7 @@ def __setitem__(self, key: BasePackage, value): self.packages[key_str] = value def _mismatched_types(self, old: UnpinnedPackage, new: UnpinnedPackage) -> NoReturn: - raise_dependency_error( - f"Cannot incorporate {new} ({new.__class__.__name__}) in {old} " - f"({old.__class__.__name__}): mismatched types" - ) + raise MismatchedDependencyTypes(new, old) def incorporate(self, package: UnpinnedPackage): key: str = self._pick_key(package) @@ -105,17 +107,9 @@ def _check_for_duplicate_project_names( for package in final_deps: project_name = package.get_project_name(config, renderer) if project_name in seen: - raise_dependency_error( - f'Found duplicate project "{project_name}". This occurs when ' - "a dependency has the same project name as some other " - "dependency." - ) + raise DuplicateProjectDependency(project_name) elif project_name == config.project_name: - raise_dependency_error( - "Found a dependency with the same name as the root project " - f'"{project_name}". Package names must be unique in a project.' - " Please rename one of these packages." - ) + raise DuplicateDependencyToRoot(project_name) seen.add(project_name) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 40881d2d3a0..4de6becd9b5 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -439,6 +439,7 @@ class DuplicateYamlKeyException(CompilationException): pass +# TODO: this was copied into jinja_exxceptions because it's in the context - eventually remove? def raise_compiler_error(msg, node=None) -> NoReturn: raise CompilationException(msg, node) @@ -447,14 +448,6 @@ def raise_parsing_error(msg, node=None) -> NoReturn: raise ParsingException(msg, node) -def raise_database_error(msg, node=None) -> NoReturn: - raise DatabaseException(msg, node) - - -def raise_dependency_error(msg) -> NoReturn: - raise DependencyException(scrub_secrets(msg, env_secrets())) - - def raise_git_cloning_error(error: CommandResultError) -> NoReturn: error.cmd = scrub_secrets(str(error.cmd), env_secrets()) raise error @@ -660,89 +653,11 @@ def materialization_not_available(model, adapter_type): ) -# def missing_materialization(model, adapter_type): -# materialization = model.get_materialization() - -# valid_types = "'default'" - -# if adapter_type != "default": -# valid_types = "'default' and '{}'".format(adapter_type) - -# raise_compiler_error( -# "No materialization '{}' was found for adapter {}! (searched types {})".format( -# materialization, adapter_type, valid_types -# ), -# model, -# ) - - def bad_package_spec(repo, spec, error_message): msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message) raise InternalException(scrub_secrets(msg, env_secrets())) -def raise_cache_inconsistent(message): - raise InternalException("Cache inconsistency detected: {}".format(message)) - - -# def missing_config(model, name): -# raise_compiler_error( -# "Model '{}' does not define a required config parameter '{}'.".format( -# model.unique_id, name -# ), -# model, -# ) - - -def missing_relation(relation, model=None): - raise_compiler_error("Relation {} not found!".format(relation), model) - - -def raise_dataclass_not_dict(obj): - msg = ( - 'The object ("{obj}") was used as a dictionary. This ' - "capability has been removed from objects of this type." - ) - raise_compiler_error(msg) - - -def relation_wrong_type(relation, expected_type, model=None): - raise_compiler_error( - ( - "Trying to create {expected_type} {relation}, " - "but it currently exists as a {current_type}. Either " - "drop {relation} manually, or run dbt with " - "`--full-refresh` and dbt will drop it for you." - ).format(relation=relation, current_type=relation.type, expected_type=expected_type), - model, - ) - - -def package_not_found(package_name): - raise_dependency_error("Package {} was not found in the package index".format(package_name)) - - -def package_version_not_found( - package_name, version_range, available_versions, should_version_check -): - base_msg = ( - "Could not find a matching compatible version for package {}\n" - " Requested range: {}\n" - " Compatible versions: {}\n" - ) - addendum = ( - ( - "\n" - " Not shown: package versions incompatible with installed version of dbt-core\n" - " To include them, run 'dbt --no-version-check deps'" - ) - if should_version_check - else "" - ) - msg = base_msg.format(package_name, version_range, available_versions) + addendum - raise_dependency_error(msg) - - def invalid_materialization_argument(name, argument): raise_compiler_error( "materialization '{}' received unknown argument '{}'.".format(name, argument) @@ -766,15 +681,6 @@ class ConnectionException(Exception): pass -def raise_dep_not_found(node, node_description, required_pkg): - raise_compiler_error( - 'Error while parsing {}.\nThe required package "{}" was not found. ' - "Is the package installed?\nHint: You may need to run " - "`dbt deps`.".format(node_description, required_pkg), - node=node, - ) - - def multiple_matching_relations(kwargs, matches): raise_compiler_error( "get_relation returned more than one relation with the given args. " @@ -821,90 +727,6 @@ def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: ) -def raise_duplicate_resource_name(node_1, node_2): - duped_name = node_1.name - node_type = NodeType(node_1.resource_type) - pluralized = ( - node_type.pluralize() - if node_1.resource_type == node_2.resource_type - else "resources" # still raise if ref() collision, e.g. model + seed - ) - - action = "looking for" - # duplicate 'ref' targets - if node_type in NodeType.refable(): - formatted_name = f'ref("{duped_name}")' - # duplicate sources - elif node_type == NodeType.Source: - duped_name = node_1.get_full_source_name() - formatted_name = node_1.get_source_representation() - # duplicate docs blocks - elif node_type == NodeType.Documentation: - formatted_name = f'doc("{duped_name}")' - # duplicate generic tests - elif node_type == NodeType.Test and hasattr(node_1, "test_metadata"): - column_name = f'column "{node_1.column_name}" in ' if node_1.column_name else "" - model_name = node_1.file_key_name - duped_name = f'{node_1.name}" defined on {column_name}"{model_name}' - action = "running" - formatted_name = "tests" - # all other resource types - else: - formatted_name = duped_name - - # should this be raise_parsing_error instead? - raise_compiler_error( - f""" -dbt found two {pluralized} with the name "{duped_name}". - -Since these resources have the same name, dbt will be unable to find the correct resource -when {action} {formatted_name}. - -To fix this, change the name of one of these resources: -- {node_1.unique_id} ({node_1.original_file_path}) -- {node_2.unique_id} ({node_2.original_file_path}) - """.strip() - ) - - -def raise_ambiguous_alias(node_1, node_2, duped_name=None): - if duped_name is None: - duped_name = f"{node_1.database}.{node_1.schema}.{node_1.alias}" - - raise_compiler_error( - 'dbt found two resources with the database representation "{}".\ndbt ' - "cannot create two resources with identical database representations. " - "To fix this,\nchange the configuration of one of these resources:" - "\n- {} ({})\n- {} ({})".format( - duped_name, - node_1.unique_id, - node_1.original_file_path, - node_2.unique_id, - node_2.original_file_path, - ) - ) - - -def raise_ambiguous_catalog_match(unique_id, match_1, match_2): - def get_match_string(match): - return "{}.{}".format( - match.get("metadata", {}).get("schema"), - match.get("metadata", {}).get("name"), - ) - - raise_compiler_error( - "dbt found two relations in your warehouse with similar database " - "identifiers. dbt\nis unable to determine which of these relations " - 'was created by the model "{unique_id}".\nIn order for dbt to ' - "correctly generate the catalog, one of the following relations must " - "be deleted or renamed:\n\n - {match_1_s}\n - {match_2_s}".format( - unique_id=unique_id, - match_1_s=get_match_string(match_1), - match_2_s=get_match_string(match_2), - ) - ) - - def raise_patch_targets_not_found(patches): patch_list = "\n\t".join( "model {} (referenced in path {})".format(p.name, p.original_file_path) @@ -925,21 +747,6 @@ def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: ) -def raise_duplicate_patch_name(patch_1, existing_patch_path): - name = patch_1.name - fix = _fix_dupe_msg( - patch_1.original_file_path, - existing_patch_path, - name, - "resource", - ) - raise_compiler_error( - f"dbt found two schema.yml entries for the same resource named " - f"{name}. Resources and their associated columns may only be " - f"described a single time. To fix this, {fix}" - ) - - def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): package_name = patch_1.package_name name = patch_1.name @@ -966,14 +773,6 @@ def raise_duplicate_source_patch_name(patch_1, patch_2): ) -def raise_invalid_property_yml_version(path, issue): - raise_compiler_error( - "The yml property file at {} is invalid because {}. Please consult the " - "documentation for more information on yml property file syntax:\n\n" - "https://docs.getdbt.com/reference/configs-and-properties".format(path, issue) - ) - - def raise_unrecognized_credentials_type(typename, supported_types): raise_compiler_error( 'Unrecognized credentials type "{}" - supported types are ({})'.format( @@ -982,10 +781,6 @@ def raise_unrecognized_credentials_type(typename, supported_types): ) -def raise_not_implemented(msg): - raise NotImplementedException("ERROR: {}".format(msg)) - - def raise_duplicate_alias( kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str ) -> NoReturn: @@ -1047,3 +842,371 @@ def inner(*args, **kwargs): def wrapped_exports(model): wrap = wrapper(model) return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} +# adapters exceptions +class UnexpectedNull(DatabaseException): + def __init__(self, field_name, source): + self.field_name = field_name + self.source = source + msg = ( + f"Expected a non-null value when querying field '{self.field_name}' of table " + f" {self.source} but received value 'null' instead" + ) + super().__init__(msg) + + +class UnexpectedNonTimestamp(DatabaseException): + def __init__(self, field_name, source, dt): + self.field_name = field_name + self.source = source + self.type_name = type(dt).__name__ + msg = ( + f"Expected a timestamp value when querying field '{self.field_name}' of table " + f"{self.source} but received value of type '{self.type_name}' instead" + ) + super().__init__(msg) + + +# start new exceptions + + +# deps exceptions +class MultipleVersionGitDeps(DependencyException): + def __init__(self, git, requested): + self.git = git + self.requested = requested + msg = ( + "git dependencies should contain exactly one version. " + f"{self.git} contains: {self.requested}" + ) + super().__init__(msg) + + +class DuplicateProjectDependency(DependencyException): + def __init__(self, project_name): + self.project_name = project_name + msg = ( + f'Found duplicate project "{self.project_name}". This occurs when ' + "a dependency has the same project name as some other dependency." + ) + super().__init__(msg) + + +class DuplicateDependencyToRoot(DependencyException): + def __init__(self, project_name): + self.project_name = project_name + msg = ( + "Found a dependency with the same name as the root project " + f'"{self.project_name}". Package names must be unique in a project.' + " Please rename one of these packages." + ) + super().__init__(msg) + + +class MismatchedDependencyTypes(DependencyException): + def __init__(self, new, old): + self.new = new + self.old = old + msg = ( + f"Cannot incorporate {self.new} ({self.new.__class__.__name__}) in {self.old} " + f"({self.old.__class__.__name__}): mismatched types" + ) + super().__init__(msg) + + +class PackageVersionNotFound(DependencyException): + def __init__(self, package_name, version_range, available_versions, should_version_check): + self.package_name = package_name + self.version_range = version_range + self.available_versions = available_versions + self.should_version_check = should_version_check + super().__init__(self.get_message()) + + def get_message(self) -> str: + base_msg = ( + "Could not find a matching compatible version for package {}\n" + " Requested range: {}\n" + " Compatible versions: {}\n" + ) + addendum = ( + ( + "\n" + " Not shown: package versions incompatible with installed version of dbt-core\n" + " To include them, run 'dbt --no-version-check deps'" + ) + if self.should_version_check + else "" + ) + msg = ( + base_msg.format(self.package_name, self.version_range, self.available_versions) + + addendum + ) + return msg + + +class PackageNotFound(DependencyException): + def __init__(self, package_name): + self.package_name = package_name + msg = f"Package {self.package_name} was not found in the package index" + super().__init__(msg) + + +# jinja exceptions +class MissingConfig(CompilationException): + def __init__(self, unique_id, name): + self.unique_id = unique_id + self.name = name + msg = ( + f"Model '{self.unique_id}' does not define a required config parameter '{self.name}'." + ) + super().__init__(msg) + + +class MissingMaterialization(CompilationException): + def __init__(self, model, adapter_type): + self.model = model + self.adapter_type = adapter_type + super().__init__(self.get_message()) + + def get_message(self) -> str: + materialization = self.model.get_materialization() + + valid_types = "'default'" + + if self.adapter_type != "default": + valid_types = f"'default' and '{self.adapter_type}'" + + msg = f"No materialization '{materialization}' was found for adapter {self.adapter_type}! (searched types {valid_types})" + return msg + + +class MissingRelation(CompilationException): + def __init__(self, relation, model=None): + self.relation = relation + self.model = model + msg = f"Relation {self.relation} not found!" + super().__init__(msg) + + +class AmbiguousAlias(CompilationException): + def __init__(self, node_1, node_2, duped_name=None): + self.node_1 = node_1 + self.node_2 = node_2 + if duped_name is None: + self.duped_name = f"{self.node_1.database}.{self.node_1.schema}.{self.node_1.alias}" + else: + self.duped_name = duped_name + super().__init__(self.get_message()) + + def get_message(self) -> str: + + msg = ( + 'dbt found two resources with the database representation "{}".\ndbt ' + "cannot create two resources with identical database representations. " + "To fix this,\nchange the configuration of one of these resources:" + "\n- {} ({})\n- {} ({})".format( + self.duped_name, + self.node_1.unique_id, + self.node_1.original_file_path, + self.node_2.unique_id, + self.node_2.original_file_path, + ) + ) + return msg + + +class AmbiguousCatalogMatch(CompilationException): + def __init__(self, unique_id, match_1, match_2): + self.unique_id = unique_id + self.match_1 = match_1 + self.match_2 = match_2 + super().__init__(self.get_message()) + + def get_match_string(self, match): + return "{}.{}".format( + match.get("metadata", {}).get("schema"), + match.get("metadata", {}).get("name"), + ) + + def get_message(self) -> str: + msg = ( + "dbt found two relations in your warehouse with similar database identifiers. " + "dbt\nis unable to determine which of these relations was created by the model " + f'"{self.unique_id}".\nIn order for dbt to correctly generate the catalog, one ' + "of the following relations must be deleted or renamed:\n\n - " + f"{self.get_match_string(self.match_1)}\n - {self.get_match_string(self.match_2)}" + ) + + return msg + + +class CacheInconsistency(InternalException): + def __init__(self, message): + self.message = message + msg = f"Cache inconsistency detected: {self.message}" + super().__init__(msg) + + +# this is part of the context and also raised in dbt.contratcs.relation.py +class DataclassNotDict(CompilationException): + def __init__(self, obj): + self.obj = obj # TODO: what kind of obj is this? + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + f'The object ("{self.obj}") was used as a dictionary. This ' + "capability has been removed from objects of this type." + ) + + return msg + + +class DependencyNotFound(CompilationException): + def __init__(self, node, node_description, required_pkg): + self.node = node + self.node_description = node_description + self.required_pkg = required_pkg + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Error while parsing {self.node_description}.\nThe required package " + f'"{self.required_pkg}" was not found. Is the package installed?\n' + "Hint: You may need to run `dbt deps`." + ) + + return msg + + +class DuplicatePatchPath(CompilationException): + def __init__(self, patch_1, existing_patch_path): + self.patch_1 = patch_1 + self.existing_patch_path = existing_patch_path + super().__init__(self.get_message()) + + def get_message(self) -> str: + name = self.patch_1.name + fix = _fix_dupe_msg( + self.patch_1.original_file_path, + self.existing_patch_path, + name, + "resource", + ) + msg = ( + f"dbt found two schema.yml entries for the same resource named " + f"{name}. Resources and their associated columns may only be " + f"described a single time. To fix this, {fix}" + ) + return msg + + +# should this inherit ParsingException instead? +class DuplicateResourceName(CompilationException): + def __init__(self, node_1, node_2): + self.node_1 = node_1 + self.node_2 = node_2 + super().__init__(self.get_message()) + + def get_message(self) -> str: + duped_name = self.node_1.name + node_type = NodeType(self.node_1.resource_type) + pluralized = ( + node_type.pluralize() + if self.node_1.resource_type == self.node_2.resource_type + else "resources" # still raise if ref() collision, e.g. model + seed + ) + + action = "looking for" + # duplicate 'ref' targets + if node_type in NodeType.refable(): + formatted_name = f'ref("{duped_name}")' + # duplicate sources + elif node_type == NodeType.Source: + duped_name = self.node_1.get_full_source_name() + formatted_name = self.node_1.get_source_representation() + # duplicate docs blocks + elif node_type == NodeType.Documentation: + formatted_name = f'doc("{duped_name}")' + # duplicate generic tests + elif node_type == NodeType.Test and hasattr(self.node_1, "test_metadata"): + column_name = ( + f'column "{self.node_1.column_name}" in ' if self.node_1.column_name else "" + ) + model_name = self.node_1.file_key_name + duped_name = f'{self.node_1.name}" defined on {column_name}"{model_name}' + action = "running" + formatted_name = "tests" + # all other resource types + else: + formatted_name = duped_name + + msg = f""" +dbt found two {pluralized} with the name "{duped_name}". + +Since these resources have the same name, dbt will be unable to find the correct resource +when {action} {formatted_name}. + +To fix this, change the name of one of these resources: +- {self.node_1.unique_id} ({self.node_1.original_file_path}) +- {self.node_2.unique_id} ({self.node_2.original_file_path}) + """.strip() + return msg + + +class InvalidPropertyYML(CompilationException): + def __init__(self, path, issue): + self.path = path + self.issue = issue + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + f"The yml property file at {self.path} is invalid because {self.issue}. " + "Please consult the documentation for more information on yml property file " + "syntax:\n\nhttps://docs.getdbt.com/reference/configs-and-properties" + ) + return msg + + +class PropertyYMLMissingVersion(InvalidPropertyYML): + def __init__(self, path): + self.path = path + self.issue = f"the yml property file {self.path} is missing a version tag" + super().__init__() + + +class PropertyYMLVersionNotInt(InvalidPropertyYML): + def __init__(self, path, version): + self.path = path + self.version = version + self.issue = ( + "its 'version:' tag must be an integer (e.g. version: 2)." + f" {self.version} is not an integer" + ) + super().__init__() + + +class PropertyYMLInvalidTag(InvalidPropertyYML): + def __init__(self, path, version): + self.path = path + self.version = version + self.issue = f"its 'version:' tag is set to {self.version}. Only 2 is supported" + super().__init__() + + +class RelationWrongType(CompilationException): + def __init__(self, relation, expected_type, model=None): + self.relation = relation + self.expected_type = expected_type + self.model = model + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Trying to create {self.expected_type} {self.relation}, " + f"but it currently exists as a {self.relation.type}. Either " + f"drop {self.relation} manually, or run dbt with " + "`--full-refresh` and dbt will drop it for you." + ) + + return msg diff --git a/core/dbt/jinja_exceptions.py b/core/dbt/jinja_exceptions.py index ce3a4ebde46..5318b040719 100644 --- a/core/dbt/jinja_exceptions.py +++ b/core/dbt/jinja_exceptions.py @@ -1,24 +1,27 @@ import functools +from typing import NoReturn from dbt.events.functions import warn_or_error +from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning from dbt.exceptions import ( RuntimeException, + MissingConfig, + MissingMaterialization, + MissingRelation, + AmbiguousAlias, + AmbiguousCatalogMatch, + InternalException, + DataclassNotDict, CompilationException, - missing_relation, - raise_ambiguous_alias, - raise_ambiguous_catalog_match, - raise_cache_inconsistent, - raise_dataclass_not_dict, - raise_compiler_error, - raise_database_error, - raise_dep_not_found, - raise_dependency_error, - raise_duplicate_patch_name, - raise_duplicate_resource_name, - raise_invalid_property_yml_version, - raise_not_implemented, - relation_wrong_type, + DatabaseException, + DependencyNotFound, + DependencyException, + DuplicatePatchPath, + DuplicateResourceName, + InvalidPropertyYML, + NotImplementedException, # TODO: this should be improved to not pass message + RelationWrongType, ) @@ -27,40 +30,68 @@ def warn(msg, node=None): return "" -class MissingConfigException(CompilationException): - def __init__(self, unique_id, name): - self.unique_id = unique_id - self.name = name - msg = ( - f"Model '{self.unique_id}' does not define a required config parameter '{self.name}'." - ) - super().__init__(msg) +def missing_config(model, name) -> NoReturn: + raise MissingConfig(unique_id=model.unique_id, name=name) -def missing_config(model, name): - raise MissingConfigException(unique_id=model.unique_id, name=name) +def missing_materialization(model, adapter_type) -> NoReturn: + raise MissingMaterialization(model=model, adapter_type=adapter_type) -class MissingMaterializationException(CompilationException): - def __init__(self, model, adapter_type): - self.model = model - self.adapter_type = adapter_type - super().__init__(self.get_message()) +def missing_relation(relation, model=None) -> NoReturn: + raise MissingRelation(relation, model) - def get_message(self) -> str: - materialization = self.model.get_materialization() - valid_types = "'default'" +def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: + raise AmbiguousAlias(node_1, node_2, duped_name) - if self.adapter_type != "default": - valid_types = f"'default' and '{self.adapter_type}'" - msg = f"No materialization '{materialization}' was found for adapter {self.adapter_type}! (searched types {valid_types})" - return msg +def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: + raise AmbiguousCatalogMatch(unique_id, match_1, match_2) -def missing_materialization(model, adapter_type): - raise MissingConfigException(model=model, adapter_type=adapter_type) +def raise_cache_inconsistent(message) -> NoReturn: + raise InternalException("Cache inconsistency detected: {}".format(message)) + + +def raise_dataclass_not_dict(obj) -> NoReturn: + raise DataclassNotDict(obj) + + +def raise_compiler_error(msg, node=None) -> NoReturn: + raise CompilationException(msg, node) + + +def raise_database_error(msg, node=None) -> NoReturn: + raise DatabaseException(msg, node) + + +def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: + raise DependencyNotFound(node, node_description, required_pkg) + + +def raise_dependency_error(msg) -> NoReturn: + raise DependencyException(scrub_secrets(msg, env_secrets())) + + +def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: + raise DuplicatePatchPath(patch_1, existing_patch_path) + + +def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: + raise DuplicateResourceName(node_1, node_2) + + +def raise_invalid_property_yml_version(path, issue) -> NoReturn: + raise InvalidPropertyYML(path, issue) + + +def raise_not_implemented(msg): + raise NotImplementedException("ERROR: {}".format(msg)) + + +def relation_wrong_type(relation, expected_type, model=None): + raise RelationWrongType(relation, expected_type, model) # Update this when a new function should be added to the diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 2e284b43cfa..1ed84746464 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -71,9 +71,7 @@ ResultNode, ) from dbt.contracts.util import Writable -from dbt.exceptions import ( - target_not_found, -) +from dbt.exceptions import target_not_found, AmbiguousAlias from dbt.parser.base import Parser from dbt.parser.analysis import AnalysisParser from dbt.parser.generic_test import GenericTestParser @@ -1017,11 +1015,11 @@ def _check_resource_uniqueness( existing_node = names_resources.get(name) if existing_node is not None: - dbt.exceptions.raise_duplicate_resource_name(existing_node, node) + dbt.exceptions.DuplicateResourceName(existing_node, node) existing_alias = alias_resources.get(full_node_name) if existing_alias is not None: - dbt.exceptions.raise_ambiguous_alias(existing_alias, node, full_node_name) + raise AmbiguousAlias(node_1=existing_alias, node_2=node, duped_name=full_node_name) names_resources[name] = node alias_resources[full_node_name] = node diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 831647d0322..028eceffb67 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -50,16 +50,18 @@ UnparsedSourceDefinition, ) from dbt.exceptions import ( - validator_error_message, + CompilationException, + DuplicatePatchPath, JSONValidationException, - raise_invalid_property_yml_version, - ValidationException, + InternalException, ParsingException, - raise_duplicate_patch_name, + PropertyYMLInvalidTag, + PropertyYMLMissingVersion, + PropertyYMLVersionNotInt, + ValidationException, + validator_error_message, raise_duplicate_macro_patch_name, - InternalException, raise_duplicate_source_patch_name, - CompilationException, ) from dbt.events.functions import warn_or_error from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroPatchNotFound @@ -554,25 +556,16 @@ def parse_file(self, block: FileBlock, dct: Dict = None) -> None: def check_format_version(file_path, yaml_dct) -> None: if "version" not in yaml_dct: - raise_invalid_property_yml_version( - file_path, - "the yml property file {} is missing a version tag".format(file_path), - ) + raise PropertyYMLMissingVersion(file_path) version = yaml_dct["version"] # if it's not an integer, the version is malformed, or not # set. Either way, only 'version: 2' is supported. if not isinstance(version, int): - raise_invalid_property_yml_version( - file_path, - "its 'version:' tag must be an integer (e.g. version: 2)." - " {} is not an integer".format(version), - ) + raise PropertyYMLVersionNotInt(file_path, version) + if version != 2: - raise_invalid_property_yml_version( - file_path, - "its 'version:' tag is set to {}. Only 2 is supported".format(version), - ) + raise PropertyYMLInvalidTag(file_path, version) Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch) @@ -932,7 +925,7 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: if node: if node.patch_path: package_name, existing_file_path = node.patch_path.split("://") - raise_duplicate_patch_name(patch, existing_file_path) + raise DuplicatePatchPath(patch, existing_file_path) source_file.append_patch(patch.yaml_key, node.unique_id) # re-calculate the node config with the patch config. Always do this diff --git a/core/dbt/task/generate.py b/core/dbt/task/generate.py index 48db2e772ba..87723a530a1 100644 --- a/core/dbt/task/generate.py +++ b/core/dbt/task/generate.py @@ -22,7 +22,7 @@ ColumnMetadata, CatalogArtifact, ) -from dbt.exceptions import InternalException +from dbt.exceptions import InternalException, AmbiguousCatalogMatch from dbt.include.global_project import DOCS_INDEX_FILE_PATH from dbt.events.functions import fire_event from dbt.events.types import ( @@ -119,7 +119,7 @@ def make_unique_id_map( unique_ids = source_map.get(table.key(), set()) for unique_id in unique_ids: if unique_id in sources: - dbt.exceptions.raise_ambiguous_catalog_match( + raise AmbiguousCatalogMatch( unique_id, sources[unique_id].to_dict(omit_none=True), table.to_dict(omit_none=True), diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index ca450101ccc..44d325f6047 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -23,10 +23,10 @@ from dbt.exceptions import ( CompilationException, InternalException, + MissingMaterialization, RuntimeException, ValidationException, ) -from dbt.jinja_exceptions import missing_materialization from dbt.events.functions import fire_event, get_invocation_id, info from dbt.events.types import ( DatabaseErrorRunningHook, @@ -252,7 +252,7 @@ def execute(self, model, manifest): ) if materialization_macro is None: - missing_materialization(model, self.adapter.type()) + raise MissingMaterialization(model=model, adapter_type=self.adapter.type()) if "config" not in context: raise InternalException( diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index f33e8f12648..afc37e4b6df 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -21,8 +21,7 @@ LogTestResult, LogStartLine, ) -from dbt.exceptions import InternalException, invalid_bool_error -from dbt.jinja_exceptions import missing_materialization +from dbt.exceptions import InternalException, MissingMaterialization, invalid_bool_error from dbt.graph import ( ResourceTypeSelector, ) @@ -98,7 +97,7 @@ def execute_test( ) if materialization_macro is None: - missing_materialization(test, self.adapter.type()) + raise MissingMaterialization(model=test, adapter_type=self.adapter.type()) if "config" not in context: raise InternalException( From 023baef7f843b2d9641126bd80d7314d131bd212 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 1 Dec 2022 13:57:39 -0600 Subject: [PATCH 03/24] add back old functions for backward compatibility --- core/dbt/exceptions.py | 92 ++++++++++++++++++++++++++++++++++++ core/dbt/jinja_exceptions.py | 8 ++-- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 4de6becd9b5..a27bf2b779a 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -5,6 +5,25 @@ from dbt.events.types import JinjaLogWarning from dbt.events.contextvars import get_node_info from dbt.node_types import NodeType +from dbt.jinja_exceptions import ( + warn, + missing_config, + missing_materialization, + missing_relation, + raise_ambiguous_alias, + raise_ambiguous_catalog_match, + raise_cache_inconsistent, + raise_dataclass_not_dict, + # raise_compiler_error, + raise_database_error, + raise_dep_not_found, + raise_dependency_error, + raise_duplicate_patch_name, + raise_duplicate_resource_name, + raise_invalid_property_yml_version, + raise_not_implemented, + relation_wrong_type, +) import dbt.dataclass_schema @@ -1210,3 +1229,76 @@ def get_message(self) -> str: ) return msg + + +# These are placeholders to not immediately break app adapters utilizing these functions as exceptions. +# They will be removed in 1 (or 2?) versions. Issue to be created to ensure it happens. +# TODO: add deprecation to functions +def warn(msg, node=None): # type: ignore[no-redef] # noqa + return warn(msg, node) + + +def missing_config(model, name) -> NoReturn: # type: ignore[no-redef] # noqa + missing_config(model, name) + + +def missing_materialization(model, adapter_type) -> NoReturn: # type: ignore[no-redef] # noqa + missing_materialization(model, adapter_type) + + +def missing_relation(relation, model=None) -> NoReturn: # type: ignore[no-redef] # noqa + missing_relation(relation, model) + + +def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: # type: ignore[no-redef] # noqa + raise_ambiguous_alias(node_1, node_2, duped_name) + + +def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: # type: ignore[no-redef] # noqa + raise_ambiguous_catalog_match(unique_id, match_1, match_2) + + +def raise_cache_inconsistent(message) -> NoReturn: # type: ignore[no-redef] # noqa + raise_cache_inconsistent(message) + + +def raise_dataclass_not_dict(obj) -> NoReturn: # type: ignore[no-redef] # noqa + raise_dataclass_not_dict(obj) + + +# this is already used all over our code so for now can't do this until it's fully +# removed from this file. otherwise casuses recurssion errors. +# def raise_compiler_error(msg, node=None) -> NoReturn: +# raise_compiler_error(msg, node=None) + + +def raise_database_error(msg, node=None) -> NoReturn: # type: ignore[no-redef] # noqa + raise_database_error(msg, node) + + +def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: # type: ignore[no-redef] # noqa + raise_dep_not_found(node, node_description, required_pkg) + + +def raise_dependency_error(msg) -> NoReturn: # type: ignore[no-redef] # noqa + raise_dependency_error(msg) + + +def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: # type: ignore[no-redef] # noqa + raise_duplicate_patch_name(patch_1, existing_patch_path) + + +def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: # type: ignore[no-redef] # noqa + raise_duplicate_resource_name(node_1, node_2) + + +def raise_invalid_property_yml_version(path, issue) -> NoReturn: # type: ignore[no-redef] # noqa + raise_invalid_property_yml_version(path, issue) + + +def raise_not_implemented(msg) -> NoReturn: # type: ignore[no-redef] # noqa + raise_not_implemented(msg) + + +def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: # type: ignore[no-redef] # noqa + relation_wrong_type(relation, expected_type, model) diff --git a/core/dbt/jinja_exceptions.py b/core/dbt/jinja_exceptions.py index 5318b040719..67c1fbc30c0 100644 --- a/core/dbt/jinja_exceptions.py +++ b/core/dbt/jinja_exceptions.py @@ -20,7 +20,7 @@ DuplicatePatchPath, DuplicateResourceName, InvalidPropertyYML, - NotImplementedException, # TODO: this should be improved to not pass message + NotImplementedException, RelationWrongType, ) @@ -50,6 +50,7 @@ def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: raise AmbiguousCatalogMatch(unique_id, match_1, match_2) +# TODO: this should be improved to not format message here def raise_cache_inconsistent(message) -> NoReturn: raise InternalException("Cache inconsistency detected: {}".format(message)) @@ -86,11 +87,12 @@ def raise_invalid_property_yml_version(path, issue) -> NoReturn: raise InvalidPropertyYML(path, issue) -def raise_not_implemented(msg): +# TODO: this should be improved to not format message here +def raise_not_implemented(msg) -> NoReturn: raise NotImplementedException("ERROR: {}".format(msg)) -def relation_wrong_type(relation, expected_type, model=None): +def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: raise RelationWrongType(relation, expected_type, model) From 0971e3238b94d676d6776e11c0bfbc849ad53325 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 2 Dec 2022 08:28:51 -0600 Subject: [PATCH 04/24] organize --- core/dbt/exceptions.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index a27bf2b779a..21052d88c3b 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -458,6 +458,18 @@ class DuplicateYamlKeyException(CompilationException): pass +class ConnectionException(Exception): + """ + There was a problem with the connection that returned a bad response, + timed out, or resulted in a file that is corrupt. + """ + + pass + + +# TODO: these are all the functins that need to be converted and deprecated + + # TODO: this was copied into jinja_exxceptions because it's in the context - eventually remove? def raise_compiler_error(msg, node=None) -> NoReturn: raise CompilationException(msg, node) @@ -691,15 +703,6 @@ def system_error(operation_name): ) -class ConnectionException(Exception): - """ - There was a problem with the connection that returned a bad response, - timed out, or resulted in a file that is corrupt. - """ - - pass - - def multiple_matching_relations(kwargs, matches): raise_compiler_error( "get_relation returned more than one relation with the given args. " From c02fc0144577b19c46f94a592424a6b110e070a8 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 5 Dec 2022 13:31:52 -0600 Subject: [PATCH 05/24] more conversions --- core/dbt/adapters/base/relation.py | 4 +- .../exceptions_jinja.py} | 1 + core/dbt/context/macro_resolver.py | 2 +- core/dbt/context/macros.py | 4 +- core/dbt/context/providers.py | 12 +- core/dbt/exceptions.py | 366 +++++++++--------- core/dbt/parser/schemas.py | 8 +- core/dbt/utils.py | 4 +- 8 files changed, 199 insertions(+), 202 deletions(-) rename core/dbt/{jinja_exceptions.py => context/exceptions_jinja.py} (99%) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index 0461990c92d..f9c4c659a01 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -11,7 +11,7 @@ Policy, Path, ) -from dbt.exceptions import InternalException +from dbt.exceptions import InternalException, ApproximateMatch from dbt.node_types import NodeType from dbt.utils import filter_null_values, deep_merge, classproperty @@ -100,7 +100,7 @@ def matches( if approximate_match and not exact_match: target = self.create(database=database, schema=schema, identifier=identifier) - dbt.exceptions.approximate_relation_match(target, self) + raise ApproximateMatch(target, self) return exact_match diff --git a/core/dbt/jinja_exceptions.py b/core/dbt/context/exceptions_jinja.py similarity index 99% rename from core/dbt/jinja_exceptions.py rename to core/dbt/context/exceptions_jinja.py index 67c1fbc30c0..7b1f08d33a9 100644 --- a/core/dbt/jinja_exceptions.py +++ b/core/dbt/context/exceptions_jinja.py @@ -4,6 +4,7 @@ from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning + from dbt.exceptions import ( RuntimeException, MissingConfig, diff --git a/core/dbt/context/macro_resolver.py b/core/dbt/context/macro_resolver.py index a108a1889b9..7b499690da0 100644 --- a/core/dbt/context/macro_resolver.py +++ b/core/dbt/context/macro_resolver.py @@ -86,7 +86,7 @@ def _add_macro_to( package_namespaces[macro.package_name] = namespace if macro.name in namespace: - raise_duplicate_macro_name(macro, macro, macro.package_name) + raise DuplicateMacroName(macro, macro, macro.package_name) package_namespaces[macro.package_name][macro.name] = macro def add_macro(self, macro: Macro): diff --git a/core/dbt/context/macros.py b/core/dbt/context/macros.py index 700109b8081..9f1c33603b3 100644 --- a/core/dbt/context/macros.py +++ b/core/dbt/context/macros.py @@ -3,7 +3,7 @@ from dbt.clients.jinja import MacroGenerator, MacroStack from dbt.contracts.graph.nodes import Macro from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME -from dbt.exceptions import raise_duplicate_macro_name, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, raise_compiler_error FlatNamespace = Dict[str, MacroGenerator] @@ -122,7 +122,7 @@ def _add_macro_to( hierarchy[macro.package_name] = namespace if macro.name in namespace: - raise_duplicate_macro_name(macro_func.macro, macro, macro.package_name) + raise DuplicateMacroName(macro_func.macro, macro, macro.package_name) hierarchy[macro.package_name][macro.name] = macro_func def add_macro(self, macro: Macro, ctx: Dict[str, Any]): diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 117b38fc5c0..8a1cb545b2d 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -19,13 +19,14 @@ from dbt.clients import agate_helper from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack from dbt.config import RuntimeConfig, Project -from .base import contextmember, contextproperty, Var -from .configured import FQNLookup -from .context_config import ContextConfig from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER +from dbt.context.base import contextmember, contextproperty, Var +from dbt.context.configured import FQNLookup +from dbt.context.context_config import ContextConfig +from dbt.context.exceptions_jinja import wrapped_exports from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace -from .macros import MacroNamespaceBuilder, MacroNamespace -from .manifest import ManifestContext +from dbt.context.macros import MacroNamespaceBuilder, MacroNamespace +from dbt.context.manifest import ManifestContext from dbt.contracts.connection import AdapterResponse from dbt.contracts.graph.manifest import Manifest, Disabled from dbt.contracts.graph.nodes import ( @@ -55,7 +56,6 @@ raise_parsing_error, disallow_secret_env_var, ) -from dbt.jinja_exceptions import wrapped_exports from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 21052d88c3b..f2a5ba08072 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,29 +1,13 @@ import builtins from typing import NoReturn, Optional, Mapping, Any +from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning from dbt.events.contextvars import get_node_info + from dbt.node_types import NodeType -from dbt.jinja_exceptions import ( - warn, - missing_config, - missing_materialization, - missing_relation, - raise_ambiguous_alias, - raise_ambiguous_catalog_match, - raise_cache_inconsistent, - raise_dataclass_not_dict, - # raise_compiler_error, - raise_database_error, - raise_dep_not_found, - raise_dependency_error, - raise_duplicate_patch_name, - raise_duplicate_resource_name, - raise_invalid_property_yml_version, - raise_not_implemented, - relation_wrong_type, -) + import dbt.dataclass_schema @@ -38,6 +22,16 @@ def validator_error_message(exc): return "at path {}: {}".format(path, exc.message) +def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: + if path_1 == path_2: + return f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" + else: + return ( + f"remove the {type_name} entry for {name} in one of these files:\n" + f" - {path_1!s}\n{path_2!s}" + ) + + class Exception(builtins.Exception): CODE = -32000 MESSAGE = "Server Error" @@ -715,155 +709,113 @@ def get_relation_returned_multiple_results(kwargs, matches): multiple_matching_relations(kwargs, matches) -def approximate_relation_match(target, relation): - raise_compiler_error( - "When searching for a relation, dbt found an approximate match. " - "Instead of guessing \nwhich relation to use, dbt will move on. " - "Please delete {relation}, or rename it to be less ambiguous." - "\nSearched for: {target}\nFound: {relation}".format(target=target, relation=relation) - ) - +# context level exceptions +class DuplicateMacroName(CompilationException): + def __init__(self, node_1, node_2, namespace): + self.node_1 = node_1 + self.node_2 = node_2 + self.namespace = namespace + super().__init__(self.get_message()) -def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: - duped_name = node_1.name - if node_1.package_name != node_2.package_name: - extra = ' ("{}" and "{}" are both in the "{}" namespace)'.format( - node_1.package_name, node_2.package_name, namespace - ) - else: - extra = "" + def get_message(self) -> str: + duped_name = self.node_1.name + if self.node_1.package_name != self.node_2.package_name: + extra = ' ("{}" and "{}" are both in the "{}" namespace)'.format( + self.node_1.package_name, self.node_2.package_name, self.namespace + ) + else: + extra = "" - raise_compiler_error( - 'dbt found two macros with the name "{}" in the namespace "{}"{}. ' - "Since these macros have the same name and exist in the same " - "namespace, dbt will be unable to decide which to call. To fix this, " - "change the name of one of these macros:\n- {} ({})\n- {} ({})".format( - duped_name, - namespace, - extra, - node_1.unique_id, - node_1.original_file_path, - node_2.unique_id, - node_2.original_file_path, + msg = ( + f'dbt found two macros with the name "{duped_name}" in the namespace "{self.namespace}"{extra}. ' + "Since these macros have the same name and exist in the same " + "namespace, dbt will be unable to decide which to call. To fix this, " + f"change the name of one of these macros:\n- {self.node_1.unique_id} " + f"({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" ) - ) + return msg -def raise_patch_targets_not_found(patches): - patch_list = "\n\t".join( - "model {} (referenced in path {})".format(p.name, p.original_file_path) - for p in patches.values() - ) - raise_compiler_error( - "dbt could not find models for the following patches:\n\t{}".format(patch_list) - ) +# parser level exceptions +class DuplicateSourcePatchName(CompilationException): + def __init__(self, patch_1, patch_2): + self.patch_1 = patch_1 + self.patch_2 = patch_2 + super().__init__(self.get_message()) -def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: - if path_1 == path_2: - return f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" - else: - return ( - f"remove the {type_name} entry for {name} in one of these files:\n" - f" - {path_1!s}\n{path_2!s}" + def get_message(self) -> str: + name = f"{self.patch_1.overrides}.{self.patch_1.name}" + fix = _fix_dupe_msg( + self.patch_1.path, + self.patch_2.path, + name, + "sources", ) + msg = ( + f"dbt found two schema.yml entries for the same source named " + f"{self.patch_1.name} in package {self.patch_1.overrides}. Sources may only be " + f"overridden a single time. To fix this, {fix}" + ) + return msg -def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): - package_name = patch_1.package_name - name = patch_1.name - fix = _fix_dupe_msg(patch_1.original_file_path, existing_patch_path, name, "macros") - raise_compiler_error( - f"dbt found two schema.yml entries for the same macro in package " - f"{package_name} named {name}. Macros may only be described a single " - f"time. To fix this, {fix}" - ) +class DuplicateMacroPatchName(CompilationException): + def __init__(self, patch_1, existing_patch_path): + self.patch_1 = patch_1 + self.existing_patch_path = existing_patch_path + super().__init__(self.get_message()) + def get_message(self) -> str: + package_name = self.patch_1.package_name + name = self.patch_1.name + fix = _fix_dupe_msg( + self.patch_1.original_file_path, self.existing_patch_path, name, "macros" + ) + msg = ( + f"dbt found two schema.yml entries for the same macro in package " + f"{package_name} named {name}. Macros may only be described a single " + f"time. To fix this, {fix}" + ) + return msg -def raise_duplicate_source_patch_name(patch_1, patch_2): - name = f"{patch_1.overrides}.{patch_1.name}" - fix = _fix_dupe_msg( - patch_1.path, - patch_2.path, - name, - "sources", - ) - raise_compiler_error( - f"dbt found two schema.yml entries for the same source named " - f"{patch_1.name} in package {patch_1.overrides}. Sources may only be " - f"overridden a single time. To fix this, {fix}" - ) +# core level exceptions +class DuplicateAlias(AliasException): + def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str): + self.kwargs = kwargs + self.aliases = aliases + self.canonical_key = canonical_key + super().__init__(self.get_message()) -def raise_unrecognized_credentials_type(typename, supported_types): - raise_compiler_error( - 'Unrecognized credentials type "{}" - supported types are ({})'.format( - typename, ", ".join('"{}"'.format(t) for t in supported_types) + def get_message(self) -> str: + # dupe found: go through the dict so we can have a nice-ish error + key_names = ", ".join( + "{}".format(k) for k in self.kwargs if self.aliases.get(k) == self.canonical_key ) - ) + msg = f'Got duplicate keys: ({key_names}) all map to "{self.canonical_key}"' + return msg -def raise_duplicate_alias( - kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str -) -> NoReturn: - # dupe found: go through the dict so we can have a nice-ish error - key_names = ", ".join("{}".format(k) for k in kwargs if aliases.get(k) == canonical_key) +# adapters exceptions +class ApproximateMatch(CompilationException): + def __init__(self, target, relation): + self.target = target + self.relation = relation + super().__init__(self.get_message()) - raise AliasException(f'Got duplicate keys: ({key_names}) all map to "{canonical_key}"') + def get_message(self) -> str: + msg = ( + "When searching for a relation, dbt found an approximate match. " + "Instead of guessing \nwhich relation to use, dbt will move on. " + f"Please delete {self.relation}, or rename it to be less ambiguous." + f"\nSearched for: {self.target}\nFound: {self.relation}" + ) -def warn(msg, node=None): - dbt.events.functions.warn_or_error( - JinjaLogWarning(msg=msg, node_info=get_node_info()), - node=node, - ) - return "" + return msg -# Update this when a new function should be added to the -# dbt context's `exceptions` key! -CONTEXT_EXPORTS = { - fn.__name__: fn - for fn in [ - warn, - missing_config, - missing_materialization, - missing_relation, - raise_ambiguous_alias, - raise_ambiguous_catalog_match, - raise_cache_inconsistent, - raise_dataclass_not_dict, - raise_compiler_error, - raise_database_error, - raise_dep_not_found, - raise_dependency_error, - raise_duplicate_patch_name, - raise_duplicate_resource_name, - raise_invalid_property_yml_version, - raise_not_implemented, - relation_wrong_type, - ] -} - - -def wrapper(model): - def wrap(func): - @functools.wraps(func) - def inner(*args, **kwargs): - try: - return func(*args, **kwargs) - except RuntimeException as exc: - exc.add_node(model) - raise exc - - return inner - - return wrap - - -def wrapped_exports(model): - wrap = wrapper(model) - return {name: wrap(export) for name, export in CONTEXT_EXPORTS.items()} # adapters exceptions class UnexpectedNull(DatabaseException): def __init__(self, field_name, source): @@ -888,9 +840,6 @@ def __init__(self, field_name, source, dt): super().__init__(msg) -# start new exceptions - - # deps exceptions class MultipleVersionGitDeps(DependencyException): def __init__(self, git, requested): @@ -1234,74 +1183,121 @@ def get_message(self) -> str: return msg -# These are placeholders to not immediately break app adapters utilizing these functions as exceptions. +# These are copies of what's in dbt/context/exceptions_jinja.py to not immediately break adapters +# utilizing these functions as exceptions. These are direct copies to avoid circular imports. # They will be removed in 1 (or 2?) versions. Issue to be created to ensure it happens. + # TODO: add deprecation to functions -def warn(msg, node=None): # type: ignore[no-redef] # noqa - return warn(msg, node) +def warn(msg, node=None): + warn_or_error(JinjaLogWarning(msg=msg, node_info=get_node_info())) + return "" -def missing_config(model, name) -> NoReturn: # type: ignore[no-redef] # noqa - missing_config(model, name) +def missing_config(model, name) -> NoReturn: + raise MissingConfig(unique_id=model.unique_id, name=name) -def missing_materialization(model, adapter_type) -> NoReturn: # type: ignore[no-redef] # noqa - missing_materialization(model, adapter_type) +def missing_materialization(model, adapter_type) -> NoReturn: + raise MissingMaterialization(model=model, adapter_type=adapter_type) -def missing_relation(relation, model=None) -> NoReturn: # type: ignore[no-redef] # noqa - missing_relation(relation, model) +def missing_relation(relation, model=None) -> NoReturn: + raise MissingRelation(relation, model) -def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: # type: ignore[no-redef] # noqa - raise_ambiguous_alias(node_1, node_2, duped_name) +def raise_ambiguous_alias(node_1, node_2, duped_name=None) -> NoReturn: + raise AmbiguousAlias(node_1, node_2, duped_name) -def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: # type: ignore[no-redef] # noqa - raise_ambiguous_catalog_match(unique_id, match_1, match_2) +def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: + raise AmbiguousCatalogMatch(unique_id, match_1, match_2) -def raise_cache_inconsistent(message) -> NoReturn: # type: ignore[no-redef] # noqa - raise_cache_inconsistent(message) +# TODO: this should be improved to not format message here +def raise_cache_inconsistent(message) -> NoReturn: + raise InternalException("Cache inconsistency detected: {}".format(message)) -def raise_dataclass_not_dict(obj) -> NoReturn: # type: ignore[no-redef] # noqa - raise_dataclass_not_dict(obj) +def raise_dataclass_not_dict(obj) -> NoReturn: + raise DataclassNotDict(obj) -# this is already used all over our code so for now can't do this until it's fully -# removed from this file. otherwise casuses recurssion errors. +# TODO: add this is once it's removed above # def raise_compiler_error(msg, node=None) -> NoReturn: -# raise_compiler_error(msg, node=None) +# raise CompilationException(msg, node) + + +def raise_database_error(msg, node=None) -> NoReturn: + raise DatabaseException(msg, node) -def raise_database_error(msg, node=None) -> NoReturn: # type: ignore[no-redef] # noqa - raise_database_error(msg, node) +def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: + raise DependencyNotFound(node, node_description, required_pkg) -def raise_dep_not_found(node, node_description, required_pkg) -> NoReturn: # type: ignore[no-redef] # noqa - raise_dep_not_found(node, node_description, required_pkg) +def raise_dependency_error(msg) -> NoReturn: + raise DependencyException(scrub_secrets(msg, env_secrets())) -def raise_dependency_error(msg) -> NoReturn: # type: ignore[no-redef] # noqa - raise_dependency_error(msg) +def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: + raise DuplicatePatchPath(patch_1, existing_patch_path) -def raise_duplicate_patch_name(patch_1, existing_patch_path) -> NoReturn: # type: ignore[no-redef] # noqa - raise_duplicate_patch_name(patch_1, existing_patch_path) +def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: + raise DuplicateResourceName(node_1, node_2) -def raise_duplicate_resource_name(node_1, node_2) -> NoReturn: # type: ignore[no-redef] # noqa - raise_duplicate_resource_name(node_1, node_2) +def raise_invalid_property_yml_version(path, issue) -> NoReturn: + raise InvalidPropertyYML(path, issue) -def raise_invalid_property_yml_version(path, issue) -> NoReturn: # type: ignore[no-redef] # noqa - raise_invalid_property_yml_version(path, issue) +# TODO: this should be improved to not format message here +def raise_not_implemented(msg) -> NoReturn: + raise NotImplementedException("ERROR: {}".format(msg)) -def raise_not_implemented(msg) -> NoReturn: # type: ignore[no-redef] # noqa - raise_not_implemented(msg) +def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: + raise RelationWrongType(relation, expected_type, model) -def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: # type: ignore[no-redef] # noqa - relation_wrong_type(relation, expected_type, model) +# these were implemented in core so deprecating here by calling the new exception directly +def raise_duplicate_alias( + kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str +) -> NoReturn: + raise DuplicateAlias(kwargs, aliases, canonical_key) + + +def raise_duplicate_source_patch_name(patch_1, patch_2): + raise DuplicateSourcePatchName(patch_1, patch_2) + + +def raise_duplicate_macro_patch_name(patch_1, existing_patch_path): + raise DuplicateMacroPatchName(patch_1, existing_patch_path) + + +def raise_duplicate_macro_name(node_1, node_2, namespace) -> NoReturn: + raise DuplicateMacroName(node_1, node_2, namespace) + + +def approximate_relation_match(target, relation): + raise ApproximateMatch(target, relation) + + +# These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework +# TODO: is this valid? Should I create a special exception class for this? +def raise_unrecognized_credentials_type(typename, supported_types): + raise_compiler_error( + 'Unrecognized credentials type "{}" - supported types are ({})'.format( + typename, ", ".join('"{}"'.format(t) for t in supported_types) + ) + ) + + +def raise_patch_targets_not_found(patches): + patch_list = "\n\t".join( + "model {} (referenced in path {})".format(p.name, p.original_file_path) + for p in patches.values() + ) + raise_compiler_error( + "dbt could not find models for the following patches:\n\t{}".format(patch_list) + ) diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 028eceffb67..466847edf3a 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -51,7 +51,9 @@ ) from dbt.exceptions import ( CompilationException, + DuplicateMacroPatchName, DuplicatePatchPath, + DuplicateSourcePatchName, JSONValidationException, InternalException, ParsingException, @@ -60,8 +62,6 @@ PropertyYMLVersionNotInt, ValidationException, validator_error_message, - raise_duplicate_macro_patch_name, - raise_duplicate_source_patch_name, ) from dbt.events.functions import warn_or_error from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroPatchNotFound @@ -696,7 +696,7 @@ def parse(self) -> List[TestBlock]: # source patches must be unique key = (patch.overrides, patch.name) if key in self.manifest.source_patches: - raise_duplicate_source_patch_name(patch, self.manifest.source_patches[key]) + raise DuplicateSourcePatchName(patch, self.manifest.source_patches[key]) self.manifest.source_patches[key] = patch source_file.source_patches.append(key) else: @@ -981,7 +981,7 @@ def parse_patch(self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef) return if macro.patch_path: package_name, existing_file_path = macro.patch_path.split("://") - raise_duplicate_macro_patch_name(patch, existing_file_path) + raise DuplicateMacroPatchName(patch, existing_file_path) source_file.macro_patches[patch.name] = unique_id macro.patch(patch) diff --git a/core/dbt/utils.py b/core/dbt/utils.py index b7cc6475319..987371b6b02 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -15,7 +15,7 @@ from pathlib import PosixPath, WindowsPath from contextlib import contextmanager -from dbt.exceptions import ConnectionException +from dbt.exceptions import ConnectionException, DuplicateAlias from dbt.events.functions import fire_event from dbt.events.types import RetryExternalCall, RecordRetryException from dbt import flags @@ -365,7 +365,7 @@ def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]: for key, value in kwargs.items(): canonical_key = self.aliases.get(key, key) if canonical_key in result: - dbt.exceptions.raise_duplicate_alias(kwargs, self.aliases, canonical_key) + raise DuplicateAlias(kwargs, self.aliases, canonical_key) result[canonical_key] = self.translate_value(value) return result From 493c997e4e9bf50d38374536fdbbb38503a6e883 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 5 Dec 2022 13:58:39 -0600 Subject: [PATCH 06/24] more conversions --- core/dbt/adapters/base/impl.py | 4 +- core/dbt/clients/system.py | 3 +- core/dbt/exceptions.py | 72 +++++++++++++++++++++++----------- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 602086bc333..813bfff168a 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -24,9 +24,9 @@ from dbt.exceptions import ( raise_compiler_error, invalid_type_error, - get_relation_returned_multiple_results, InternalException, NotImplementedException, + RelationReturnedMultipleResults, RuntimeException, UnexpectedNull, UnexpectedNonTimestamp, @@ -769,7 +769,7 @@ def get_relation(self, database: str, schema: str, identifier: str) -> Optional[ "schema": schema, "database": database, } - get_relation_returned_multiple_results(kwargs, matches) + raise RelationReturnedMultipleResults(kwargs, matches) elif matches: return matches[0] diff --git a/core/dbt/clients/system.py b/core/dbt/clients/system.py index b1cd1b5a074..b776e91b1d0 100644 --- a/core/dbt/clients/system.py +++ b/core/dbt/clients/system.py @@ -144,7 +144,8 @@ def make_symlink(source: str, link_path: str) -> None: Create a symlink at `link_path` referring to `source`. """ if not supports_symlinks(): - dbt.exceptions.system_error("create a symbolic link") + # TODO: why not import these at top? + raise dbt.exceptions.SymbolicLinkError() os.symlink(source, link_path) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index f2a5ba08072..75d8e4ed4e2 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -689,24 +689,19 @@ def invalid_materialization_argument(name, argument): ) -def system_error(operation_name): - raise_compiler_error( - "dbt encountered an error when attempting to {}. " - "If this error persists, please create an issue at: \n\n" - "https://github.com/dbt-labs/dbt-core".format(operation_name) - ) - - -def multiple_matching_relations(kwargs, matches): - raise_compiler_error( - "get_relation returned more than one relation with the given args. " - "Please specify a database or schema to narrow down the result set." - "\n{}\n\n{}".format(kwargs, matches) - ) +# client level exceptions +class SymbolicLinkError(CompilationException): + def __init__(self): + super().__init__(self.get_message()) + def get_message(self) -> str: + msg = ( + "dbt encountered an error when attempting to create a symbolic link. " + "If this error persists, please create an issue at: \n\n" + "https://github.com/dbt-labs/dbt-core" + ) -def get_relation_returned_multiple_results(kwargs, matches): - multiple_matching_relations(kwargs, matches) + return msg # context level exceptions @@ -798,6 +793,21 @@ def get_message(self) -> str: # adapters exceptions +class RelationReturnedMultipleResults(CompilationException): + def __init__(self, kwargs, matches): + self.kwargs = kwargs + self.matches = matches + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + "get_relation returned more than one relation with the given args. " + "Please specify a database or schema to narrow down the result set." + f"\n{self.kwargs}\n\n{self.matches}" + ) + return msg + + class ApproximateMatch(CompilationException): def __init__(self, target, relation): self.target = target @@ -1283,14 +1293,27 @@ def approximate_relation_match(target, relation): raise ApproximateMatch(target, relation) +def get_relation_returned_multiple_results(kwargs, matches): + raise RelationReturnedMultipleResults(kwargs, matches) + + +def system_error(operation_name): + # Note: This was converted for core to use SymbolicLinkError because it's the only way it was used. Maintaining flexibility here for now. + msg = ( + f"dbt encountered an error when attempting to {operation_name}. " + "If this error persists, please create an issue at: \n\n" + "https://github.com/dbt-labs/dbt-core" + ) + raise CompilationException(msg) + + # These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework # TODO: is this valid? Should I create a special exception class for this? def raise_unrecognized_credentials_type(typename, supported_types): - raise_compiler_error( - 'Unrecognized credentials type "{}" - supported types are ({})'.format( - typename, ", ".join('"{}"'.format(t) for t in supported_types) - ) + msg = 'Unrecognized credentials type "{}" - supported types are ({})'.format( + typename, ", ".join('"{}"'.format(t) for t in supported_types) ) + raise CompilationException(msg) def raise_patch_targets_not_found(patches): @@ -1298,6 +1321,9 @@ def raise_patch_targets_not_found(patches): "model {} (referenced in path {})".format(p.name, p.original_file_path) for p in patches.values() ) - raise_compiler_error( - "dbt could not find models for the following patches:\n\t{}".format(patch_list) - ) + msg = f"dbt could not find models for the following patches:\n\t{patch_list}" + raise CompilationException(msg) + + +def multiple_matching_relations(kwargs, matches): + raise RelationReturnedMultipleResults(kwargs, matches) From 9f1d65cf41d9dfd219599e93f7ac3415f16cd911 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 5 Dec 2022 14:20:16 -0600 Subject: [PATCH 07/24] add changelog --- .../unreleased/Breaking Changes-20221205-141937.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changes/unreleased/Breaking Changes-20221205-141937.yaml diff --git a/.changes/unreleased/Breaking Changes-20221205-141937.yaml b/.changes/unreleased/Breaking Changes-20221205-141937.yaml new file mode 100644 index 00000000000..be840b20a99 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20221205-141937.yaml @@ -0,0 +1,9 @@ +kind: Breaking Changes +body: Cleaned up exceptions to directly raise in code. Removed use of all exception + functions in the code base and marked them all as deprecated to be removed next + minor release. +time: 2022-12-05T14:19:37.863032-06:00 +custom: + Author: emmyoop + Issue: "6339" + PR: "6347" From 0c5d7201a7e8de43338f381e96753d472e7e843b Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 5 Dec 2022 14:36:08 -0600 Subject: [PATCH 08/24] split out CacheInconsistency --- core/dbt/adapters/cache.py | 40 ++++++++------------------ core/dbt/exceptions.py | 58 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 162f7045fe5..90c4cab27fb 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -1,4 +1,3 @@ -import re import threading from copy import deepcopy from typing import Any, Dict, Iterable, List, Optional, Set, Tuple @@ -9,7 +8,13 @@ _make_msg_from_ref_key, _ReferenceKey, ) -from dbt.exceptions import CacheInconsistency +from dbt.exceptions import ( + DependentLinkNotCached, + NewNameAlreadyInCache, + NoneRelationFound, + ReferencedLinkNotCached, + TruncatedModelNameCausedCollision, +) from dbt.events.functions import fire_event, fire_event_if from dbt.events.types import ( AddLink, @@ -150,9 +155,7 @@ def rename_key(self, old_key, new_key): :raises InternalError: If the new key already exists. """ if new_key in self.referenced_by: - raise CacheInconsistency( - f'in rename of "{old_key}" -> "{new_key}", new name is in the cache already' - ) + raise NewNameAlreadyInCache(old_key, new_key) if old_key not in self.referenced_by: return @@ -268,15 +271,11 @@ def _add_link(self, referenced_key, dependent_key): if referenced is None: return if referenced is None: - raise CacheInconsistency( - f"in add_link, referenced link key {referenced_key} not in cache!" - ) + raise ReferencedLinkNotCached(referenced_key) dependent = self.relations.get(dependent_key) if dependent is None: - raise CacheInconsistency( - f"in add_link, dependent link key {dependent_key} not in cache!" - ) + raise DependentLinkNotCached(dependent_key) assert dependent is not None # we just raised! @@ -428,22 +427,7 @@ def _check_rename_constraints(self, old_key, new_key): if new_key in self.relations: # Tell user when collision caused by model names truncated during # materialization. - match = re.search("__dbt_backup|__dbt_tmp$", new_key.identifier) - if match: - truncated_model_name_prefix = new_key.identifier[: match.start()] - message_addendum = ( - "\n\nName collisions can occur when the length of two " - "models' names approach your database's builtin limit. " - "Try restructuring your project such that no two models " - "share the prefix '{}'.".format(truncated_model_name_prefix) - + " Then, clean your warehouse of any removed models." - ) - else: - message_addendum = "" - - raise CacheInconsistency( - f"in rename, new key {new_key} already in cache: {list(self.relations.keys())}{message_addendum}" - ) + raise TruncatedModelNameCausedCollision(new_key, self.relations) if old_key not in self.relations: fire_event(TemporaryRelation(key=_make_msg_from_ref_key(old_key))) @@ -501,7 +485,7 @@ def get_relations(self, database: Optional[str], schema: Optional[str]) -> List[ ] if None in results: - raise CacheInconsistency("in get_relations, a None relation was found in the cache!") + raise NoneRelationFound() return results def clear(self): diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 75d8e4ed4e2..12a0c7e5e9b 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,4 +1,5 @@ import builtins +import re from typing import NoReturn, Optional, Mapping, Any from dbt.events.functions import warn_or_error @@ -1027,6 +1028,63 @@ def __init__(self, message): super().__init__(msg) +class NewNameAlreadyInCache(CacheInconsistency): + def __init__(self, old_key, new_key): + self.old_key = old_key + self.new_key = new_key + msg = ( + f'in rename of "{self.old_key}" -> "{self.new_key}", new name is in the cache already' + ) + super().__init__(msg) + + +class ReferencedLinkNotCached(CacheInconsistency): + def __init__(self, referenced_key): + self.referenced_key = referenced_key + msg = f"in add_link, referenced link key {self.referenced_key} not in cache!" + super().__init__(msg) + + +class DependentLinkNotCached(CacheInconsistency): + def __init__(self, dependent_key): + self.dependent_key = dependent_key + msg = f"in add_link, dependent link key {self.dependent_key} not in cache!" + super().__init__(msg) + + +class TruncatedModelNameCausedCollision(CacheInconsistency): + def __init__(self, new_key, relations): + self.new_key = new_key + self.relations = relations + super().__init__(self.get_message()) + + def get_message(self) -> str: + # Tell user when collision caused by model names truncated during + # materialization. + match = re.search("__dbt_backup|__dbt_tmp$", self.new_key.identifier) + if match: + truncated_model_name_prefix = self.new_key.identifier[: match.start()] + message_addendum = ( + "\n\nName collisions can occur when the length of two " + "models' names approach your database's builtin limit. " + "Try restructuring your project such that no two models " + "share the prefix '{}'.".format(truncated_model_name_prefix) + + " Then, clean your warehouse of any removed models." + ) + else: + message_addendum = "" + + msg = f"in rename, new key {self.new_key} already in cache: {list(self.relations.keys())}{message_addendum}" + + return msg + + +class NoneRelationFound(CacheInconsistency): + def __init__(self): + msg = "in get_relations, a None relation was found in the cache!" + super().__init__(msg) + + # this is part of the context and also raised in dbt.contratcs.relation.py class DataclassNotDict(CompilationException): def __init__(self, obj): From 685d58a4156ea07e83831999d61136a02a063806 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 5 Dec 2022 15:31:16 -0600 Subject: [PATCH 09/24] more conversions --- core/dbt/clients/git.py | 25 ++--- core/dbt/clients/jinja.py | 6 +- core/dbt/compilation.py | 4 +- core/dbt/context/providers.py | 10 +- core/dbt/exceptions.py | 176 +++++++++++++++++++++++----------- 5 files changed, 139 insertions(+), 82 deletions(-) diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 9eaa93203e0..0c3ba5e1e11 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -14,10 +14,10 @@ ) from dbt.exceptions import ( CommandResultError, + GitCheckoutError, + GitCloningError, + GitCloningProblem, RuntimeException, - bad_package_spec, - raise_git_cloning_error, - raise_git_cloning_problem, ) from packaging import version @@ -27,16 +27,6 @@ def _is_commit(revision: str) -> bool: return bool(re.match(r"\b[0-9a-f]{40}\b", revision)) -def _raise_git_cloning_error(repo, revision, error): - stderr = error.stderr.strip() - if "usage: git" in stderr: - stderr = stderr.split("\nusage: git")[0] - if re.match("fatal: destination path '(.+)' already exists", stderr): - raise_git_cloning_error(error) - - bad_package_spec(repo, revision, stderr) - - def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None): has_revision = revision is not None is_commit = _is_commit(revision or "") @@ -64,7 +54,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec try: result = run_cmd(cwd, clone_cmd, env={"LC_ALL": "C"}) except CommandResultError as exc: - _raise_git_cloning_error(repo, revision, exc) + raise GitCloningError(repo, revision, exc) if subdirectory: cwd_subdir = os.path.join(cwd, dirname or "") @@ -72,7 +62,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec try: run_cmd(cwd_subdir, clone_cmd_subdir) except CommandResultError as exc: - _raise_git_cloning_error(repo, revision, exc) + raise GitCloningError(repo, revision, exc) if remove_git_dir: rmdir(os.path.join(dirname, ".git")) @@ -115,8 +105,7 @@ def checkout(cwd, repo, revision=None): try: return _checkout(cwd, repo, revision) except CommandResultError as exc: - stderr = exc.stderr.strip() - bad_package_spec(repo, revision, stderr) + raise GitCheckoutError(repo, revision, exc) def get_current_sha(cwd): @@ -145,7 +134,7 @@ def clone_and_checkout( err = exc.stderr exists = re.match("fatal: destination path '(.+)' already exists", err) if not exists: - raise_git_cloning_problem(repo) + raise GitCloningProblem(repo) directory = None start_sha = None diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index ac04bb86cb4..dcf5f480b2a 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -31,9 +31,9 @@ InternalException, raise_compiler_error, CompilationException, - invalid_materialization_argument, - MacroReturn, + InvalidMaterializationArg, JinjaRenderingException, + MacroReturn, UndefinedMacroException, ) from dbt import flags @@ -376,7 +376,7 @@ def parse(self, parser): node.defaults.append(languages) else: - invalid_materialization_argument(materialization_name, target.name) + raise InvalidMaterializationArg(materialization_name, target.name) if SUPPORTED_LANG_ARG not in node.args: node.args.append(SUPPORTED_LANG_ARG) diff --git a/core/dbt/compilation.py b/core/dbt/compilation.py index fcf98b4e914..4ae78fd3485 100644 --- a/core/dbt/compilation.py +++ b/core/dbt/compilation.py @@ -21,7 +21,7 @@ SeedNode, ) from dbt.exceptions import ( - dependency_not_found, + GraphDependencyNotFound, InternalException, RuntimeException, ) @@ -399,7 +399,7 @@ def link_node(self, linker: Linker, node: GraphMemberNode, manifest: Manifest): elif dependency in manifest.metrics: linker.dependency(node.unique_id, (manifest.metrics[dependency].unique_id)) else: - dependency_not_found(node, dependency) + raise GraphDependencyNotFound(node, dependency) def link_graph(self, linker: Linker, manifest: Manifest, add_test_edges: bool = False): for source in manifest.sources.values(): diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 8a1cb545b2d..11c6cfe3ea6 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -42,12 +42,12 @@ from dbt.events.functions import get_metadata_vars from dbt.exceptions import ( CompilationException, - ParsingException, InternalException, - ValidationException, - RuntimeException, + MacroInvalidDispatchArg, MissingConfig, - macro_invalid_dispatch_arg, + ParsingException, + RuntimeException, + ValidationException, raise_compiler_error, ref_invalid_args, metric_invalid_args, @@ -139,7 +139,7 @@ def dispatch( raise CompilationException(msg) if packages is not None: - raise macro_invalid_dispatch_arg(macro_name) + raise MacroInvalidDispatchArg(macro_name) namespace = macro_namespace diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 12a0c7e5e9b..ec96c146ce9 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -474,20 +474,6 @@ def raise_parsing_error(msg, node=None) -> NoReturn: raise ParsingException(msg, node) -def raise_git_cloning_error(error: CommandResultError) -> NoReturn: - error.cmd = scrub_secrets(str(error.cmd), env_secrets()) - raise error - - -def raise_git_cloning_problem(repo) -> NoReturn: - repo = scrub_secrets(repo, env_secrets()) - msg = """\ - Something went wrong while cloning {} - Check the debug logs for more information - """ - raise RuntimeException(msg.format(repo)) - - def disallow_secret_env_var(env_var_name) -> NoReturn: """Raise an error when a secret env var is referenced outside allowed rendering contexts""" @@ -638,59 +624,74 @@ def target_not_found( raise_compiler_error(msg, node) -def dependency_not_found(model, target_model_name): - raise_compiler_error( - "'{}' depends on '{}' which is not in the graph!".format( - model.unique_id, target_model_name - ), - model, - ) - - -def macro_not_found(model, target_macro_id): - raise_compiler_error( - model, - "'{}' references macro '{}' which is not defined!".format( - model.unique_id, target_macro_id - ), - ) - +# compilation level exceptions +class GraphDependencyNotFound(CompilationException): + def __init__(self, node, dependency): + self.node = node + self.dependency = dependency + super().__init__(self.get_message()) -def macro_invalid_dispatch_arg(macro_name) -> NoReturn: - msg = """\ - The "packages" argument of adapter.dispatch() has been deprecated. - Use the "macro_namespace" argument instead. + def get_message(self) -> str: + msg = f"'{self.node.unique_id}' depends on '{self.dependency}' which is not in the graph!" + return msg - Raised during dispatch for: {} - For more information, see: +# client level exceptions +class GitCloningProblem(RuntimeException): + def __init__(self, repo): + self.repo = scrub_secrets(repo, env_secrets()) + super().__init__(self.get_message()) - https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch - """ - raise_compiler_error(msg.format(macro_name)) + def get_message(self) -> str: + msg = f"""\ + Something went wrong while cloning {self.repo} + Check the debug logs for more information + """ + return msg -def materialization_not_available(model, adapter_type): - materialization = model.get_materialization() +class GitCloningError(InternalException): + def __init__(self, repo, revision, error): + self.repo = repo + self.revision = revision + self.error = error + super().__init__(self.get_message()) - raise_compiler_error( - "Materialization '{}' is not available for {}!".format(materialization, adapter_type), - model, - ) + def get_message(self) -> str: + stderr = self.error.stderr.strip() + if "usage: git" in stderr: + stderr = stderr.split("\nusage: git")[0] + if re.match("fatal: destination path '(.+)' already exists", stderr): + self.error.cmd = scrub_secrets(str(self.error.cmd), env_secrets()) + raise self.error + + msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{stderr}" + return scrub_secrets(msg, env_secrets()) + + +class GitCheckoutError(InternalException): + def __init__(self, repo, revision, error): + self.repo = repo + self.revision = revision + self.stderr = error.stderr.strip() + super().__init__(self.get_message()) + def get_message(self) -> str: + msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{self.stderr}" + return scrub_secrets(msg, env_secrets()) -def bad_package_spec(repo, spec, error_message): - msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message) - raise InternalException(scrub_secrets(msg, env_secrets())) +class InvalidMaterializationArg(CompilationException): + def __init__(self, name, argument): + self.name = name + self.argument = argument + super().__init__(self.get_message()) -def invalid_materialization_argument(name, argument): - raise_compiler_error( - "materialization '{}' received unknown argument '{}'.".format(name, argument) - ) + def get_message(self) -> str: + msg = f"materialization '{self.name}' received unknown argument '{self.argument}'." + return msg -# client level exceptions class SymbolicLinkError(CompilationException): def __init__(self): super().__init__(self.get_message()) @@ -706,6 +707,25 @@ def get_message(self) -> str: # context level exceptions +class MacroInvalidDispatchArg(CompilationException): + def __init__(self, macro_name): + self.macro_name = macro_name + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = f"""\ + The "packages" argument of adapter.dispatch() has been deprecated. + Use the "macro_namespace" argument instead. + + Raised during dispatch for: {self.macro_name} + + For more information, see: + + https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch + """ + return msg + + class DuplicateMacroName(CompilationException): def __init__(self, node_1, node_2, namespace): self.node_1 = node_1 @@ -794,6 +814,18 @@ def get_message(self) -> str: # adapters exceptions +class MaterializationNotAvailable(CompilationException): + def __init__(self, model, adapter_type): + self.model = model + self.adapter_type = adapter_type + super().__init__(self.get_message()) + + def get_message(self) -> str: + materialization = self.model.get_materialization() + msg = f"Materialization '{materialization}' is not available for {self.adapter_type}!" + return msg + + class RelationReturnedMultipleResults(CompilationException): def __init__(self, kwargs, matches): self.kwargs = kwargs @@ -1365,6 +1397,32 @@ def system_error(operation_name): raise CompilationException(msg) +def invalid_materialization_argument(name, argument): + raise InvalidMaterializationArg(name, argument) + + +def bad_package_spec(repo, spec, error_message): + msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message) + raise InternalException(scrub_secrets(msg, env_secrets())) + + +def raise_git_cloning_error(error: CommandResultError) -> NoReturn: + error.cmd = scrub_secrets(str(error.cmd), env_secrets()) + raise error + + +def raise_git_cloning_problem(repo) -> NoReturn: + raise GitCloningProblem(repo) + + +def macro_invalid_dispatch_arg(macro_name) -> NoReturn: + raise MacroInvalidDispatchArg(macro_name) + + +def dependency_not_found(node, dependency): + raise GraphDependencyNotFound(node, dependency) + + # These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework # TODO: is this valid? Should I create a special exception class for this? def raise_unrecognized_credentials_type(typename, supported_types): @@ -1385,3 +1443,13 @@ def raise_patch_targets_not_found(patches): def multiple_matching_relations(kwargs, matches): raise RelationReturnedMultipleResults(kwargs, matches) + + +# while this isn't in our code I wouldn't be surpised it's in adapter code +def materialization_not_available(model, adapter_type): + raise MaterializationNotAvailable(model, adapter_type) + + +def macro_not_found(model, target_macro_id): + msg = f"'{model.unique_id}' references macro '{target_macro_id}' which is not defined!" + raise CompilationException(msg=msg, node=model) From 9478bb355557d0b8b52e8e797a01cab1e4fbfa3b Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 5 Dec 2022 16:44:05 -0600 Subject: [PATCH 10/24] convert even more --- core/dbt/adapters/base/impl.py | 17 +- core/dbt/context/base.py | 4 +- core/dbt/context/configured.py | 4 +- core/dbt/context/docs.py | 10 +- core/dbt/context/providers.py | 32 +-- core/dbt/exceptions.py | 371 ++++++++++++++++++++------------- core/dbt/parser/manifest.py | 4 +- core/dbt/task/test.py | 8 +- 8 files changed, 266 insertions(+), 184 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 813bfff168a..ad93d4f3a8a 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -23,8 +23,8 @@ from dbt.exceptions import ( raise_compiler_error, - invalid_type_error, InternalException, + InvalidMacroArgType, NotImplementedException, RelationReturnedMultipleResults, RuntimeException, @@ -608,19 +608,21 @@ def get_missing_columns( to_relation. """ if not isinstance(from_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="get_missing_columns", arg_name="from_relation", got_value=from_relation, expected_type=self.Relation, + version="0.13.0", ) if not isinstance(to_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="get_missing_columns", arg_name="to_relation", got_value=to_relation, expected_type=self.Relation, + version="0.13.0", ) from_columns = {col.name: col for col in self.get_columns_in_relation(from_relation)} @@ -641,11 +643,12 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: incorrect. """ if not isinstance(relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="valid_snapshot_target", arg_name="relation", got_value=relation, expected_type=self.Relation, + version="0.13.0", ) columns = self.get_columns_in_relation(relation) @@ -679,19 +682,21 @@ def expand_target_column_types( self, from_relation: BaseRelation, to_relation: BaseRelation ) -> None: if not isinstance(from_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="expand_target_column_types", arg_name="from_relation", got_value=from_relation, expected_type=self.Relation, + version="0.13.0", ) if not isinstance(to_relation, self.Relation): - invalid_type_error( + raise InvalidMacroArgType( method_name="expand_target_column_types", arg_name="to_relation", got_value=to_relation, expected_type=self.Relation, + version="0.13.0", ) self.expand_column_types(from_relation, to_relation) diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index e57c3edac56..b8305889a76 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -11,10 +11,10 @@ from dbt.contracts.graph.nodes import Resource from dbt.exceptions import ( CompilationException, + DisallowSecretEnvVar, MacroReturn, raise_compiler_error, raise_parsing_error, - disallow_secret_env_var, ) from dbt.events.functions import fire_event, get_invocation_id from dbt.events.types import JinjaLogInfo, JinjaLogDebug @@ -300,7 +300,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: """ return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index ae2ee10baec..1df0c458a64 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -8,7 +8,7 @@ from dbt.context.base import contextproperty, contextmember, Var from dbt.context.target import TargetContext -from dbt.exceptions import raise_parsing_error, disallow_secret_env_var +from dbt.exceptions import raise_parsing_error, DisallowSecretEnvVar class ConfiguredContext(TargetContext): @@ -86,7 +86,7 @@ def var(self) -> ConfiguredVar: def env_var(self, var: str, default: Optional[str] = None) -> str: return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: diff --git a/core/dbt/context/docs.py b/core/dbt/context/docs.py index 4908829d414..89a652736dd 100644 --- a/core/dbt/context/docs.py +++ b/core/dbt/context/docs.py @@ -1,8 +1,8 @@ from typing import Any, Dict, Union from dbt.exceptions import ( - doc_invalid_args, - doc_target_not_found, + DocTargetNotFound, + InvalidDocArgs, ) from dbt.config.runtime import RuntimeConfig from dbt.contracts.graph.manifest import Manifest @@ -52,7 +52,7 @@ def doc(self, *args: str) -> str: elif len(args) == 2: doc_package_name, doc_name = args else: - doc_invalid_args(self.node, args) + raise InvalidDocArgs(self.node, args) # Documentation target_doc = self.manifest.resolve_doc( @@ -68,7 +68,9 @@ def doc(self, *args: str) -> str: # TODO CT-211 source_file.add_node(self.node.unique_id) # type: ignore[union-attr] else: - doc_target_not_found(self.node, doc_name, doc_package_name) + raise DocTargetNotFound( + node=self.node, target_doc_name=doc_name, target_doc_package=doc_package_name + ) return target_doc.block_contents diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 11c6cfe3ea6..ea780abb498 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -42,19 +42,19 @@ from dbt.events.functions import get_metadata_vars from dbt.exceptions import ( CompilationException, + DisallowSecretEnvVar, InternalException, MacroInvalidDispatchArg, + MetricInvalidArgs, MissingConfig, ParsingException, + RefBadContext, + RefInvalidArgs, RuntimeException, + TargetNotFound, ValidationException, raise_compiler_error, - ref_invalid_args, - metric_invalid_args, - target_not_found, - ref_bad_context, raise_parsing_error, - disallow_secret_env_var, ) from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage @@ -233,7 +233,7 @@ def __call__(self, *args: str) -> RelationProxy: elif len(args) == 2: package, name = args else: - ref_invalid_args(self.model, args) + raise RefInvalidArgs(node=self.model, args=args) self.validate_args(name, package) return self.resolve(name, package) @@ -294,7 +294,7 @@ def __call__(self, *args: str) -> MetricReference: elif len(args) == 2: package, name = args else: - metric_invalid_args(self.model, args) + raise MetricInvalidArgs(node=self.model, args=args) self.validate_args(name, package) return self.resolve(name, package) @@ -472,7 +472,7 @@ def resolve(self, target_name: str, target_package: Optional[str] = None) -> Rel ) if target_model is None or isinstance(target_model, Disabled): - target_not_found( + raise TargetNotFound( node=self.model, target_name=target_name, target_kind="node", @@ -494,7 +494,7 @@ def validate( ) -> None: if resolved.unique_id not in self.model.depends_on.nodes: args = self._repack_args(target_name, target_package) - ref_bad_context(self.model, args) + raise RefBadContext(node=self.model, args=args) class OperationRefResolver(RuntimeRefResolver): @@ -538,7 +538,7 @@ def resolve(self, source_name: str, table_name: str): ) if target_source is None or isinstance(target_source, Disabled): - target_not_found( + raise TargetNotFound( node=self.model, target_name=f"{source_name}.{table_name}", target_kind="source", @@ -565,7 +565,7 @@ def resolve(self, target_name: str, target_package: Optional[str] = None) -> Met ) if target_metric is None or isinstance(target_metric, Disabled): - target_not_found( + raise TargetNotFound( node=self.model, target_name=target_name, target_kind="metric", @@ -1208,7 +1208,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: """ return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: @@ -1423,7 +1423,7 @@ def generate_runtime_macro_context( class ExposureRefResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) not in (1, 2): - ref_invalid_args(self.model, args) + raise RefInvalidArgs(node=self.model, args=args) self.model.refs.append(list(args)) return "" @@ -1441,7 +1441,7 @@ def __call__(self, *args) -> str: class ExposureMetricResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) not in (1, 2): - metric_invalid_args(self.model, args) + raise MetricInvalidArgs(node=self.model, args=args) self.model.metrics.append(list(args)) return "" @@ -1483,7 +1483,7 @@ def __call__(self, *args) -> str: elif len(args) == 2: package, name = args else: - ref_invalid_args(self.model, args) + raise RefInvalidArgs(node=self.model, args=args) self.validate_args(name, package) self.model.refs.append(list(args)) return "" @@ -1573,7 +1573,7 @@ def _build_test_namespace(self): def env_var(self, var: str, default: Optional[str] = None) -> str: return_value = None if var.startswith(SECRET_ENV_PREFIX): - disallow_secret_env_var(var) + raise DisallowSecretEnvVar(var) if var in os.environ: return_value = os.environ[var] elif default is not None: diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index ec96c146ce9..da90aecbd54 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -474,156 +474,6 @@ def raise_parsing_error(msg, node=None) -> NoReturn: raise ParsingException(msg, node) -def disallow_secret_env_var(env_var_name) -> NoReturn: - """Raise an error when a secret env var is referenced outside allowed - rendering contexts""" - msg = ( - "Secret env vars are allowed only in profiles.yml or packages.yml. " - "Found '{env_var_name}' referenced elsewhere." - ) - raise_parsing_error(msg.format(env_var_name=env_var_name)) - - -def invalid_type_error( - method_name, arg_name, got_value, expected_type, version="0.13.0" -) -> NoReturn: - """Raise a CompilationException when an adapter method available to macros - has changed. - """ - got_type = type(got_value) - msg = ( - "As of {version}, 'adapter.{method_name}' expects argument " - "'{arg_name}' to be of type '{expected_type}', instead got " - "{got_value} ({got_type})" - ) - raise_compiler_error( - msg.format( - version=version, - method_name=method_name, - arg_name=arg_name, - expected_type=expected_type, - got_value=got_value, - got_type=got_type, - ) - ) - - -def invalid_bool_error(got_value, macro_name) -> NoReturn: - """Raise a CompilationException when a macro expects a boolean but gets some - other value. - """ - msg = ( - "Macro '{macro_name}' returns '{got_value}'. It is not type 'bool' " - "and cannot not be converted reliably to a bool." - ) - raise_compiler_error(msg.format(macro_name=macro_name, got_value=got_value)) - - -def ref_invalid_args(model, args) -> NoReturn: - raise_compiler_error("ref() takes at most two arguments ({} given)".format(len(args)), model) - - -def metric_invalid_args(model, args) -> NoReturn: - raise_compiler_error( - "metric() takes at most two arguments ({} given)".format(len(args)), model - ) - - -def ref_bad_context(model, args) -> NoReturn: - ref_args = ", ".join("'{}'".format(a) for a in args) - ref_string = "{{{{ ref({}) }}}}".format(ref_args) - - base_error_msg = """dbt was unable to infer all dependencies for the model "{model_name}". -This typically happens when ref() is placed within a conditional block. - -To fix this, add the following hint to the top of the model "{model_name}": - --- depends_on: {ref_string}""" - # This explicitly references model['name'], instead of model['alias'], for - # better error messages. Ex. If models foo_users and bar_users are aliased - # to 'users', in their respective schemas, then you would want to see - # 'bar_users' in your error messge instead of just 'users'. - if isinstance(model, dict): # TODO: remove this path - model_name = model["name"] - model_path = model["path"] - else: - model_name = model.name - model_path = model.path - error_msg = base_error_msg.format( - model_name=model_name, model_path=model_path, ref_string=ref_string - ) - raise_compiler_error(error_msg, model) - - -def doc_invalid_args(model, args) -> NoReturn: - raise_compiler_error("doc() takes at most two arguments ({} given)".format(len(args)), model) - - -def doc_target_not_found( - model, target_doc_name: str, target_doc_package: Optional[str] -) -> NoReturn: - target_package_string = "" - - if target_doc_package is not None: - target_package_string = "in package '{}' ".format(target_doc_package) - - msg = ("Documentation for '{}' depends on doc '{}' {} which was not found").format( - model.unique_id, target_doc_name, target_package_string - ) - raise_compiler_error(msg, model) - - -def get_not_found_or_disabled_msg( - original_file_path, - unique_id, - resource_type_title, - target_name: str, - target_kind: str, - target_package: Optional[str] = None, - disabled: Optional[bool] = None, -) -> str: - if disabled is None: - reason = "was not found or is disabled" - elif disabled is True: - reason = "is disabled" - else: - reason = "was not found" - - target_package_string = "" - if target_package is not None: - target_package_string = "in package '{}' ".format(target_package) - - return "{} '{}' ({}) depends on a {} named '{}' {}which {}".format( - resource_type_title, - unique_id, - original_file_path, - target_kind, - target_name, - target_package_string, - reason, - ) - - -def target_not_found( - node, - target_name: str, - target_kind: str, - target_package: Optional[str] = None, - disabled: Optional[bool] = None, -) -> NoReturn: - msg = get_not_found_or_disabled_msg( - original_file_path=node.original_file_path, - unique_id=node.unique_id, - resource_type_title=node.resource_type.title(), - target_name=target_name, - target_kind=target_kind, - target_package=target_package, - disabled=disabled, - ) - - raise_compiler_error(msg, node) - - # compilation level exceptions class GraphDependencyNotFound(CompilationException): def __init__(self, node, dependency): @@ -707,6 +557,129 @@ def get_message(self) -> str: # context level exceptions +class DisallowSecretEnvVar(ParsingException): + def __init__(self, env_var_name): + self.env_var_name = env_var_name + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + "Secret env vars are allowed only in profiles.yml or packages.yml. " + f"Found '{self.env_var_name}' referenced elsewhere." + ) + return msg + + +class InvalidMacroArgType(CompilationException): + def __init__(self, method_name, arg_name, got_value, expected_type, version): + self.method_name = method_name + self.arg_name = arg_name + self.got_value = got_value + self.expected_type = expected_type + self.version = version + super().__init__(self.get_message()) + + def get_message(self) -> str: + got_type = type(self.got_value) + msg = ( + f"As of {self.version}, 'adapter.{self.method_name}' expects argument " + f"'{self.arg_name}' to be of type '{self.expected_type}', instead got " + f"{self.got_value} ({got_type})" + ) + return msg + + +class InvalidBoolean(CompilationException): + def __init__(self, return_value, macro_name): + self.return_value = return_value + self.macro_name = macro_name + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Macro '{self.macro_name}' returns '{self.return_value}'. It is not type 'bool' " + "and cannot not be converted reliably to a bool." + ) + return msg + + +class RefInvalidArgs(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = f"ref() takes at most two arguments ({len(self.args)} given)" + return msg + + +class MetricInvalidArgs(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = f"metric() takes at most two arguments ({len(self.args)} given)" + return msg + + +class RefBadContext(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(self.get_message()) + + def get_message(self) -> str: + # This explicitly references model['name'], instead of model['alias'], for + # better error messages. Ex. If models foo_users and bar_users are aliased + # to 'users', in their respective schemas, then you would want to see + # 'bar_users' in your error messge instead of just 'users'. + if isinstance(self.node, dict): + model_name = self.node["name"] + else: + model_name = self.node.name + + ref_args = ", ".join("'{}'".format(a) for a in self.args) + ref_string = f"{{{{ ref({ref_args}) }}}}" + + msg = f"""dbt was unable to infer all dependencies for the model "{model_name}". +This typically happens when ref() is placed within a conditional block. + +To fix this, add the following hint to the top of the model "{model_name}": + +-- depends_on: {ref_string}""" + + return msg + + +class InvalidDocArgs(CompilationException): + def __init__(self, node, args): + self.node = node + self.args = args + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = f"doc() takes at most two arguments ({len(self.args)} given)" + return msg + + +class DocTargetNotFound(CompilationException): + def __init__(self, node, target_doc_name: str, target_doc_package: Optional[str]): + self.node = node + self.target_doc_name = target_doc_name + self.target_doc_package = target_doc_package + super().__init__(self.get_message()) + + def get_message(self) -> str: + target_package_string = "" + if self.target_doc_package is not None: + target_package_string = f"in package '{self. target_doc_package}' " + msg = f"Documentation for '{self.node.unique_id}' depends on doc '{self.target_doc_name}' {target_package_string} which was not found" + return msg + + class MacroInvalidDispatchArg(CompilationException): def __init__(self, macro_name): self.macro_name = macro_name @@ -754,6 +727,45 @@ def get_message(self) -> str: # parser level exceptions +class TargetNotFound(CompilationException): + def __init__( + self, + node, + target_name: str, + target_kind: str, + target_package: Optional[str] = None, + disabled: Optional[bool] = None, + ): + self.node = node + self.target_name = target_name + self.target_kind = target_kind + self.target_package = target_package + self.disabled = disabled + super().__init__(self.get_message()) + + def get_message(self) -> str: + original_file_path = self.node.original_file_path + unique_id = self.node.unique_id + resource_type_title = self.node.resource_type.title() + + if self.disabled is None: + reason = "was not found or is disabled" + elif self.disabled is True: + reason = "is disabled" + else: + reason = "was not found" + + target_package_string = "" + if self.target_package is not None: + target_package_string = f"in package '{self.target_package}' " + + msg = ( + f"{resource_type_title} '{unique_id}' ({original_file_path}) depends on a " + f"{self.target_kind} named '{self.target_name}' {target_package_string}which {reason}" + ) + return msg + + class DuplicateSourcePatchName(CompilationException): def __init__(self, patch_1, patch_2): self.patch_1 = patch_1 @@ -1423,6 +1435,65 @@ def dependency_not_found(node, dependency): raise GraphDependencyNotFound(node, dependency) +def target_not_found( + node, + target_name: str, + target_kind: str, + target_package: Optional[str] = None, + disabled: Optional[bool] = None, +) -> NoReturn: + raise TargetNotFound( + node=node, + target_name=target_name, + target_kind=target_kind, + target_package=target_package, + disabled=disabled, + ) + + +def doc_target_not_found( + model, target_doc_name: str, target_doc_package: Optional[str] +) -> NoReturn: + raise DocTargetNotFound( + node=model, target_doc_name=target_doc_name, target_doc_package=target_doc_package + ) + + +def doc_invalid_args(model, args) -> NoReturn: + raise InvalidDocArgs(node=model, args=args) + + +def ref_bad_context(model, args) -> NoReturn: + raise RefBadContext(node=model, args=args) + + +def metric_invalid_args(model, args) -> NoReturn: + raise MetricInvalidArgs(node=model, args=args) + + +def ref_invalid_args(model, args) -> NoReturn: + raise RefInvalidArgs(node=model, args=args) + + +def invalid_bool_error(got_value, macro_name) -> NoReturn: + raise InvalidBoolean(return_value=got_value, macro_name=macro_name) + + +def invalid_type_error( + method_name, arg_name, got_value, expected_type, version="0.13.0" +) -> NoReturn: + """Raise a CompilationException when an adapter method available to macros + has changed. + """ + raise InvalidMacroArgType(method_name, arg_name, got_value, expected_type, version) + + +def disallow_secret_env_var(env_var_name) -> NoReturn: + """Raise an error when a secret env var is referenced outside allowed + rendering contexts""" + raise DisallowSecretEnvVar(env_var_name) + + # These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework # TODO: is this valid? Should I create a special exception class for this? def raise_unrecognized_credentials_type(typename, supported_types): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 1ed84746464..9e3204a21ff 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -71,7 +71,7 @@ ResultNode, ) from dbt.contracts.util import Writable -from dbt.exceptions import target_not_found, AmbiguousAlias +from dbt.exceptions import TargetNotFound, AmbiguousAlias from dbt.parser.base import Parser from dbt.parser.analysis import AnalysisParser from dbt.parser.generic_test import GenericTestParser @@ -987,7 +987,7 @@ def invalid_target_fail_unless_test( ) ) else: - target_not_found( + raise TargetNotFound( node=node, target_name=target_name, target_kind=target_kind, diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index afc37e4b6df..26d6d46f028 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -21,7 +21,11 @@ LogTestResult, LogStartLine, ) -from dbt.exceptions import InternalException, MissingMaterialization, invalid_bool_error +from dbt.exceptions import ( + InternalException, + InvalidBoolean, + MissingMaterialization, +) from dbt.graph import ( ResourceTypeSelector, ) @@ -47,7 +51,7 @@ def convert_bool_type(field) -> bool: try: return bool(strtobool(field)) # type: ignore except ValueError: - raise invalid_bool_error(field, "get_test_sql") + raise InvalidBoolean(field, "get_test_sql") # need this so we catch both true bools and 0/1 return bool(field) From 1037aeb77214b46c5fc7f7a3661c71c54e617e48 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 6 Dec 2022 11:19:23 -0600 Subject: [PATCH 11/24] convert parsingexceptions --- core/dbt/context/base.py | 5 +- core/dbt/context/configured.py | 5 +- core/dbt/context/providers.py | 8 ++- core/dbt/context/secret.py | 5 +- core/dbt/events/functions.py | 5 +- core/dbt/exceptions.py | 69 +++++++++++++++++++++++- core/dbt/parser/generic_test_builders.py | 26 +++++---- 7 files changed, 91 insertions(+), 32 deletions(-) diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index b8305889a76..9e98f0c1ab3 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -12,9 +12,9 @@ from dbt.exceptions import ( CompilationException, DisallowSecretEnvVar, + EnvVarMissing, MacroReturn, raise_compiler_error, - raise_parsing_error, ) from dbt.events.functions import fire_event, get_invocation_id from dbt.events.types import JinjaLogInfo, JinjaLogDebug @@ -315,8 +315,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) if os.environ.get("DBT_MACRO_DEBUGGING"): diff --git a/core/dbt/context/configured.py b/core/dbt/context/configured.py index 1df0c458a64..ca1de35423b 100644 --- a/core/dbt/context/configured.py +++ b/core/dbt/context/configured.py @@ -8,7 +8,7 @@ from dbt.context.base import contextproperty, contextmember, Var from dbt.context.target import TargetContext -from dbt.exceptions import raise_parsing_error, DisallowSecretEnvVar +from dbt.exceptions import EnvVarMissing, DisallowSecretEnvVar class ConfiguredContext(TargetContext): @@ -104,8 +104,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) class MacroResolvingContext(ConfiguredContext): diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index ea780abb498..173ca0af424 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -43,6 +43,7 @@ from dbt.exceptions import ( CompilationException, DisallowSecretEnvVar, + EnvVarMissing, InternalException, MacroInvalidDispatchArg, MetricInvalidArgs, @@ -54,7 +55,6 @@ TargetNotFound, ValidationException, raise_compiler_error, - raise_parsing_error, ) from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage @@ -1241,8 +1241,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: source_file.env_vars.append(var) # type: ignore[union-attr] return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) @contextproperty def selected_resources(self) -> List[str]: @@ -1599,8 +1598,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: source_file.add_env_var(var, yaml_key, name) # type: ignore[union-attr] return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) def generate_test_context( diff --git a/core/dbt/context/secret.py b/core/dbt/context/secret.py index 11a6dc54f07..da13509ef50 100644 --- a/core/dbt/context/secret.py +++ b/core/dbt/context/secret.py @@ -4,7 +4,7 @@ from .base import BaseContext, contextmember from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER -from dbt.exceptions import raise_parsing_error +from dbt.exceptions import EnvVarMissing SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$" @@ -50,8 +50,7 @@ def env_var(self, var: str, default: Optional[str] = None) -> str: self.env_vars[var] = return_value if var in os.environ else DEFAULT_ENV_PLACEHOLDER return return_value else: - msg = f"Env var required but not provided: '{var}'" - raise_parsing_error(msg) + raise EnvVarMissing(var) def generate_secret_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]: diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 36dd2e9ba79..f061606632e 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -159,9 +159,10 @@ def event_to_dict(event: BaseEvent) -> dict: def warn_or_error(event, node=None): if flags.WARN_ERROR: - from dbt.exceptions import raise_compiler_error + # TODO: resolve this circular import when at top + from dbt.exceptions import EventCompilationException - raise_compiler_error(scrub_secrets(event.info.msg, env_secrets()), node) + raise EventCompilationException(event.info.msg, node) else: fire_event(event) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index da90aecbd54..f88f456ccdf 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -470,8 +470,12 @@ def raise_compiler_error(msg, node=None) -> NoReturn: raise CompilationException(msg, node) -def raise_parsing_error(msg, node=None) -> NoReturn: - raise ParsingException(msg, node) +# event level exception +class EventCompilationException(CompilationException): + def __init__(self, msg, node): + self.msg = scrub_secrets(msg, env_secrets()) + self.node = node + super().__init__(self.msg) # compilation level exceptions @@ -727,6 +731,63 @@ def get_message(self) -> str: # parser level exceptions +class TestNameNotString(ParsingException): + def __init__(self, test_name): + self.test_name = test_name + super().__init__(self.get_message()) + + def get_message(self) -> str: + + msg = f"test name must be a str, got {type(self.test_name)} (value {self.test_name})" + return msg + + +class TestArgsNotDict(ParsingException): + def __init__(self, test_args): + self.test_args = test_args + super().__init__(self.get_message()) + + def get_message(self) -> str: + + msg = f"test arguments must be a dict, got {type(self.test_args)} (value {self.test_args})" + return msg + + +class TestDefinitionDictLength(ParsingException): + def __init__(self, test): + self.test = test + super().__init__(self.get_message()) + + def get_message(self) -> str: + + msg = ( + "test definition dictionary must have exactly one key, got" + f" {self.test} instead ({len(self.test)} keys)" + ) + return msg + + +class TestInvalidType(ParsingException): + def __init__(self, test): + self.test = test + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = f"test must be dict or str, got {type(self.test)} (value {self.test})" + return msg + + +# This is triggered across multiple files +class EnvVarMissing(ParsingException): + def __init__(self, var): + self.var = var + super().__init__(self.get_message()) + + def get_message(self) -> str: + msg = f"Env var required but not provided: '{self.var}'" + return msg + + class TargetNotFound(CompilationException): def __init__( self, @@ -1494,6 +1555,10 @@ def disallow_secret_env_var(env_var_name) -> NoReturn: raise DisallowSecretEnvVar(env_var_name) +def raise_parsing_error(msg, node=None) -> NoReturn: + raise ParsingException(msg, node) + + # These are the exceptions functions that were not called within dbt-core but will remain here but deprecated to give a chance to rework # TODO: is this valid? Should I create a special exception class for this? def raise_unrecognized_credentials_type(typename, supported_types): diff --git a/core/dbt/parser/generic_test_builders.py b/core/dbt/parser/generic_test_builders.py index 3b1149e53a5..409322678ae 100644 --- a/core/dbt/parser/generic_test_builders.py +++ b/core/dbt/parser/generic_test_builders.py @@ -21,7 +21,14 @@ UnparsedNodeUpdate, UnparsedExposure, ) -from dbt.exceptions import raise_compiler_error, raise_parsing_error, UndefinedMacroException +from dbt.exceptions import ( + TestArgsNotDict, + TestDefinitionDictLength, + TestInvalidType, + TestNameNotString, + UndefinedMacroException, + raise_compiler_error, +) from dbt.parser.search import FileBlock @@ -314,9 +321,7 @@ def _bad_type(self) -> TypeError: @staticmethod def extract_test_args(test, name=None) -> Tuple[str, Dict[str, Any]]: if not isinstance(test, dict): - raise_parsing_error( - "test must be dict or str, got {} (value {})".format(type(test), test) - ) + raise TestInvalidType(test) # If the test is a dictionary with top-level keys, the test name is "test_name" # and the rest are arguments @@ -330,20 +335,13 @@ def extract_test_args(test, name=None) -> Tuple[str, Dict[str, Any]]: else: test = list(test.items()) if len(test) != 1: - raise_parsing_error( - "test definition dictionary must have exactly one key, got" - " {} instead ({} keys)".format(test, len(test)) - ) + raise TestDefinitionDictLength(test) test_name, test_args = test[0] if not isinstance(test_args, dict): - raise_parsing_error( - "test arguments must be dict, got {} (value {})".format(type(test_args), test_args) - ) + raise TestArgsNotDict(test_args) if not isinstance(test_name, str): - raise_parsing_error( - "test name must be a str, got {} (value {})".format(type(test_name), test_name) - ) + raise TestNameNotString(test_name) test_args = deepcopy(test_args) if name is not None: test_args["column_name"] = name From 4b56659cf986500ce8a64989b11d141c0dcfac76 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 6 Dec 2022 16:43:00 -0600 Subject: [PATCH 12/24] fix tests --- core/dbt/clients/git.py | 2 +- core/dbt/exceptions.py | 115 ++++++++++++------ core/dbt/parser/manifest.py | 2 +- .../duplicates/test_duplicate_model.py | 4 +- .../functional/exit_codes/test_exit_codes.py | 2 +- .../schema_tests/test_schema_v2_tests.py | 6 +- 6 files changed, 84 insertions(+), 47 deletions(-) diff --git a/core/dbt/clients/git.py b/core/dbt/clients/git.py index 0c3ba5e1e11..4ddbb1969ee 100644 --- a/core/dbt/clients/git.py +++ b/core/dbt/clients/git.py @@ -105,7 +105,7 @@ def checkout(cwd, repo, revision=None): try: return _checkout(cwd, repo, revision) except CommandResultError as exc: - raise GitCheckoutError(repo, revision, exc) + raise GitCheckoutError(repo=repo, revision=revision, error=exc) def get_current_sha(cwd): diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index f88f456ccdf..17c8767e4a5 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -55,7 +55,44 @@ def __init__(self, value): class InternalException(Exception): - pass + def __init__(self, msg): + self.stack = [] + self.msg = scrub_secrets(msg, env_secrets()) + + @property + def type(self): + return "Internal" + + def process_stack(self): + lines = [] + stack = self.stack + first = True + + if len(stack) > 1: + lines.append("") + + for item in stack: + msg = "called by" + + if first: + msg = "in" + first = False + + lines.append(f"> {msg}") + + return lines + + def __str__(self): + if hasattr(self.msg, "split"): + split_msg = self.msg.split("\n") + else: + split_msg = str(self.msg).split("\n") + + lines = ["{}".format(self.type + " Error")] + split_msg + + lines += self.process_stack() + + return lines[0] + "\n" + "\n".join([" " + line for line in lines[1:]]) class RuntimeException(RuntimeError, Exception): @@ -272,7 +309,7 @@ def __init__(self, expected: str, found: Optional[str]): self.found = found self.filename = "input file" - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def add_filename(self, filename: str): self.filename = filename @@ -483,7 +520,7 @@ class GraphDependencyNotFound(CompilationException): def __init__(self, node, dependency): self.node = node self.dependency = dependency - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"'{self.node.unique_id}' depends on '{self.dependency}' which is not in the graph!" @@ -494,7 +531,7 @@ def get_message(self) -> str: class GitCloningProblem(RuntimeException): def __init__(self, repo): self.repo = scrub_secrets(repo, env_secrets()) - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"""\ @@ -509,7 +546,7 @@ def __init__(self, repo, revision, error): self.repo = repo self.revision = revision self.error = error - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: stderr = self.error.stderr.strip() @@ -528,7 +565,7 @@ def __init__(self, repo, revision, error): self.repo = repo self.revision = revision self.stderr = error.stderr.strip() - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{self.stderr}" @@ -539,7 +576,7 @@ class InvalidMaterializationArg(CompilationException): def __init__(self, name, argument): self.name = name self.argument = argument - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"materialization '{self.name}' received unknown argument '{self.argument}'." @@ -548,7 +585,7 @@ def get_message(self) -> str: class SymbolicLinkError(CompilationException): def __init__(self): - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -564,7 +601,7 @@ def get_message(self) -> str: class DisallowSecretEnvVar(ParsingException): def __init__(self, env_var_name): self.env_var_name = env_var_name - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -581,7 +618,7 @@ def __init__(self, method_name, arg_name, got_value, expected_type, version): self.got_value = got_value self.expected_type = expected_type self.version = version - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: got_type = type(self.got_value) @@ -597,7 +634,7 @@ class InvalidBoolean(CompilationException): def __init__(self, return_value, macro_name): self.return_value = return_value self.macro_name = macro_name - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -611,7 +648,7 @@ class RefInvalidArgs(CompilationException): def __init__(self, node, args): self.node = node self.args = args - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"ref() takes at most two arguments ({len(self.args)} given)" @@ -622,7 +659,7 @@ class MetricInvalidArgs(CompilationException): def __init__(self, node, args): self.node = node self.args = args - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"metric() takes at most two arguments ({len(self.args)} given)" @@ -633,7 +670,7 @@ class RefBadContext(CompilationException): def __init__(self, node, args): self.node = node self.args = args - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: # This explicitly references model['name'], instead of model['alias'], for @@ -662,7 +699,7 @@ class InvalidDocArgs(CompilationException): def __init__(self, node, args): self.node = node self.args = args - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"doc() takes at most two arguments ({len(self.args)} given)" @@ -674,7 +711,7 @@ def __init__(self, node, target_doc_name: str, target_doc_package: Optional[str] self.node = node self.target_doc_name = target_doc_name self.target_doc_package = target_doc_package - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: target_package_string = "" @@ -687,7 +724,7 @@ def get_message(self) -> str: class MacroInvalidDispatchArg(CompilationException): def __init__(self, macro_name): self.macro_name = macro_name - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"""\ @@ -708,7 +745,7 @@ def __init__(self, node_1, node_2, namespace): self.node_1 = node_1 self.node_2 = node_2 self.namespace = namespace - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: duped_name = self.node_1.name @@ -734,7 +771,7 @@ def get_message(self) -> str: class TestNameNotString(ParsingException): def __init__(self, test_name): self.test_name = test_name - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -745,7 +782,7 @@ def get_message(self) -> str: class TestArgsNotDict(ParsingException): def __init__(self, test_args): self.test_args = test_args - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -756,7 +793,7 @@ def get_message(self) -> str: class TestDefinitionDictLength(ParsingException): def __init__(self, test): self.test = test - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -770,7 +807,7 @@ def get_message(self) -> str: class TestInvalidType(ParsingException): def __init__(self, test): self.test = test - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"test must be dict or str, got {type(self.test)} (value {self.test})" @@ -781,7 +818,7 @@ def get_message(self) -> str: class EnvVarMissing(ParsingException): def __init__(self, var): self.var = var - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = f"Env var required but not provided: '{self.var}'" @@ -802,7 +839,7 @@ def __init__( self.target_kind = target_kind self.target_package = target_package self.disabled = disabled - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: original_file_path = self.node.original_file_path @@ -831,7 +868,7 @@ class DuplicateSourcePatchName(CompilationException): def __init__(self, patch_1, patch_2): self.patch_1 = patch_1 self.patch_2 = patch_2 - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: name = f"{self.patch_1.overrides}.{self.patch_1.name}" @@ -853,7 +890,7 @@ class DuplicateMacroPatchName(CompilationException): def __init__(self, patch_1, existing_patch_path): self.patch_1 = patch_1 self.existing_patch_path = existing_patch_path - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: package_name = self.patch_1.package_name @@ -875,7 +912,7 @@ def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canoni self.kwargs = kwargs self.aliases = aliases self.canonical_key = canonical_key - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: # dupe found: go through the dict so we can have a nice-ish error @@ -891,7 +928,7 @@ class MaterializationNotAvailable(CompilationException): def __init__(self, model, adapter_type): self.model = model self.adapter_type = adapter_type - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: materialization = self.model.get_materialization() @@ -903,7 +940,7 @@ class RelationReturnedMultipleResults(CompilationException): def __init__(self, kwargs, matches): self.kwargs = kwargs self.matches = matches - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -918,7 +955,7 @@ class ApproximateMatch(CompilationException): def __init__(self, target, relation): self.target = target self.relation = relation - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -1052,7 +1089,7 @@ class MissingMaterialization(CompilationException): def __init__(self, model, adapter_type): self.model = model self.adapter_type = adapter_type - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: materialization = self.model.get_materialization() @@ -1082,7 +1119,7 @@ def __init__(self, node_1, node_2, duped_name=None): self.duped_name = f"{self.node_1.database}.{self.node_1.schema}.{self.node_1.alias}" else: self.duped_name = duped_name - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -1106,7 +1143,7 @@ def __init__(self, unique_id, match_1, match_2): self.unique_id = unique_id self.match_1 = match_1 self.match_2 = match_2 - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_match_string(self, match): return "{}.{}".format( @@ -1194,7 +1231,7 @@ def __init__(self): class DataclassNotDict(CompilationException): def __init__(self, obj): self.obj = obj # TODO: what kind of obj is this? - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -1210,7 +1247,7 @@ def __init__(self, node, node_description, required_pkg): self.node = node self.node_description = node_description self.required_pkg = required_pkg - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -1226,7 +1263,7 @@ class DuplicatePatchPath(CompilationException): def __init__(self, patch_1, existing_patch_path): self.patch_1 = patch_1 self.existing_patch_path = existing_patch_path - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: name = self.patch_1.name @@ -1249,7 +1286,7 @@ class DuplicateResourceName(CompilationException): def __init__(self, node_1, node_2): self.node_1 = node_1 self.node_2 = node_2 - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: duped_name = self.node_1.name @@ -1301,7 +1338,7 @@ class InvalidPropertyYML(CompilationException): def __init__(self, path, issue): self.path = path self.issue = issue - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( @@ -1343,7 +1380,7 @@ def __init__(self, relation, expected_type, model=None): self.relation = relation self.expected_type = expected_type self.model = model - super().__init__(self.get_message()) + super().__init__(msg=self.get_message()) def get_message(self) -> str: msg = ( diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 9e3204a21ff..9da68736031 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -1015,7 +1015,7 @@ def _check_resource_uniqueness( existing_node = names_resources.get(name) if existing_node is not None: - dbt.exceptions.DuplicateResourceName(existing_node, node) + raise dbt.exceptions.DuplicateResourceName(existing_node, node) existing_alias = alias_resources.get(full_node_name) if existing_alias is not None: diff --git a/tests/functional/duplicates/test_duplicate_model.py b/tests/functional/duplicates/test_duplicate_model.py index 031ba6236c0..fbcd1b79671 100644 --- a/tests/functional/duplicates/test_duplicate_model.py +++ b/tests/functional/duplicates/test_duplicate_model.py @@ -1,6 +1,6 @@ import pytest -from dbt.exceptions import CompilationException +from dbt.exceptions import CompilationException, DuplicateResourceName from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt, get_manifest @@ -108,7 +108,7 @@ def packages(self): def test_duplicate_model_enabled_across_packages(self, project): run_dbt(["deps"]) message = "dbt found two models with the name" - with pytest.raises(CompilationException) as exc: + with pytest.raises(DuplicateResourceName) as exc: run_dbt(["run"]) assert message in str(exc.value) diff --git a/tests/functional/exit_codes/test_exit_codes.py b/tests/functional/exit_codes/test_exit_codes.py index 955953a0dc0..54b5cb6865e 100644 --- a/tests/functional/exit_codes/test_exit_codes.py +++ b/tests/functional/exit_codes/test_exit_codes.py @@ -99,7 +99,7 @@ def packages(self): } def test_deps_fail(self, project): - with pytest.raises(dbt.exceptions.InternalException) as exc: + with pytest.raises(dbt.exceptions.GitCheckoutError) as exc: run_dbt(['deps']) expected_msg = "Error checking out spec='bad-branch'" assert expected_msg in str(exc.value) diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py index 00c14cd711b..44a6696931b 100644 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ b/tests/functional/schema_tests/test_schema_v2_tests.py @@ -95,7 +95,7 @@ alt_local_utils__macros__type_timestamp_sql, all_quotes_schema__schema_yml, ) -from dbt.exceptions import ParsingException, CompilationException +from dbt.exceptions import ParsingException, CompilationException, DuplicateResourceName from dbt.contracts.results import TestStatus @@ -904,9 +904,9 @@ def test_generic_test_collision( project, ): """These tests collide, since only the configs differ""" - with pytest.raises(CompilationException) as exc: + with pytest.raises(DuplicateResourceName) as exc: run_dbt() - assert "dbt found two tests with the name" in str(exc) + assert "dbt found two tests with the name" in str(exc.value) class TestGenericTestsConfigCustomMacros: From 3a739b8ff30f3e54256e31d281b033f872f320d3 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 7 Dec 2022 13:13:48 -0600 Subject: [PATCH 13/24] more conversions --- core/dbt/clients/_jinja_blocks.py | 55 +++++--------- core/dbt/clients/jinja_static.py | 12 +--- core/dbt/exceptions.py | 116 +++++++++++++++++++++++++++++- 3 files changed, 132 insertions(+), 51 deletions(-) diff --git a/core/dbt/clients/_jinja_blocks.py b/core/dbt/clients/_jinja_blocks.py index c1ef31acf44..fa74a317649 100644 --- a/core/dbt/clients/_jinja_blocks.py +++ b/core/dbt/clients/_jinja_blocks.py @@ -1,7 +1,15 @@ import re from collections import namedtuple -import dbt.exceptions +from dbt.exceptions import ( + BlockDefinitionNotAtTop, + InternalException, + MissingCloseTag, + MissingControlFlowStartTag, + NestedTags, + UnexpectedControlFlowEndTag, + UnexpectedMacroEOF, +) def regex(pat): @@ -139,10 +147,7 @@ def _first_match(self, *patterns, **kwargs): def _expect_match(self, expected_name, *patterns, **kwargs): match = self._first_match(*patterns, **kwargs) if match is None: - msg = 'unexpected EOF, expected {}, got "{}"'.format( - expected_name, self.data[self.pos :] - ) - dbt.exceptions.raise_compiler_error(msg) + raise UnexpectedMacroEOF(expected_name, self.data[self.pos :]) return match def handle_expr(self, match): @@ -256,7 +261,7 @@ def find_tags(self): elif block_type_name is not None: yield self.handle_tag(match) else: - raise dbt.exceptions.InternalException( + raise InternalException( "Invalid regex match in next_block, expected block start, " "expr start, or comment start" ) @@ -265,13 +270,6 @@ def __iter__(self): return self.find_tags() -duplicate_tags = ( - "Got nested tags: {outer.block_type_name} (started at {outer.start}) did " - "not have a matching {{% end{outer.block_type_name} %}} before a " - "subsequent {inner.block_type_name} was found (started at {inner.start})" -) - - _CONTROL_FLOW_TAGS = { "if": "endif", "for": "endfor", @@ -319,33 +317,16 @@ def find_blocks(self, allowed_blocks=None, collect_raw_data=True): found = self.stack.pop() else: expected = _CONTROL_FLOW_END_TAGS[tag.block_type_name] - dbt.exceptions.raise_compiler_error( - ( - "Got an unexpected control flow end tag, got {} but " - "never saw a preceeding {} (@ {})" - ).format(tag.block_type_name, expected, self.tag_parser.linepos(tag.start)) - ) + raise UnexpectedControlFlowEndTag(tag, expected, self.tag_parser) expected = _CONTROL_FLOW_TAGS[found] if expected != tag.block_type_name: - dbt.exceptions.raise_compiler_error( - ( - "Got an unexpected control flow end tag, got {} but " - "expected {} next (@ {})" - ).format(tag.block_type_name, expected, self.tag_parser.linepos(tag.start)) - ) + raise MissingControlFlowStartTag(tag, expected, self.tag_parser) if tag.block_type_name in allowed_blocks: if self.stack: - dbt.exceptions.raise_compiler_error( - ( - "Got a block definition inside control flow at {}. " - "All dbt block definitions must be at the top level" - ).format(self.tag_parser.linepos(tag.start)) - ) + raise BlockDefinitionNotAtTop(self.tag_parser, tag.start) if self.current is not None: - dbt.exceptions.raise_compiler_error( - duplicate_tags.format(outer=self.current, inner=tag) - ) + raise NestedTags(outer=self.current, inner=tag) if collect_raw_data: raw_data = self.data[self.last_position : tag.start] self.last_position = tag.start @@ -366,11 +347,7 @@ def find_blocks(self, allowed_blocks=None, collect_raw_data=True): if self.current: linecount = self.data[: self.current.end].count("\n") + 1 - dbt.exceptions.raise_compiler_error( - ("Reached EOF without finding a close tag for {} (searched from line {})").format( - self.current.block_type_name, linecount - ) - ) + raise MissingCloseTag(self.current.block_type_name, linecount) if collect_raw_data: raw_data = self.data[self.last_position :] diff --git a/core/dbt/clients/jinja_static.py b/core/dbt/clients/jinja_static.py index 337a25eadda..d71211cea6e 100644 --- a/core/dbt/clients/jinja_static.py +++ b/core/dbt/clients/jinja_static.py @@ -1,6 +1,6 @@ import jinja2 from dbt.clients.jinja import get_environment -from dbt.exceptions import raise_compiler_error +from dbt.exceptions import MacroNamespaceNotString, MacroNameNotString def statically_extract_macro_calls(string, ctx, db_wrapper=None): @@ -117,20 +117,14 @@ def statically_parse_adapter_dispatch(func_call, ctx, db_wrapper): func_name = kwarg.value.value possible_macro_calls.append(func_name) else: - raise_compiler_error( - f"The macro_name parameter ({kwarg.value.value}) " - "to adapter.dispatch was not a string" - ) + raise MacroNameNotString(kwarg_value=kwarg.value.value) elif kwarg.key == "macro_namespace": # This will remain to enable static resolution kwarg_type = type(kwarg.value).__name__ if kwarg_type == "Const": macro_namespace = kwarg.value.value else: - raise_compiler_error( - "The macro_namespace parameter to adapter.dispatch " - f"is a {kwarg_type}, not a string" - ) + raise MacroNamespaceNotString(kwarg_type) # positional arguments if packages_arg: diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 17c8767e4a5..e8f4c8de351 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -512,7 +512,7 @@ class EventCompilationException(CompilationException): def __init__(self, msg, node): self.msg = scrub_secrets(msg, env_secrets()) self.node = node - super().__init__(self.msg) + super().__init__(msg=self.msg) # compilation level exceptions @@ -528,6 +528,116 @@ def get_message(self) -> str: # client level exceptions +class MacroNameNotString(CompilationException): + def __init__(self, kwarg_value): + self.kwarg_value = kwarg_value + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"The macro_name parameter ({self.kwarg_value}) " + "to adapter.dispatch was not a string" + ) + return msg + + +class MissingControlFlowStartTag(CompilationException): + def __init__(self, tag, expected_tag, tag_parser): + self.tag = tag + self.expected_tag = expected_tag + self.tag_parser = tag_parser + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + linepos = self.tag_parser.linepos(self.tag.start) + msg = ( + f"Got an unexpected control flow end tag, got {self.tag.block_type_name} but " + f"expected {self.expected_tag} next (@ {linepos})" + ) + return msg + + +class UnexpectedControlFlowEndTag(CompilationException): + def __init__(self, tag, expected_tag, tag_parser): + self.tag = tag + self.expected_tag = expected_tag + self.tag_parser = tag_parser + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + linepos = self.tag_parser.linepos(self.tag.start) + msg = ( + f"Got an unexpected control flow end tag, got {self.tag.block_type_name} but " + f"never saw a preceeding {self.expected_tag} (@ {linepos})" + ) + return msg + + +class UnexpectedMacroEOF(CompilationException): + def __init__(self, expected_name, actual_name): + self.expected_name = expected_name + self.actual_name = actual_name + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f'unexpected EOF, expected {self.expected_name}, got "{self.actual_name}"' + return msg + + +class MacroNamespaceNotString(CompilationException): + def __init__(self, kwarg_type): + self.kwarg_type = kwarg_type + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + "The macro_namespace parameter to adapter.dispatch " + f"is a {self.kwarg_type}, not a string" + ) + return msg + + +class NestedTags(CompilationException): + def __init__(self, outer, inner): + self.outer = outer + self.inner = inner + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Got nested tags: {self.outer.block_type_name} (started at {self.outer.start}) did " + f"not have a matching {{{{% end{self.outer.block_type_name} %}}}} before a " + f"subsequent {self.inner.block_type_name} was found (started at {self.inner.start})" + ) + return msg + + +class BlockDefinitionNotAtTop(CompilationException): + def __init__(self, tag_parser, tag_start): + self.tag_parser = tag_parser + self.tag_start = tag_start + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + position = self.tag_parser.linepos(self.tag_start) + msg = ( + f"Got a block definition inside control flow at {position}. " + "All dbt block definitions must be at the top level" + ) + return msg + + +class MissingCloseTag(CompilationException): + def __init__(self, block_type_name, linecount): + self.block_type_name = block_type_name + self.linecount = linecount + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"Reached EOF without finding a close tag for {self.block_type_name} (searched from line {self.linecount})" + return msg + + class GitCloningProblem(RuntimeException): def __init__(self, repo): self.repo = scrub_secrets(repo, env_secrets()) @@ -1082,7 +1192,7 @@ def __init__(self, unique_id, name): msg = ( f"Model '{self.unique_id}' does not define a required config parameter '{self.name}'." ) - super().__init__(msg) + super().__init__(msg=msg) class MissingMaterialization(CompilationException): @@ -1108,7 +1218,7 @@ def __init__(self, relation, model=None): self.relation = relation self.model = model msg = f"Relation {self.relation} not found!" - super().__init__(msg) + super().__init__(msg=msg) class AmbiguousAlias(CompilationException): From 72a00726eba2ec80c09287a6e2af5a6ab295f8e0 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 7 Dec 2022 14:58:18 -0600 Subject: [PATCH 14/24] more conversions --- core/dbt/adapters/base/relation.py | 4 +- core/dbt/adapters/sql/impl.py | 7 +- core/dbt/config/runtime.py | 20 ++-- core/dbt/config/utils.py | 8 +- core/dbt/context/base.py | 8 +- core/dbt/context/macro_resolver.py | 4 +- core/dbt/context/macros.py | 4 +- core/dbt/contracts/graph/manifest.py | 32 +----- core/dbt/exceptions.py | 155 ++++++++++++++++++++++++++- 9 files changed, 175 insertions(+), 67 deletions(-) diff --git a/core/dbt/adapters/base/relation.py b/core/dbt/adapters/base/relation.py index f9c4c659a01..5bc0c56b264 100644 --- a/core/dbt/adapters/base/relation.py +++ b/core/dbt/adapters/base/relation.py @@ -11,7 +11,7 @@ Policy, Path, ) -from dbt.exceptions import InternalException, ApproximateMatch +from dbt.exceptions import ApproximateMatch, InternalException, MultipleDatabasesNotAllowed from dbt.node_types import NodeType from dbt.utils import filter_null_values, deep_merge, classproperty @@ -438,7 +438,7 @@ def flatten(self, allow_multiple_databases: bool = False): if not allow_multiple_databases: seen = {r.database.lower() for r in self if r.database} if len(seen) > 1: - dbt.exceptions.raise_compiler_error(str(seen)) + raise MultipleDatabasesNotAllowed(seen) for information_schema_name, schema in self.search(): path = {"database": information_schema_name.database, "schema": schema} diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index 20241d9e53d..4606b046f54 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -1,9 +1,8 @@ import agate from typing import Any, Optional, Tuple, Type, List -import dbt.clients.agate_helper from dbt.contracts.connection import Connection -import dbt.exceptions +from dbt.exceptions import RelationTypeNull from dbt.adapters.base import BaseAdapter, available from dbt.adapters.cache import _make_ref_key_msg from dbt.adapters.sql import SQLConnectionManager @@ -132,9 +131,7 @@ def alter_column_type(self, relation, column_name, new_column_type) -> None: def drop_relation(self, relation): if relation.type is None: - dbt.exceptions.raise_compiler_error( - "Tried to drop relation {}, but its type is null.".format(relation) - ) + raise RelationTypeNull(relation) self.cache_dropped(relation) self.execute_macro(DROP_RELATION_MACRO_NAME, kwargs={"relation": relation}) diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index 236baf497a6..0e011e2efe8 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -26,8 +26,9 @@ from dbt.dataclass_schema import ValidationError from dbt.exceptions import ( DbtProjectError, + NonUniquePackageName, RuntimeException, - raise_compiler_error, + UninstalledPackagesFound, validator_error_message, ) from dbt.events.functions import warn_or_error @@ -352,22 +353,15 @@ def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]: count_packages_specified = len(self.packages.packages) # type: ignore count_packages_installed = len(tuple(self._get_project_directories())) if count_packages_specified > count_packages_installed: - raise_compiler_error( - f"dbt found {count_packages_specified} package(s) " - f"specified in packages.yml, but only " - f"{count_packages_installed} package(s) installed " - f'in {self.packages_install_path}. Run "dbt deps" to ' - f"install package dependencies." + raise UninstalledPackagesFound( + count_packages_specified, + count_packages_installed, + self.packages_install_path, ) project_paths = itertools.chain(internal_packages, self._get_project_directories()) for project_name, project in self.load_projects(project_paths): if project_name in all_projects: - raise_compiler_error( - f"dbt found more than one package with the name " - f'"{project_name}" included in this project. Package ' - f"names must be unique in a project. Please rename " - f"one of these packages." - ) + raise NonUniquePackageName(project_name) all_projects[project_name] = project self.dependencies = all_projects return self.dependencies diff --git a/core/dbt/config/utils.py b/core/dbt/config/utils.py index 728e558ebbd..921626ba088 100644 --- a/core/dbt/config/utils.py +++ b/core/dbt/config/utils.py @@ -9,7 +9,7 @@ from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer from dbt.events.functions import fire_event from dbt.events.types import InvalidVarsYAML -from dbt.exceptions import ValidationException, raise_compiler_error +from dbt.exceptions import ValidationException, VarsArgNotYamlDict def parse_cli_vars(var_string: str) -> Dict[str, Any]: @@ -19,11 +19,7 @@ def parse_cli_vars(var_string: str) -> Dict[str, Any]: if var_type is dict: return cli_vars else: - type_name = var_type.__name__ - raise_compiler_error( - "The --vars argument must be a YAML dictionary, but was " - "of type '{}'".format(type_name) - ) + raise VarsArgNotYamlDict(var_type) except ValidationException: fire_event(InvalidVarsYAML()) raise diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 9e98f0c1ab3..98efaf69a50 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -14,7 +14,7 @@ DisallowSecretEnvVar, EnvVarMissing, MacroReturn, - raise_compiler_error, + RequiredVarNotFound, ) from dbt.events.functions import fire_event, get_invocation_id from dbt.events.types import JinjaLogInfo, JinjaLogDebug @@ -128,7 +128,6 @@ def __new__(mcls, name, bases, dct): class Var: - UndefinedVarError = "Required var '{}' not found in config:\nVars supplied to {} = {}" _VAR_NOTSET = object() def __init__( @@ -153,10 +152,7 @@ def node_name(self): return "" def get_missing_var(self, var_name): - dct = {k: self._merged[k] for k in self._merged} - pretty_vars = json.dumps(dct, sort_keys=True, indent=4) - msg = self.UndefinedVarError.format(var_name, self.node_name, pretty_vars) - raise_compiler_error(msg, self._node) + raise RequiredVarNotFound(var_name, self._merged, self._node) def has_var(self, var_name: str): return var_name in self._merged diff --git a/core/dbt/context/macro_resolver.py b/core/dbt/context/macro_resolver.py index 7b499690da0..6e70bafd05e 100644 --- a/core/dbt/context/macro_resolver.py +++ b/core/dbt/context/macro_resolver.py @@ -1,6 +1,6 @@ from typing import Dict, MutableMapping, Optional from dbt.contracts.graph.nodes import Macro -from dbt.exceptions import raise_duplicate_macro_name, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, PackageNotFoundForMacro from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME from dbt.clients.jinja import MacroGenerator @@ -187,7 +187,7 @@ def get_from_package(self, package_name: Optional[str], name: str) -> Optional[M elif package_name in self.macro_resolver.packages: macro = self.macro_resolver.packages[package_name].get(name) else: - raise_compiler_error(f"Could not find package '{package_name}'") + raise PackageNotFoundForMacro(package_name) if not macro: return None macro_func = MacroGenerator(macro, self.ctx, self.node, self.thread_ctx) diff --git a/core/dbt/context/macros.py b/core/dbt/context/macros.py index 9f1c33603b3..921480ec05a 100644 --- a/core/dbt/context/macros.py +++ b/core/dbt/context/macros.py @@ -3,7 +3,7 @@ from dbt.clients.jinja import MacroGenerator, MacroStack from dbt.contracts.graph.nodes import Macro from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME -from dbt.exceptions import DuplicateMacroName, raise_compiler_error +from dbt.exceptions import DuplicateMacroName, PackageNotFoundForMacro FlatNamespace = Dict[str, MacroGenerator] @@ -75,7 +75,7 @@ def get_from_package(self, package_name: Optional[str], name: str) -> Optional[M elif package_name in self.packages: return self.packages[package_name].get(name) else: - raise_compiler_error(f"Could not find package '{package_name}'") + raise PackageNotFoundForMacro(package_name) # This class builds the MacroNamespace by adding macros to diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 95acf8438ec..c43012ec521 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -42,13 +42,13 @@ from dbt.exceptions import ( CompilationException, DuplicateResourceName, - raise_compiler_error, + DuplicateMacroInPackage, + DuplicateMaterializationName, ) from dbt.helper_types import PathSet from dbt.events.functions import fire_event from dbt.events.types import MergedFromState from dbt.node_types import NodeType -from dbt.ui import line_wrap_message from dbt import flags from dbt import tracking import dbt.utils @@ -398,12 +398,7 @@ def __eq__(self, other: object) -> bool: return NotImplemented equal = self.specificity == other.specificity and self.locality == other.locality if equal: - raise_compiler_error( - "Found two materializations with the name {} (packages {} and " - "{}). dbt cannot resolve this ambiguity".format( - self.macro.name, self.macro.package_name, other.macro.package_name - ) - ) + raise DuplicateMaterializationName(self.macro, other) return equal @@ -1040,26 +1035,7 @@ def merge_from_artifact( def add_macro(self, source_file: SourceFile, macro: Macro): if macro.unique_id in self.macros: # detect that the macro exists and emit an error - other_path = self.macros[macro.unique_id].original_file_path - # subtract 2 for the "Compilation Error" indent - # note that the line wrap eats newlines, so if you want newlines, - # this is the result :( - msg = line_wrap_message( - f"""\ - dbt found two macros named "{macro.name}" in the project - "{macro.package_name}". - - - To fix this error, rename or remove one of the following - macros: - - - {macro.original_file_path} - - - {other_path} - """, - subtract=2, - ) - raise_compiler_error(msg) + raise DuplicateMacroInPackage(macro=macro, macro_mapping=self.macros) self.macros[macro.unique_id] = macro source_file.macros.append(macro.unique_id) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index e8f4c8de351..432b7abfc76 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,4 +1,5 @@ import builtins +import json import re from typing import NoReturn, Optional, Mapping, Any @@ -6,9 +7,8 @@ from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning from dbt.events.contextvars import get_node_info - from dbt.node_types import NodeType - +from dbt.ui import line_wrap_message import dbt.dataclass_schema @@ -708,6 +708,35 @@ def get_message(self) -> str: # context level exceptions + + +class RequiredVarNotFound(CompilationException): + def __init__(self, var_name, merged, node): + self.var_name = var_name + self.merged = merged + self.node = node + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + if self.node is not None: + node_name = self.node.name + else: + node_name = "" + + dct = {k: self.merged[k] for k in self.merged} + pretty_vars = json.dumps(dct, sort_keys=True, indent=4) + + msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}" + return msg + + +class PackageNotFoundForMacro(CompilationException): + def __init__(self, package_name): + self.package_name = package_name + msg = f"Could not find package '{self.package_name}'" + super().__init__(msg) + + class DisallowSecretEnvVar(ParsingException): def __init__(self, env_var_name): self.env_var_name = env_var_name @@ -1034,6 +1063,25 @@ def get_message(self) -> str: # adapters exceptions + + +class MultipleDatabasesNotAllowed(CompilationException): + def __init__(self, databases): + self.databases = databases + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = str(self.databases) + return msg + + +class RelationTypeNull(CompilationException): + def __init__(self, relation): + self.relation = relation + self.msg = f"Tried to drop relation {self.relation}, but its type is null." + super().__init__(msg=self.msg) + + class MaterializationNotAvailable(CompilationException): def __init__(self, model, adapter_type): self.model = model @@ -1184,6 +1232,107 @@ def __init__(self, package_name): super().__init__(msg) +# config level exceptions + + +class NonUniquePackageName(CompilationException): + def __init__(self, project_name): + self.project_name = project_name + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + "dbt found more than one package with the name " + f'"{self.project_name}" included in this project. Package ' + "names must be unique in a project. Please rename " + "one of these packages." + ) + return msg + + +class UninstalledPackagesFound(CompilationException): + def __init__(self, count_packages_specified, count_packages_installed, packages_install_path): + self.count_packages_specified = count_packages_specified + self.count_packages_installed = count_packages_installed + self.packages_install_path = packages_install_path + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"dbt found {self.count_packages_specified} package(s) " + "specified in packages.yml, but only " + f"{self.count_packages_installed} package(s) installed " + f'in {self.packages_install_path}. Run "dbt deps" to ' + "install package dependencies." + ) + return msg + + +class VarsArgNotYamlDict(CompilationException): + def __init__(self, var_type): + self.var_type = var_type + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + type_name = self.var_type.__name__ + + msg = "The --vars argument must be a YAML dictionary, but was " "of type '{}'".format( + type_name + ) + return msg + + +# contracts level + + +class DuplicateMacroInPackage(CompilationException): + def __init__(self, macro, macro_mapping): + self.macro = macro + self.macro_mapping = macro_mapping + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + other_path = self.macro_mapping[self.macro.unique_id].original_file_path + # subtract 2 for the "Compilation Error" indent + # note that the line wrap eats newlines, so if you want newlines, + # this is the result :( + msg = line_wrap_message( + f"""\ + dbt found two macros named "{self.macro.name}" in the project + "{self.macro.package_name}". + + + To fix this error, rename or remove one of the following + macros: + + - {self.macro.original_file_path} + + - {other_path} + """, + subtract=2, + ) + return msg + + +class DuplicateMaterializationName(CompilationException): + def __init__(self, macro, other_macro): + self.macro = macro + self.other_macro = other_macro + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + macro_name = self.macro.name + macro_package_name = self.macro.package_name + other_package_name = self.other_macro.macro.package_name + + msg = ( + f"Found two materializations with the name {macro_name} (packages " + f"{macro_package_name} and {other_package_name}). dbt cannot resolve " + "this ambiguity" + ) + return msg + + # jinja exceptions class MissingConfig(CompilationException): def __init__(self, unique_id, name): @@ -1337,7 +1486,7 @@ def __init__(self): super().__init__(msg) -# this is part of the context and also raised in dbt.contratcs.relation.py +# this is part of the context and also raised in dbt.contracts.relation.py class DataclassNotDict(CompilationException): def __init__(self, obj): self.obj = obj # TODO: what kind of obj is this? From d98f4f39240ebf6a6cfd23dc7edac638a2d7f021 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 7 Dec 2022 21:54:30 -0600 Subject: [PATCH 15/24] finish converting exception functions --- core/dbt/adapters/base/impl.py | 39 ++-- core/dbt/clients/jinja.py | 18 +- core/dbt/context/providers.py | 56 ++--- core/dbt/exceptions.py | 263 +++++++++++++++++++++-- core/dbt/parser/generic_test_builders.py | 47 ++-- 5 files changed, 314 insertions(+), 109 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index ad93d4f3a8a..48c4c4f2989 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -22,12 +22,18 @@ import pytz from dbt.exceptions import ( - raise_compiler_error, InternalException, InvalidMacroArgType, + InvalidMacroResult, + InvalidQuoteConfigType, NotImplementedException, + NullRelationCacheAttempted, + NullRelationDropAttempted, RelationReturnedMultipleResults, + RenameToNoneAttempted, RuntimeException, + SnapshotTargetIncomplete, + SnapshotTargetNotSnapshotTable, UnexpectedNull, UnexpectedNonTimestamp, ) @@ -427,7 +433,7 @@ def cache_added(self, relation: Optional[BaseRelation]) -> str: """Cache a new relation in dbt. It will show up in `list relations`.""" if relation is None: name = self.nice_connection_name() - raise_compiler_error("Attempted to cache a null relation for {}".format(name)) + raise NullRelationCacheAttempted(name) self.cache.add(relation) # so jinja doesn't render things return "" @@ -439,7 +445,7 @@ def cache_dropped(self, relation: Optional[BaseRelation]) -> str: """ if relation is None: name = self.nice_connection_name() - raise_compiler_error("Attempted to drop a null relation for {}".format(name)) + raise NullRelationDropAttempted(name) self.cache.drop(relation) return "" @@ -456,9 +462,7 @@ def cache_renamed( name = self.nice_connection_name() src_name = _relation_name(from_relation) dst_name = _relation_name(to_relation) - raise_compiler_error( - "Attempted to rename {} to {} for {}".format(src_name, dst_name, name) - ) + raise RenameToNoneAttempted(src_name, dst_name, name) self.cache.rename(from_relation, to_relation) return "" @@ -665,17 +669,9 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: if missing: if extra: - msg = ( - 'Snapshot target has ("{}") but not ("{}") - is it an ' - "unmigrated previous version archive?".format( - '", "'.join(extra), '", "'.join(missing) - ) - ) + raise SnapshotTargetIncomplete(extra, missing) else: - msg = 'Snapshot target is not a snapshot table (missing "{}")'.format( - '", "'.join(missing) - ) - raise_compiler_error(msg) + raise SnapshotTargetNotSnapshotTable(missing) @available.parse_none def expand_target_column_types( @@ -838,10 +834,7 @@ def quote_seed_column(self, column: str, quote_config: Optional[bool]) -> str: elif quote_config is None: pass else: - raise_compiler_error( - f'The seed configuration value of "quote_columns" has an ' - f"invalid type {type(quote_config)}" - ) + raise InvalidQuoteConfigType(quote_config) if quote_columns: return self.quote(column) @@ -1091,11 +1084,7 @@ def calculate_freshness( # now we have a 1-row table of the maximum `loaded_at_field` value and # the current time according to the db. if len(table) != 1 or len(table[0]) != 2: - raise_compiler_error( - 'Got an invalid result from "{}" macro: {}'.format( - FRESHNESS_MACRO_NAME, [tuple(r) for r in table] - ) - ) + raise InvalidMacroResult(FRESHNESS_MACRO_NAME, table) if table[0][0] is None: # no records in the table, so really the max_loaded_at was # infinitely long ago. Just call it 0:00 January 1 year UTC diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index dcf5f480b2a..c1b8865e33e 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -28,12 +28,16 @@ from dbt.contracts.graph.nodes import GenericTestNode from dbt.exceptions import ( - InternalException, - raise_compiler_error, + CaughtMacroException, + CaughtMacroExceptionWithNode, CompilationException, + InternalException, InvalidMaterializationArg, JinjaRenderingException, MacroReturn, + MaterializtionMacroNotUsed, + NoSupportedLanguagesFound, + UndefinedCompilation, UndefinedMacroException, ) from dbt import flags @@ -237,7 +241,7 @@ def exception_handler(self) -> Iterator[None]: try: yield except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e: - raise_compiler_error(str(e)) + raise CaughtMacroException(e) def call_macro(self, *args, **kwargs): # called from __call__ methods @@ -296,7 +300,7 @@ def exception_handler(self) -> Iterator[None]: try: yield except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e: - raise_compiler_error(str(e), self.macro) + raise CaughtMacroExceptionWithNode(exc=e, node=self.macro) except CompilationException as e: e.stack.append(self.macro) raise e @@ -451,7 +455,7 @@ def __call__(self, *args, **kwargs): return self def __reduce__(self): - raise_compiler_error(f"{self.name} is undefined", node=node) + raise UndefinedCompilation(name=self.name, node=node) return Undefined @@ -651,13 +655,13 @@ def _convert_function(value: Any, keypath: Tuple[Union[str, int], ...]) -> Any: def get_supported_languages(node: jinja2.nodes.Macro) -> List[ModelLanguage]: if "materialization" not in node.name: - raise_compiler_error("Only materialization macros can be used with this function") + raise MaterializtionMacroNotUsed(node=node) no_kwargs = not node.defaults no_langs_found = SUPPORTED_LANG_ARG not in node.args if no_kwargs or no_langs_found: - raise_compiler_error(f"No supported_languages found in materialization macro {node.name}") + raise NoSupportedLanguagesFound(node=node) lang_idx = node.args.index(SUPPORTED_LANG_ARG) # indexing defaults from the end diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index 173ca0af424..b6567ba8def 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -42,19 +42,27 @@ from dbt.events.functions import get_metadata_vars from dbt.exceptions import ( CompilationException, + ConflictingConfigKeys, DisallowSecretEnvVar, EnvVarMissing, InternalException, + InvalidInlineModelConfig, + InvalidNumberSourceArgs, + InvalidPersistDocsValueType, + LoadAgateTableNotSeed, + LoadAgateTableValueError, MacroInvalidDispatchArg, + MacrosSourcesUnWriteable, MetricInvalidArgs, MissingConfig, + OperationsCannotRefEphemeralNodes, + PackageNotInDeps, ParsingException, RefBadContext, RefInvalidArgs, RuntimeException, TargetNotFound, ValidationException, - raise_compiler_error, ) from dbt.config import IsFQNResource from dbt.node_types import NodeType, ModelLanguage @@ -257,9 +265,7 @@ def validate_args(self, source_name: str, table_name: str): def __call__(self, *args: str) -> RelationProxy: if len(args) != 2: - raise_compiler_error( - f"source() takes exactly two arguments ({len(args)} given)", self.model - ) + raise InvalidNumberSourceArgs(args, node=self.model) self.validate_args(args[0], args[1]) return self.resolve(args[0], args[1]) @@ -315,12 +321,7 @@ def _transform_config(self, config): if oldkey in config: newkey = oldkey.replace("_", "-") if newkey in config: - raise_compiler_error( - 'Invalid config, has conflicting keys "{}" and "{}"'.format( - oldkey, newkey - ), - self.model, - ) + raise ConflictingConfigKeys(oldkey, newkey, node=self.model) config[newkey] = config.pop(oldkey) return config @@ -330,7 +331,7 @@ def __call__(self, *args, **kwargs): elif len(args) == 0 and len(kwargs) > 0: opts = kwargs else: - raise_compiler_error("Invalid inline model config", self.model) + raise InvalidInlineModelConfig(node=self.model) opts = self._transform_config(opts) @@ -400,20 +401,14 @@ def get(self, name, default=None, validator=None): def persist_relation_docs(self) -> bool: persist_docs = self.get("persist_docs", default={}) if not isinstance(persist_docs, dict): - raise_compiler_error( - f"Invalid value provided for 'persist_docs'. Expected dict " - f"but received {type(persist_docs)}" - ) + raise InvalidPersistDocsValueType(persist_docs) return persist_docs.get("relation", False) def persist_column_docs(self) -> bool: persist_docs = self.get("persist_docs", default={}) if not isinstance(persist_docs, dict): - raise_compiler_error( - f"Invalid value provided for 'persist_docs'. Expected dict " - f"but received {type(persist_docs)}" - ) + raise InvalidPersistDocsValueType(persist_docs) return persist_docs.get("columns", False) @@ -510,12 +505,7 @@ def create_relation(self, target_model: ManifestNode, name: str) -> RelationProx if target_model.is_ephemeral_model: # In operations, we can't ref() ephemeral nodes, because # Macros do not support set_cte - raise_compiler_error( - "Operations can not ref() ephemeral nodes, but {} is ephemeral".format( - target_model.name - ), - self.model, - ) + raise OperationsCannotRefEphemeralNodes(target_model.name, node=self.model) else: return super().create_relation(target_model, name) @@ -594,7 +584,7 @@ def packages_for_node(self) -> Iterable[Project]: if package_name != self._config.project_name: if package_name not in dependencies: # I don't think this is actually reachable - raise_compiler_error(f"Node package named {package_name} not found!", self._node) + raise PackageNotInDeps(package_name, node=self._node) yield dependencies[package_name] yield self._config @@ -777,7 +767,7 @@ def inner(value: T) -> None: def write(self, payload: str) -> str: # macros/source defs aren't 'writeable'. if isinstance(self.model, (Macro, SourceDefinition)): - raise_compiler_error('cannot "write" macros or sources') + raise MacrosSourcesUnWriteable(node=self.model) self.model.build_path = self.model.write_node(self.config.target_path, "run", payload) return "" @@ -792,21 +782,19 @@ def try_or_compiler_error( try: return func(*args, **kwargs) except Exception: - raise_compiler_error(message_if_exception, self.model) + raise CompilationException(message_if_exception, self.model) @contextmember def load_agate_table(self) -> agate.Table: if not isinstance(self.model, SeedNode): - raise_compiler_error( - "can only load_agate_table for seeds (got a {})".format(self.model.resource_type) - ) + raise LoadAgateTableNotSeed(self.model.resource_type, node=self.model) assert self.model.root_path path = os.path.join(self.model.root_path, self.model.original_file_path) column_types = self.model.config.column_types try: table = agate_helper.from_csv(path, text_columns=column_types) except ValueError as e: - raise_compiler_error(str(e)) + raise LoadAgateTableValueError(str(e), node=self.model) table.original_abspath = os.path.abspath(path) return table @@ -1430,9 +1418,7 @@ def __call__(self, *args) -> str: class ExposureSourceResolver(BaseResolver): def __call__(self, *args) -> str: if len(args) != 2: - raise_compiler_error( - f"source() takes exactly two arguments ({len(args)} given)", self.model - ) + raise InvalidNumberSourceArgs(args, node=self.model) self.model.sources.append(list(args)) return "" diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 432b7abfc76..f012da3a776 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -499,14 +499,6 @@ class ConnectionException(Exception): pass -# TODO: these are all the functins that need to be converted and deprecated - - -# TODO: this was copied into jinja_exxceptions because it's in the context - eventually remove? -def raise_compiler_error(msg, node=None) -> NoReturn: - raise CompilationException(msg, node) - - # event level exception class EventCompilationException(CompilationException): def __init__(self, msg, node): @@ -528,6 +520,43 @@ def get_message(self) -> str: # client level exceptions + + +class NoSupportedLanguagesFound(CompilationException): + def __init__(self, node): + self.node = node + self.msg = f"No supported_languages found in materialization macro {self.node.name}" + super().__init__(msg=self.msg) + + +class MaterializtionMacroNotUsed(CompilationException): + def __init__(self, node): + self.node = node + self.msg = "Only materialization macros can be used with this function" + super().__init__(msg=self.msg) + + +class UndefinedCompilation(CompilationException): + def __init__(self, name, node): + self.name = name + self.node = node + self.msg = f"{self.name} is undefined" + super().__init__(msg=self.msg) + + +class CaughtMacroExceptionWithNode(CompilationException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + super().__init__(msg=str(exc)) + + +class CaughtMacroException(CompilationException): + def __init__(self, exc): + self.exc = exc + super().__init__(msg=str(exc)) + + class MacroNameNotString(CompilationException): def __init__(self, kwarg_value): self.kwarg_value = kwarg_value @@ -710,6 +739,79 @@ def get_message(self) -> str: # context level exceptions +class LoadAgateTableValueError(CompilationException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + msg = str(self.exc) + super().__init__(msg=msg) + + +class LoadAgateTableNotSeed(CompilationException): + def __init__(self, resource_type, node): + self.resource_type = resource_type + self.node = node + msg = f"can only load_agate_table for seeds (got a {self.resource_type})" + super().__init__(msg=msg) + + +class MacrosSourcesUnWriteable(CompilationException): + def __init__(self, node): + self.node = node + msg = 'cannot "write" macros or sources' + super().__init__(msg=msg) + + +class PackageNotInDeps(CompilationException): + def __init__(self, package_name, node): + self.package_name = package_name + self.node = node + msg = f"Node package named {self.package_name} not found!" + super().__init__(msg=msg) + + +class OperationsCannotRefEphemeralNodes(CompilationException): + def __init__(self, target_name, node): + self.target_name = target_name + self.node = node + msg = f"Operations can not ref() ephemeral nodes, but {target_name} is ephemeral" + super().__init__(msg=msg) + + +class InvalidPersistDocsValueType(CompilationException): + def __init__(self, persist_docs): + self.persist_docs = persist_docs + msg = ( + "Invalid value provided for 'persist_docs'. Expected dict " + f"but received {type(self.persist_docs)}" + ) + super().__init__(msg=msg) + + +class InvalidInlineModelConfig(CompilationException): + def __init__(self, node): + self.node = node + msg = "Invalid inline model config" + super().__init__(msg=msg) + + +class ConflictingConfigKeys(CompilationException): + def __init__(self, oldkey, newkey, node): + self.oldkey = oldkey + self.newkey = newkey + self.node = node + msg = f'Invalid config, has conflicting keys "{self.oldkey}" and "{self.newkey}"' + super().__init__(msg=msg) + + +class InvalidNumberSourceArgs(CompilationException): + def __init__(self, args, node): + self.args = args + self.node = node + msg = f"source() takes exactly two arguments ({len(self.args)} given)" + super().__init__(msg=msg) + + class RequiredVarNotFound(CompilationException): def __init__(self, var_name, merged, node): self.var_name = var_name @@ -734,7 +836,7 @@ class PackageNotFoundForMacro(CompilationException): def __init__(self, package_name): self.package_name = package_name msg = f"Could not find package '{self.package_name}'" - super().__init__(msg) + super().__init__(msg=msg) class DisallowSecretEnvVar(ParsingException): @@ -907,6 +1009,69 @@ def get_message(self) -> str: # parser level exceptions +class SameKeyNested(CompilationException): + def __init__(self): + msg = "Test cannot have the same key at the top-level and in config" + super().__init__(msg=msg) + + +class TestArgIncludesModel(CompilationException): + def __init__(self): + msg = 'Test arguments include "model", which is a reserved argument' + super().__init__(msg=msg) + + +class UnexpectedTestNamePattern(CompilationException): + def __init__(self, test_name): + self.test_name = test_name + msg = f"Test name string did not match expected pattern: {self.test_name}" + super().__init__(msg=msg) + + +class CustomMacroPopulatingConfigValues(CompilationException): + def __init__(self, target_name, column_name, name, key, err_msg): + self.target_name = target_name + self.column_name = column_name + self.name = name + self.key = key + self.err_msg = err_msg + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + # Generic tests do not include custom macros in the Jinja + # rendering context, so this will almost always fail. As it + # currently stands, the error message is inscrutable, which + # has caused issues for some projects migrating from + # pre-0.20.0 to post-0.20.0. + # See https://github.com/dbt-labs/dbt-core/issues/4103 + # and https://github.com/dbt-labs/dbt-core/issues/5294 + + msg = ( + f"The {self.target_name}.{self.column_name} column's " + f'"{self.name}" test references an undefined ' + f"macro in its {self.key} configuration argument. " + f"The macro {self.err_msg}.\n" + "Please note that the generic test configuration parser " + "currently does not support using custom macros to " + "populate configuration values" + ) + return msg + + +class TagsNotListOfStrings(CompilationException): + def __init__(self, tags): + self.tags = tags + msg = f"got {self.tags} ({type(self.tags)}) for tags, expected a list of strings" + super().__init__(msg=msg) + + +class TagNotString(CompilationException): + def __init__(self, tag): + self.tag = tag + msg = f"got {self.tag} ({type(self.tag)}) for tag, expected a str" + super().__init__(msg=msg) + + class TestNameNotString(ParsingException): def __init__(self, test_name): self.test_name = test_name @@ -1063,6 +1228,80 @@ def get_message(self) -> str: # adapters exceptions +class InvalidMacroResult(CompilationException): + def __init__(self, freshness_macro_name, table): + self.freshness_macro_name = freshness_macro_name + self.table = table + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = 'Got an invalid result from "{self.freshness_macro_name}" macro: {[tuple(r) for r in self.table]}' + + return msg + + +class SnapshotTargetNotSnapshotTable(CompilationException): + def __init__(self, missing): + self.missing = missing + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = 'Snapshot target is not a snapshot table (missing "{}")'.format( + '", "'.join(self.missing) + ) + return msg + + +class SnapshotTargetIncomplete(CompilationException): + def __init__(self, extra, missing): + self.extra = extra + self.missing = missing + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + 'Snapshot target has ("{}") but not ("{}") - is it an ' + "unmigrated previous version archive?".format( + '", "'.join(self.extra), '", "'.join(self.missing) + ) + ) + return msg + + +class RenameToNoneAttempted(CompilationException): + def __init__(self, src_name, dst_name, name): + self.src_name = src_name + self.dst_name = dst_name + self.name = name + self.msg = f"Attempted to rename {self.src_name} to {self.dst_name} for {self.name}" + super().__init__(msg=self.msg) + + +class NullRelationDropAttempted(CompilationException): + def __init__(self, name): + self.name = name + self.msg = f"Attempted to drop a null relation for {self.name}" + super().__init__(msg=self.msg) + + +class NullRelationCacheAttempted(CompilationException): + def __init__(self, name): + self.name = name + self.msg = f"Attempted to cache a null relation for {self.name}" + super().__init__(msg=self.msg) + + +class InvalidQuoteConfigType(CompilationException): + def __init__(self, quote_config): + self.quote_config = quote_config + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + 'The seed configuration value of "quote_columns" has an ' + f"invalid type {type(self.quote_config)}" + ) + return msg class MultipleDatabasesNotAllowed(CompilationException): @@ -1691,9 +1930,9 @@ def raise_dataclass_not_dict(obj) -> NoReturn: raise DataclassNotDict(obj) -# TODO: add this is once it's removed above -# def raise_compiler_error(msg, node=None) -> NoReturn: -# raise CompilationException(msg, node) +# note: this is called all over the code in addition to in jinja +def raise_compiler_error(msg, node=None) -> NoReturn: + raise CompilationException(msg, node) def raise_database_error(msg, node=None) -> NoReturn: diff --git a/core/dbt/parser/generic_test_builders.py b/core/dbt/parser/generic_test_builders.py index 409322678ae..af0282c953f 100644 --- a/core/dbt/parser/generic_test_builders.py +++ b/core/dbt/parser/generic_test_builders.py @@ -22,12 +22,17 @@ UnparsedExposure, ) from dbt.exceptions import ( + CustomMacroPopulatingConfigValues, + SameKeyNested, + TagNotString, + TagsNotListOfStrings, + TestArgIncludesModel, TestArgsNotDict, TestDefinitionDictLength, TestInvalidType, TestNameNotString, + UnexpectedTestNamePattern, UndefinedMacroException, - raise_compiler_error, ) from dbt.parser.search import FileBlock @@ -229,9 +234,7 @@ def __init__( test_name, test_args = self.extract_test_args(test, column_name) self.args: Dict[str, Any] = test_args if "model" in self.args: - raise_compiler_error( - 'Test arguments include "model", which is a reserved argument', - ) + raise TestArgIncludesModel() self.package_name: str = package_name self.target: Testable = target @@ -239,9 +242,7 @@ def __init__( match = self.TEST_NAME_PATTERN.match(test_name) if match is None: - raise_compiler_error( - "Test name string did not match expected pattern: {}".format(test_name) - ) + raise UnexpectedTestNamePattern(test_name) groups = match.groupdict() self.name: str = groups["test_name"] @@ -258,9 +259,7 @@ def __init__( value = self.args.pop(key, None) # 'modifier' config could be either top level arg or in config if value and "config" in self.args and key in self.args["config"]: - raise_compiler_error( - "Test cannot have the same key at the top-level and in config" - ) + raise SameKeyNested() if not value and "config" in self.args: value = self.args["config"].pop(key, None) if isinstance(value, str): @@ -268,22 +267,12 @@ def __init__( try: value = get_rendered(value, render_ctx, native=True) except UndefinedMacroException as e: - - # Generic tests do not include custom macros in the Jinja - # rendering context, so this will almost always fail. As it - # currently stands, the error message is inscrutable, which - # has caused issues for some projects migrating from - # pre-0.20.0 to post-0.20.0. - # See https://github.com/dbt-labs/dbt-core/issues/4103 - # and https://github.com/dbt-labs/dbt-core/issues/5294 - raise_compiler_error( - f"The {self.target.name}.{column_name} column's " - f'"{self.name}" test references an undefined ' - f"macro in its {key} configuration argument. " - f"The macro {e.msg}.\n" - "Please note that the generic test configuration parser " - "currently does not support using custom macros to " - "populate configuration values" + raise CustomMacroPopulatingConfigValues( + target_name=self.target.name, + column_name=column_name, + name=self.name, + key=key, + err_msg=e.msg ) if value is not None: @@ -432,12 +421,10 @@ def tags(self) -> List[str]: if isinstance(tags, str): tags = [tags] if not isinstance(tags, list): - raise_compiler_error( - f"got {tags} ({type(tags)}) for tags, expected a list of strings" - ) + raise TagsNotListOfStrings(tags) for tag in tags: if not isinstance(tag, str): - raise_compiler_error(f"got {tag} ({type(tag)}) for tag, expected a str") + raise TagNotString(tag) return tags[:] def macro_name(self) -> str: From 62f8b421bc5d03ec0ad3989f0767f2391079b3ab Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 12 Dec 2022 19:54:47 -0600 Subject: [PATCH 16/24] convert more tests --- core/dbt/config/profile.py | 16 +- core/dbt/config/project.py | 22 +- core/dbt/config/runtime.py | 4 +- core/dbt/context/exceptions_jinja.py | 7 +- core/dbt/exceptions.py | 252 ++++++++++++++++-- core/dbt/parser/base.py | 8 +- core/dbt/parser/models.py | 21 +- core/dbt/parser/schemas.py | 49 +--- core/dbt/parser/snapshots.py | 4 +- .../postgres/dbt/adapters/postgres/impl.py | 31 +-- 10 files changed, 287 insertions(+), 127 deletions(-) diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py index 39679baa109..e8bf85dbd27 100644 --- a/core/dbt/config/profile.py +++ b/core/dbt/config/profile.py @@ -9,12 +9,14 @@ from dbt.clients.yaml_helper import load_yaml_text from dbt.contracts.connection import Credentials, HasCredentials from dbt.contracts.project import ProfileConfig, UserConfig -from dbt.exceptions import CompilationException -from dbt.exceptions import DbtProfileError -from dbt.exceptions import DbtProjectError -from dbt.exceptions import ValidationException -from dbt.exceptions import RuntimeException -from dbt.exceptions import validator_error_message +from dbt.exceptions import ( + CompilationException, + DbtProfileError, + DbtProjectError, + ValidationException, + RuntimeException, + ProfileConfigInvalid, +) from dbt.events.types import MissingProfileTarget from dbt.events.functions import fire_event from dbt.utils import coerce_dict_str @@ -156,7 +158,7 @@ def validate(self): dct = self.to_profile_info(serialize_credentials=True) ProfileConfig.validate(dct) except ValidationError as exc: - raise DbtProfileError(validator_error_message(exc)) from exc + raise ProfileConfigInvalid(exc) from exc @staticmethod def _credentials_from_profile( diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 9521dd29882..69c6b79866c 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -16,19 +16,19 @@ import os from dbt import flags, deprecations -from dbt.clients.system import resolve_path_from_base -from dbt.clients.system import path_exists -from dbt.clients.system import load_file_contents +from dbt.clients.system import path_exists, resolve_path_from_base, load_file_contents from dbt.clients.yaml_helper import load_yaml_text from dbt.contracts.connection import QueryComment -from dbt.exceptions import DbtProjectError -from dbt.exceptions import SemverException -from dbt.exceptions import validator_error_message -from dbt.exceptions import RuntimeException +from dbt.exceptions import ( + DbtProjectError, + SemverException, + ProjectContractBroken, + ProjectContractInvalid, + RuntimeException, +) from dbt.graph import SelectionSpec from dbt.helper_types import NoValue -from dbt.semver import VersionSpecifier -from dbt.semver import versions_compatible +from dbt.semver import VersionSpecifier, versions_compatible from dbt.version import get_installed_version from dbt.utils import MultiDict from dbt.node_types import NodeType @@ -325,7 +325,7 @@ def create_project(self, rendered: RenderComponents) -> "Project": ProjectContract.validate(rendered.project_dict) cfg = ProjectContract.from_dict(rendered.project_dict) except ValidationError as e: - raise DbtProjectError(validator_error_message(e)) from e + raise ProjectContractInvalid(e) from e # name/version are required in the Project definition, so we can assume # they are present name = cfg.name @@ -642,7 +642,7 @@ def validate(self): try: ProjectContract.validate(self.to_project_config()) except ValidationError as e: - raise DbtProjectError(validator_error_message(e)) from e + raise ProjectContractBroken(e) from e @classmethod def partial_load(cls, project_root: str, *, verify_version: bool = False) -> PartialProject: diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index 0e011e2efe8..8b1b30f383b 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -25,11 +25,11 @@ from dbt.contracts.relation import ComponentName from dbt.dataclass_schema import ValidationError from dbt.exceptions import ( + ConfigContractBroken, DbtProjectError, NonUniquePackageName, RuntimeException, UninstalledPackagesFound, - validator_error_message, ) from dbt.events.functions import warn_or_error from dbt.events.types import UnusedResourceConfigPath @@ -187,7 +187,7 @@ def validate(self): try: Configuration.validate(self.serialize()) except ValidationError as e: - raise DbtProjectError(validator_error_message(e)) from e + raise ConfigContractBroken(e) from e @classmethod def _get_rendered_profile( diff --git a/core/dbt/context/exceptions_jinja.py b/core/dbt/context/exceptions_jinja.py index 7b1f08d33a9..f167b5ee8da 100644 --- a/core/dbt/context/exceptions_jinja.py +++ b/core/dbt/context/exceptions_jinja.py @@ -12,7 +12,7 @@ MissingRelation, AmbiguousAlias, AmbiguousCatalogMatch, - InternalException, + CacheInconsistency, DataclassNotDict, CompilationException, DatabaseException, @@ -51,9 +51,8 @@ def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: raise AmbiguousCatalogMatch(unique_id, match_1, match_2) -# TODO: this should be improved to not format message here def raise_cache_inconsistent(message) -> NoReturn: - raise InternalException("Cache inconsistency detected: {}".format(message)) + raise CacheInconsistency(message) def raise_dataclass_not_dict(obj) -> NoReturn: @@ -90,7 +89,7 @@ def raise_invalid_property_yml_version(path, issue) -> NoReturn: # TODO: this should be improved to not format message here def raise_not_implemented(msg) -> NoReturn: - raise NotImplementedException("ERROR: {}".format(msg)) + raise NotImplementedException(msg) def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index f012da3a776..fab60133983 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,8 +1,9 @@ import builtins import json import re -from typing import NoReturn, Optional, Mapping, Any +from typing import Any, Mapping, NoReturn, Optional +from dbt.dataclass_schema import ValidationError from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets from dbt.events.types import JinjaLogWarning @@ -13,26 +14,6 @@ import dbt.dataclass_schema -def validator_error_message(exc): - """Given a dbt.dataclass_schema.ValidationError (which is basically a - jsonschema.ValidationError), return the relevant parts as a string - """ - if not isinstance(exc, dbt.dataclass_schema.ValidationError): - return str(exc) - path = "[%s]" % "][".join(map(repr, exc.relative_path)) - return "at path {}: {}".format(path, exc.message) - - -def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str: - if path_1 == path_2: - return f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" - else: - return ( - f"remove the {type_name} entry for {name} in one of these files:\n" - f" - {path_1!s}\n{path_2!s}" - ) - - class Exception(builtins.Exception): CODE = -32000 MESSAGE = "Server Error" @@ -147,6 +128,15 @@ def process_stack(self): return lines + def validator_error_message(self, exc): + """Given a dbt.dataclass_schema.ValidationError (which is basically a + jsonschema.ValidationError), return the relevant parts as a string + """ + if not isinstance(exc, dbt.dataclass_schema.ValidationError): + return str(exc) + path = "[%s]" % "][".join(map(repr, exc.relative_path)) + return "at path {}: {}".format(path, exc.message) + def __str__(self, prefix="! "): node_string = "" @@ -269,6 +259,17 @@ class CompilationException(RuntimeException): def type(self): return "Compilation" + def _fix_dupe_msg(self, path_1: str, path_2: str, name: str, type_name: str) -> str: + if path_1 == path_2: + return ( + f"remove one of the {type_name} entries for {name} in this file:\n - {path_1!s}\n" + ) + else: + return ( + f"remove the {type_name} entry for {name} in one of these files:\n" + f" - {path_1!s}\n{path_2!s}" + ) + class RecursionException(RuntimeException): pass @@ -423,7 +424,10 @@ class VersionsNotCompatibleException(SemverException): class NotImplementedException(Exception): - pass + def __init__(self, message): + self.message = message + self.msg = f"ERROR: {self.message}" + super().__init__(msg=self.msg) class FailedToConnectException(DatabaseException): @@ -1009,6 +1013,118 @@ def get_message(self) -> str: # parser level exceptions +class InvalidDictParse(ParsingException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + msg = self.validator_error_message(exc) + super().__init__(msg=msg) + + +class InvalidConfigUpdate(ParsingException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + msg = self.validator_error_message(exc) + super().__init__(msg=msg) + + +class PythonParsingException(ParsingException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + validated_exc = self.validator_error_message(self.exc) + msg = f"{validated_exc}\n{self.exc.text}" + return msg + + +class PythonLiteralEval(ParsingException): + def __init__(self, exc, node): + self.exc = exc + self.node = node + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = self.validator_error_message( + f"Error when trying to literal_eval an arg to dbt.ref(), dbt.source(), dbt.config() or dbt.config.get() \n{self.exc}\n" + "https://docs.python.org/3/library/ast.html#ast.literal_eval\n" + "In dbt python model, `dbt.ref`, `dbt.source`, `dbt.config`, `dbt.config.get` function args only support Python literal structures" + ) + + return msg + + +class InvalidModelConfig(ParsingException): + def __init__(self, exc, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class YamlParseFailure(ParsingException): + def __init__( + self, + path, + key, + yaml_data, + cause, + ): + self.path = path + self.key = key + self.yaml_data = yaml_data + self.cause = cause + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + if isinstance(self.cause, str): + reason = self.cause + elif isinstance(self.cause, ValidationError): + reason = self.validator_error_message(self.cause) + else: + reason = self.cause.msg + msg = f"Invalid {self.key} config given in {self.path} @ {self.key}: {self.yaml_data} - {reason}" + return msg + + +class YamlLoadFailure(ParsingException): + def __init__(self, project_name, path, exc): + self.project_name = project_name + self.path = path + self.exc = exc + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + reason = self.validator_error_message(self.exc) + + msg = f"Error reading {self.project_name}: {self.path} - {reason}" + + return msg + + +class InvalidTestConfig(ParsingException): + def __init__(self, exc, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class InvalidSchemaConfig(ParsingException): + def __init__(self, exc, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + +class InvalidSnapshopConfig(ParsingException): + def __init__(self, exc, node): + self.msg = self.validator_error_message(exc) + self.node = node + super().__init__(msg=self.msg) + + class SameKeyNested(CompilationException): def __init__(self): msg = "Test cannot have the same key at the top-level and in config" @@ -1176,7 +1292,7 @@ def __init__(self, patch_1, patch_2): def get_message(self) -> str: name = f"{self.patch_1.overrides}.{self.patch_1.name}" - fix = _fix_dupe_msg( + fix = self._fix_dupe_msg( self.patch_1.path, self.patch_2.path, name, @@ -1199,7 +1315,7 @@ def __init__(self, patch_1, existing_patch_path): def get_message(self) -> str: package_name = self.patch_1.package_name name = self.patch_1.name - fix = _fix_dupe_msg( + fix = self._fix_dupe_msg( self.patch_1.original_file_path, self.existing_patch_path, name, "macros" ) msg = ( @@ -1227,6 +1343,57 @@ def get_message(self) -> str: return msg +# Postgres Exceptions + + +class UnexpectedDbReference(NotImplementedException): + def __init__(self, adapter, database, expected): + self.adapter = adapter + self.database = database + self.expected = expected + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"Cross-db references not allowed in {self.adapter} ({self.database} vs {self.expected})" + return msg + + +class CrossDbReferenceProhibited(CompilationException): + def __init__(self, adapter, exc_msg): + self.adapter = adapter + self.exc_msg = exc_msg + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = f"Cross-db references not allowed in adapter {self.adapter}: Got {self.exc_msg}" + return msg + + +class IndexConfigNotDict(CompilationException): + def __init__(self, raw_index): + self.raw_index = raw_index + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + msg = ( + f"Invalid index config:\n" + f" Got: {self.raw_index}\n" + f' Expected a dictionary with at minimum a "columns" key' + ) + return msg + + +class InvalidIndexConfig(CompilationException): + def __init__(self, exc): + self.exc = exc + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + validator_msg = self.validator_error_message(self.exc) + msg = f"Could not parse index config: {validator_msg}" + return msg + + # adapters exceptions class InvalidMacroResult(CompilationException): def __init__(self, freshness_macro_name, table): @@ -1235,7 +1402,7 @@ def __init__(self, freshness_macro_name, table): super().__init__(msg=self.get_message()) def get_message(self) -> str: - msg = 'Got an invalid result from "{self.freshness_macro_name}" macro: {[tuple(r) for r in self.table]}' + msg = f'Got an invalid result from "{self.freshness_macro_name}" macro: {[tuple(r) for r in self.table]}' return msg @@ -1474,6 +1641,34 @@ def __init__(self, package_name): # config level exceptions +class ProfileConfigInvalid(DbtProfileError): + def __init__(self, exc): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class ProjectContractInvalid(DbtProjectError): + def __init__(self, exc): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class ProjectContractBroken(DbtProjectError): + def __init__(self, exc): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + +class ConfigContractBroken(DbtProjectError): + def __init__(self, exc): + self.exc = exc + msg = self.validator_error_message(self.exc) + super().__init__(msg=msg) + + class NonUniquePackageName(CompilationException): def __init__(self, project_name): self.project_name = project_name @@ -1765,7 +1960,7 @@ def __init__(self, patch_1, existing_patch_path): def get_message(self) -> str: name = self.patch_1.name - fix = _fix_dupe_msg( + fix = self._fix_dupe_msg( self.patch_1.original_file_path, self.existing_patch_path, name, @@ -1921,9 +2116,8 @@ def raise_ambiguous_catalog_match(unique_id, match_1, match_2) -> NoReturn: raise AmbiguousCatalogMatch(unique_id, match_1, match_2) -# TODO: this should be improved to not format message here def raise_cache_inconsistent(message) -> NoReturn: - raise InternalException("Cache inconsistency detected: {}".format(message)) + raise CacheInconsistency(message) def raise_dataclass_not_dict(obj) -> NoReturn: @@ -1961,7 +2155,7 @@ def raise_invalid_property_yml_version(path, issue) -> NoReturn: # TODO: this should be improved to not format message here def raise_not_implemented(msg) -> NoReturn: - raise NotImplementedException("ERROR: {}".format(msg)) + raise NotImplementedException(msg) def relation_wrong_type(relation, expected_type, model=None) -> NoReturn: diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index 21bc74fbfc5..9c245214d83 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -18,7 +18,7 @@ from dbt.contracts.graph.manifest import Manifest from dbt.contracts.graph.nodes import ManifestNode, BaseNode from dbt.contracts.graph.unparsed import UnparsedNode, Docs -from dbt.exceptions import ParsingException, validator_error_message, InternalException +from dbt.exceptions import InternalException, InvalidConfigUpdate, InvalidDictParse from dbt import hooks from dbt.node_types import NodeType, ModelLanguage from dbt.parser.search import FileBlock @@ -216,7 +216,6 @@ def _create_parsetime_node( try: return self.parse_from_dict(dct, validate=True) except ValidationError as exc: - msg = validator_error_message(exc) # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( @@ -225,7 +224,7 @@ def _create_parsetime_node( original_file_path=block.path.original_file_path, raw_code=block.contents, ) - raise ParsingException(msg, node=node) + raise InvalidDictParse(exc, node=node) def _context_for(self, parsed_node: IntermediateNode, config: ContextConfig) -> Dict[str, Any]: return generate_parser_model_context(parsed_node, self.root_project, self.manifest, config) @@ -364,8 +363,7 @@ def render_update(self, node: IntermediateNode, config: ContextConfig) -> None: self.update_parsed_node_config(node, config, context=context) except ValidationError as exc: # we got a ValidationError - probably bad types in config() - msg = validator_error_message(exc) - raise ParsingException(msg, node=node) from exc + raise InvalidConfigUpdate(exc, node=node) from exc def add_result_node(self, block: FileBlock, node: ManifestNode): if node.config.enabled: diff --git a/core/dbt/parser/models.py b/core/dbt/parser/models.py index 8303e2f9c52..41ddfe0a5f3 100644 --- a/core/dbt/parser/models.py +++ b/core/dbt/parser/models.py @@ -29,7 +29,13 @@ # New for Python models :p import ast from dbt.dataclass_schema import ValidationError -from dbt.exceptions import ParsingException, validator_error_message, UndefinedMacroException +from dbt.exceptions import ( + InvalidModelConfig, + ParsingException, + PythonLiteralEval, + PythonParsingException, + UndefinedMacroException, +) dbt_function_key_words = set(["ref", "source", "config", "get"]) @@ -91,12 +97,7 @@ def _safe_eval(self, node): try: return ast.literal_eval(node) except (SyntaxError, ValueError, TypeError, MemoryError, RecursionError) as exc: - msg = validator_error_message( - f"Error when trying to literal_eval an arg to dbt.ref(), dbt.source(), dbt.config() or dbt.config.get() \n{exc}\n" - "https://docs.python.org/3/library/ast.html#ast.literal_eval\n" - "In dbt python model, `dbt.ref`, `dbt.source`, `dbt.config`, `dbt.config.get` function args only support Python literal structures" - ) - raise ParsingException(msg, node=self.dbt_node) from exc + raise PythonLiteralEval(exc, node=self.dbt_node) from exc def _get_call_literals(self, node): # List of literals @@ -199,8 +200,7 @@ def parse_python_model(self, node, config, context): try: tree = ast.parse(node.raw_code, filename=node.original_file_path) except SyntaxError as exc: - msg = validator_error_message(exc) - raise ParsingException(f"{msg}\n{exc.text}", node=node) from exc + raise PythonParsingException(exc, node=node) from exc # We are doing a validator and a parser because visit_FunctionDef in parser # would actually make the parser not doing the visit_Calls any more @@ -251,8 +251,7 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None: except ValidationError as exc: # we got a ValidationError - probably bad types in config() - msg = validator_error_message(exc) - raise ParsingException(msg, node=node) from exc + raise InvalidModelConfig(exc, node=node) from exc return elif not flags.STATIC_PARSER: diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index 466847edf3a..fa0c999713b 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -56,12 +56,15 @@ DuplicateSourcePatchName, JSONValidationException, InternalException, + InvalidSchemaConfig, + InvalidTestConfig, ParsingException, PropertyYMLInvalidTag, PropertyYMLMissingVersion, PropertyYMLVersionNotInt, ValidationException, - validator_error_message, + YamlLoadFailure, + YamlParseFailure, ) from dbt.events.functions import warn_or_error from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroPatchNotFound @@ -93,34 +96,13 @@ ) -def error_context( - path: str, - key: str, - data: Any, - cause: Union[str, ValidationException, JSONValidationException], -) -> str: - """Provide contextual information about an error while parsing""" - if isinstance(cause, str): - reason = cause - elif isinstance(cause, ValidationError): - reason = validator_error_message(cause) - else: - reason = cause.msg - return "Invalid {key} config given in {path} @ {key}: {data} - {reason}".format( - key=key, path=path, data=data, reason=reason - ) - - def yaml_from_file(source_file: SchemaSourceFile) -> Dict[str, Any]: """If loading the yaml fails, raise an exception.""" path = source_file.path.relative_path try: return load_yaml_text(source_file.contents, source_file.path) except ValidationException as e: - reason = validator_error_message(e) - raise ParsingException( - "Error reading {}: {} - {}".format(source_file.project_name, path, reason) - ) + raise YamlLoadFailure(source_file.project_name, path, e) class ParserRef: @@ -264,7 +246,6 @@ def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List GenericTestNode.validate(dct) return GenericTestNode.from_dict(dct) except ValidationError as exc: - msg = validator_error_message(exc) # this is a bit silly, but build an UnparsedNode just for error # message reasons node = self._create_error_node( @@ -273,7 +254,7 @@ def get_hashable_md(data: Union[str, int, float, List, Dict]) -> Union[str, List original_file_path=target.original_file_path, raw_code=raw_code, ) - raise ParsingException(msg, node=node) from exc + raise InvalidTestConfig(exc, node) # lots of time spent in this method def _parse_generic_test( @@ -415,8 +396,7 @@ def render_test_update(self, node, config, builder, schema_file_id): # env_vars should have been updated in the context env_var method except ValidationError as exc: # we got a ValidationError - probably bad types in config() - msg = validator_error_message(exc) - raise ParsingException(msg, node=node) from exc + raise InvalidSchemaConfig(exc, node=node) from exc def parse_node(self, block: GenericTestBlock) -> GenericTestNode: """In schema parsing, we rewrite most of the part of parse_node that @@ -626,8 +606,7 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: # check that entry is a dict and that all dict values # are strings if coerce_dict_str(entry) is None: - msg = error_context(path, self.key, data, "expected a dict with string keys") - raise ParsingException(msg) + raise YamlParseFailure(path, self.key, data, "expected a dict with string keys") if "name" not in entry: raise ParsingException("Entry did not contain a name") @@ -674,8 +653,7 @@ def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T: cls.validate(data) return cls.from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseFailure(path, self.key, data, exc) # The other parse method returns TestBlocks. This one doesn't. # This takes the yaml dictionaries in 'sources' keys and uses them @@ -800,8 +778,7 @@ def get_unparsed_target(self) -> Iterable[NonSourceTarget]: self.normalize_docs_attribute(data, path) node = self._target_type().from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseFailure(path, self.key, data, exc) else: yield node @@ -1084,8 +1061,7 @@ def parse(self): UnparsedExposure.validate(data) unparsed = UnparsedExposure.from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(self.yaml.path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseFailure(self.yaml.path, self.key, data, exc) self.parse_exposure(unparsed) @@ -1202,6 +1178,5 @@ def parse(self): unparsed = UnparsedMetric.from_dict(data) except (ValidationError, JSONValidationException) as exc: - msg = error_context(self.yaml.path, self.key, data, exc) - raise ParsingException(msg) from exc + raise YamlParseFailure(self.yaml.path, self.key, data, exc) self.parse_metric(unparsed) diff --git a/core/dbt/parser/snapshots.py b/core/dbt/parser/snapshots.py index 7fc46d1a05a..dffc7d90641 100644 --- a/core/dbt/parser/snapshots.py +++ b/core/dbt/parser/snapshots.py @@ -4,7 +4,7 @@ from dbt.dataclass_schema import ValidationError from dbt.contracts.graph.nodes import IntermediateSnapshotNode, SnapshotNode -from dbt.exceptions import ParsingException, validator_error_message +from dbt.exceptions import InvalidSnapshopConfig from dbt.node_types import NodeType from dbt.parser.base import SQLParser from dbt.parser.search import BlockContents, BlockSearcher, FileBlock @@ -68,7 +68,7 @@ def transform(self, node: IntermediateSnapshotNode) -> SnapshotNode: self.set_snapshot_attributes(parsed_node) return parsed_node except ValidationError as exc: - raise ParsingException(validator_error_message(exc), node) + raise InvalidSnapshopConfig(exc, node) def parse_file(self, file_block: FileBlock) -> None: blocks = BlockSearcher( diff --git a/plugins/postgres/dbt/adapters/postgres/impl.py b/plugins/postgres/dbt/adapters/postgres/impl.py index 3664e8d2a51..78b86234eae 100644 --- a/plugins/postgres/dbt/adapters/postgres/impl.py +++ b/plugins/postgres/dbt/adapters/postgres/impl.py @@ -8,7 +8,13 @@ from dbt.adapters.postgres import PostgresColumn from dbt.adapters.postgres import PostgresRelation from dbt.dataclass_schema import dbtClassMixin, ValidationError -import dbt.exceptions +from dbt.exceptions import ( + CrossDbReferenceProhibited, + IndexConfigNotDict, + InvalidIndexConfig, + RuntimeException, + UnexpectedDbReference, +) import dbt.utils @@ -40,14 +46,9 @@ def parse(cls, raw_index) -> Optional["PostgresIndexConfig"]: cls.validate(raw_index) return cls.from_dict(raw_index) except ValidationError as exc: - msg = dbt.exceptions.validator_error_message(exc) - dbt.exceptions.raise_compiler_error(f"Could not parse index config: {msg}") + raise InvalidIndexConfig(exc) except TypeError: - dbt.exceptions.raise_compiler_error( - f"Invalid index config:\n" - f" Got: {raw_index}\n" - f' Expected a dictionary with at minimum a "columns" key' - ) + raise IndexConfigNotDict(raw_index) @dataclass @@ -73,11 +74,7 @@ def verify_database(self, database): database = database.strip('"') expected = self.config.credentials.database if database.lower() != expected.lower(): - raise dbt.exceptions.NotImplementedException( - "Cross-db references not allowed in {} ({} vs {})".format( - self.type(), database, expected - ) - ) + raise UnexpectedDbReference(self.type(), database, expected) # return an empty string on success so macros can call this return "" @@ -110,12 +107,8 @@ def _get_catalog_schemas(self, manifest): schemas = super()._get_catalog_schemas(manifest) try: return schemas.flatten() - except dbt.exceptions.RuntimeException as exc: - dbt.exceptions.raise_compiler_error( - "Cross-db references not allowed in adapter {}: Got {}".format( - self.type(), exc.msg - ) - ) + except RuntimeException as exc: + raise CrossDbReferenceProhibited(self.type(), exc.msg) def _link_cached_relations(self, manifest): schemas: Set[str] = set() From d397c3859072753c14ea916eee40b119f96024d3 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 12 Dec 2022 20:03:20 -0600 Subject: [PATCH 17/24] standardize to msg --- core/dbt/exceptions.py | 56 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index fab60133983..8a0cd93263d 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -199,13 +199,13 @@ class RPCKilledException(RuntimeException): def __init__(self, signum): self.signum = signum - self.message = "RPC process killed by signal {}".format(self.signum) - super().__init__(self.message) + self.msg = f"RPC process killed by signal {self.signum}" + super().__init__(self.msg) def data(self): return { "signum": self.signum, - "message": self.message, + "message": self.msg, } @@ -227,11 +227,11 @@ class RPCLoadException(RuntimeException): def __init__(self, cause): self.cause = cause - self.message = "{}: {}".format(self.MESSAGE, self.cause["message"]) - super().__init__(self.message) + self.msg = f'{self.MESSAGE}: {self.cause["message"]}' + super().__init__(self.msg) def data(self): - return {"cause": self.cause, "message": self.message} + return {"cause": self.cause, "message": self.msg} class DatabaseException(RuntimeException): @@ -371,9 +371,9 @@ class DbtConfigError(RuntimeException): CODE = 10007 MESSAGE = "DBT Configuration Error" - def __init__(self, message, project=None, result_type="invalid_project", path=None): + def __init__(self, msg, project=None, result_type="invalid_project", path=None): self.project = project - super().__init__(message) + super().__init__(msg) self.result_type = result_type self.path = path @@ -389,8 +389,8 @@ class FailFastException(RuntimeException): CODE = 10013 MESSAGE = "FailFast Error" - def __init__(self, message, result=None, node=None): - super().__init__(msg=message, node=node) + def __init__(self, msg, result=None, node=None): + super().__init__(msg=msg, node=node) self.result = result @property @@ -424,10 +424,10 @@ class VersionsNotCompatibleException(SemverException): class NotImplementedException(Exception): - def __init__(self, message): - self.message = message - self.msg = f"ERROR: {self.message}" - super().__init__(msg=self.msg) + def __init__(self, msg): + self.msg = msg + self.formatted_msg = f"ERROR: {self.msg}" + super().__init__(msg=self.formatted_msg) class FailedToConnectException(DatabaseException): @@ -435,12 +435,12 @@ class FailedToConnectException(DatabaseException): class CommandError(RuntimeException): - def __init__(self, cwd, cmd, message="Error running command"): + def __init__(self, cwd, cmd, msg="Error running command"): cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd) - super().__init__(message) + super().__init__(msg) self.cwd = cwd self.cmd = cmd_scrubbed - self.args = (cwd, cmd_scrubbed, message) + self.args = (cwd, cmd_scrubbed, msg) def __str__(self): if len(self.cmd) == 0: @@ -449,25 +449,25 @@ def __str__(self): class ExecutableError(CommandError): - def __init__(self, cwd, cmd, message): - super().__init__(cwd, cmd, message) + def __init__(self, cwd, cmd, msg): + super().__init__(cwd, cmd, msg) class WorkingDirectoryError(CommandError): - def __init__(self, cwd, cmd, message): - super().__init__(cwd, cmd, message) + def __init__(self, cwd, cmd, msg): + super().__init__(cwd, cmd, msg) def __str__(self): return '{}: "{}"'.format(self.msg, self.cwd) class CommandResultError(CommandError): - def __init__(self, cwd, cmd, returncode, stdout, stderr, message="Got a non-zero returncode"): - super().__init__(cwd, cmd, message) + def __init__(self, cwd, cmd, returncode, stdout, stderr, msg="Got a non-zero returncode"): + super().__init__(cwd, cmd, msg) self.returncode = returncode self.stdout = scrub_secrets(stdout.decode("utf-8"), env_secrets()) self.stderr = scrub_secrets(stderr.decode("utf-8"), env_secrets()) - self.args = (cwd, self.cmd, returncode, self.stdout, self.stderr, message) + self.args = (cwd, self.cmd, returncode, self.stdout, self.stderr, msg) def __str__(self): return "{} running: {}".format(self.msg, self.cmd) @@ -1857,10 +1857,10 @@ def get_message(self) -> str: class CacheInconsistency(InternalException): - def __init__(self, message): - self.message = message - msg = f"Cache inconsistency detected: {self.message}" - super().__init__(msg) + def __init__(self, msg): + self.msg = msg + formatted_msg = f"Cache inconsistency detected: {self.msg}" + super().__init__(msg=formatted_msg) class NewNameAlreadyInCache(CacheInconsistency): From 1925c113829bd26b492cb03d228f35c152777ec5 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 12 Dec 2022 20:04:22 -0600 Subject: [PATCH 18/24] remove some TODOs --- core/dbt/context/exceptions_jinja.py | 1 - core/dbt/exceptions.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/dbt/context/exceptions_jinja.py b/core/dbt/context/exceptions_jinja.py index f167b5ee8da..5663b4701e0 100644 --- a/core/dbt/context/exceptions_jinja.py +++ b/core/dbt/context/exceptions_jinja.py @@ -87,7 +87,6 @@ def raise_invalid_property_yml_version(path, issue) -> NoReturn: raise InvalidPropertyYML(path, issue) -# TODO: this should be improved to not format message here def raise_not_implemented(msg) -> NoReturn: raise NotImplementedException(msg) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 8a0cd93263d..75764b78d35 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1923,7 +1923,7 @@ def __init__(self): # this is part of the context and also raised in dbt.contracts.relation.py class DataclassNotDict(CompilationException): def __init__(self, obj): - self.obj = obj # TODO: what kind of obj is this? + self.obj = obj super().__init__(msg=self.get_message()) def get_message(self) -> str: @@ -2153,7 +2153,6 @@ def raise_invalid_property_yml_version(path, issue) -> NoReturn: raise InvalidPropertyYML(path, issue) -# TODO: this should be improved to not format message here def raise_not_implemented(msg) -> NoReturn: raise NotImplementedException(msg) From c5f9d303500c975e306d63e5be88b733bb68cd5d Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 13 Dec 2022 13:03:19 -0600 Subject: [PATCH 19/24] fix test param and check the rest --- core/dbt/task/runnable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 226005497e4..14005203296 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -243,7 +243,7 @@ def call_runner(self, runner): if result.status in (NodeStatus.Error, NodeStatus.Fail) and fail_fast: self._raise_next_tick = FailFastException( - message="Failing early due to test failure or runtime error", + msg="Failing early due to test failure or runtime error", result=result, node=getattr(result, "node", None), ) From c0406c87ac72910c7f76cbb38caed25809786026 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 13 Dec 2022 16:13:54 -0600 Subject: [PATCH 20/24] add comment, move exceptions --- core/dbt/exceptions.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 75764b78d35..040d3323984 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -14,6 +14,17 @@ import dbt.dataclass_schema +class MacroReturn(builtins.BaseException): + """ + Hack of all hacks + This is not actually an exception. + It's how we return a value from a macro. + """ + + def __init__(self, value): + self.value = value + + class Exception(builtins.Exception): CODE = -32000 MESSAGE = "Server Error" @@ -26,15 +37,6 @@ def data(self): } -class MacroReturn(builtins.BaseException): - """ - Hack of all hacks - """ - - def __init__(self, value): - self.value = value - - class InternalException(Exception): def __init__(self, msg): self.stack = [] From b3bf0be4d4fbc3459541a1afaf9468361c90af38 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 14 Dec 2022 20:09:04 -0600 Subject: [PATCH 21/24] add types --- core/dbt/context/providers.py | 2 +- core/dbt/exceptions.py | 315 +++++++++++++++++----------------- 2 files changed, 162 insertions(+), 155 deletions(-) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index b6567ba8def..2e7af0a79f2 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -794,7 +794,7 @@ def load_agate_table(self) -> agate.Table: try: table = agate_helper.from_csv(path, text_columns=column_types) except ValueError as e: - raise LoadAgateTableValueError(str(e), node=self.model) + raise LoadAgateTableValueError(e, node=self.model) table.original_abspath = os.path.abspath(path) return table diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 040d3323984..0683098d58a 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,8 +1,9 @@ import builtins import json import re -from typing import Any, Mapping, NoReturn, Optional +from typing import Any, Dict, List, Mapping, NoReturn, Optional +# from dbt.contracts.graph import ManifestNode # or ParsedNode? from dbt.dataclass_schema import ValidationError from dbt.events.functions import warn_or_error from dbt.events.helpers import env_secrets, scrub_secrets @@ -38,8 +39,8 @@ def data(self): class InternalException(Exception): - def __init__(self, msg): - self.stack = [] + def __init__(self, msg: str): + self.stack: List = [] self.msg = scrub_secrets(msg, env_secrets()) @property @@ -82,8 +83,8 @@ class RuntimeException(RuntimeError, Exception): CODE = 10001 MESSAGE = "Runtime error" - def __init__(self, msg, node=None): - self.stack = [] + def __init__(self, msg: str, node=None): + self.stack: List = [] self.node = node self.msg = scrub_secrets(msg, env_secrets()) @@ -102,14 +103,14 @@ def node_to_string(self, node): return "" if not hasattr(node, "name"): # we probably failed to parse a block, so we can't know the name - return "{} ({})".format(node.resource_type, node.original_file_path) + return f"{node.resource_type} ({node.original_file_path})" if hasattr(node, "contents"): # handle FileBlocks. They aren't really nodes but we want to render # out the path we know at least. This indicates an error during # block parsing. - return "{}".format(node.path.original_file_path) - return "{} {} ({})".format(node.resource_type, node.name, node.original_file_path) + return f"{node.path.original_file_path}" + return f"{node.resource_type} {node.name} ({node.original_file_path})" def process_stack(self): lines = [] @@ -126,24 +127,24 @@ def process_stack(self): msg = "in" first = False - lines.append("> {} {}".format(msg, self.node_to_string(item))) + lines.append(f"> {msg} {self.node_to_string(item)}") return lines - def validator_error_message(self, exc): + def validator_error_message(self, exc: builtins.Exception): """Given a dbt.dataclass_schema.ValidationError (which is basically a jsonschema.ValidationError), return the relevant parts as a string """ if not isinstance(exc, dbt.dataclass_schema.ValidationError): return str(exc) path = "[%s]" % "][".join(map(repr, exc.relative_path)) - return "at path {}: {}".format(path, exc.message) + return f"at path {path}: {exc.message}" - def __str__(self, prefix="! "): + def __str__(self, prefix: str = "! "): node_string = "" if self.node is not None: - node_string = " in {}".format(self.node_to_string(self.node)) + node_string = f" in {self.node_to_string(self.node)}" if hasattr(self.msg, "split"): split_msg = self.msg.split("\n") @@ -180,7 +181,7 @@ class RPCTimeoutException(RuntimeException): CODE = 10008 MESSAGE = "RPC timeout error" - def __init__(self, timeout): + def __init__(self, timeout: Optional[float]): super().__init__(self.MESSAGE) self.timeout = timeout @@ -189,7 +190,7 @@ def data(self): result.update( { "timeout": self.timeout, - "message": "RPC timed out after {}s".format(self.timeout), + "message": f"RPC timed out after {self.timeout}s", } ) return result @@ -199,7 +200,7 @@ class RPCKilledException(RuntimeException): CODE = 10009 MESSAGE = "RPC process killed" - def __init__(self, signum): + def __init__(self, signum: int): self.signum = signum self.msg = f"RPC process killed by signal {self.signum}" super().__init__(self.msg) @@ -215,7 +216,7 @@ class RPCCompiling(RuntimeException): CODE = 10010 MESSAGE = 'RPC server is compiling the project, call the "status" method for' " compile status" - def __init__(self, msg=None, node=None): + def __init__(self, msg: str = None, node=None): if msg is None: msg = "compile in progress" super().__init__(msg, node) @@ -227,7 +228,7 @@ class RPCLoadException(RuntimeException): 'RPC server failed to compile project, call the "status" method for' " compile status" ) - def __init__(self, cause): + def __init__(self, cause: Dict[str, Any]): self.cause = cause self.msg = f'{self.MESSAGE}: {self.cause["message"]}' super().__init__(self.msg) @@ -244,7 +245,7 @@ def process_stack(self): lines = [] if hasattr(self.node, "build_path") and self.node.build_path: - lines.append("compiled Code at {}".format(self.node.build_path)) + lines.append(f"compiled Code at {self.node.build_path}") return lines + RuntimeException.process_stack(self) @@ -291,14 +292,13 @@ def type(self): return "Parsing" +# TODO: this isn't raised in the core codebase. Is it raised elsewhere? class JSONValidationException(ValidationException): def __init__(self, typename, errors): self.typename = typename self.errors = errors self.errors_message = ", ".join(errors) - msg = 'Invalid arguments passed to "{}" instance: {}'.format( - self.typename, self.errors_message - ) + msg = f'Invalid arguments passed to "{self.typename}" instance: {self.errors_message}' super().__init__(msg) def __reduce__(self): @@ -339,7 +339,7 @@ class JinjaRenderingException(CompilationException): class UndefinedMacroException(CompilationException): - def __str__(self, prefix="! ") -> str: + def __str__(self, prefix: str = "! ") -> str: msg = super().__str__(prefix) return ( f"{msg}. This can happen when calling a macro that does " @@ -356,7 +356,7 @@ def __init__(self, task_id): self.task_id = task_id def __str__(self): - return "{}: {}".format(self.MESSAGE, self.task_id) + return f"{self.MESSAGE}: {self.task_id}" class AliasException(ValidationException): @@ -373,7 +373,7 @@ class DbtConfigError(RuntimeException): CODE = 10007 MESSAGE = "DBT Configuration Error" - def __init__(self, msg, project=None, result_type="invalid_project", path=None): + def __init__(self, msg: str, project=None, result_type="invalid_project", path=None): self.project = project super().__init__(msg) self.result_type = result_type @@ -391,7 +391,7 @@ class FailFastException(RuntimeException): CODE = 10013 MESSAGE = "FailFast Error" - def __init__(self, msg, result=None, node=None): + def __init__(self, msg: str, result=None, node=None): super().__init__(msg=msg, node=node) self.result = result @@ -413,7 +413,7 @@ class DbtProfileError(DbtConfigError): class SemverException(Exception): - def __init__(self, msg=None): + def __init__(self, msg: str = None): self.msg = msg if msg is not None: super().__init__(msg) @@ -426,10 +426,10 @@ class VersionsNotCompatibleException(SemverException): class NotImplementedException(Exception): - def __init__(self, msg): + def __init__(self, msg: str): self.msg = msg self.formatted_msg = f"ERROR: {self.msg}" - super().__init__(msg=self.formatted_msg) + super().__init__(self.formatted_msg) class FailedToConnectException(DatabaseException): @@ -437,7 +437,7 @@ class FailedToConnectException(DatabaseException): class CommandError(RuntimeException): - def __init__(self, cwd, cmd, msg="Error running command"): + def __init__(self, cwd: str, cmd: List[str], msg: str = "Error running command"): cmd_scrubbed = list(scrub_secrets(cmd_txt, env_secrets()) for cmd_txt in cmd) super().__init__(msg) self.cwd = cwd @@ -446,25 +446,33 @@ def __init__(self, cwd, cmd, msg="Error running command"): def __str__(self): if len(self.cmd) == 0: - return "{}: No arguments given".format(self.msg) - return '{}: "{}"'.format(self.msg, self.cmd[0]) + return f"{self.msg}: No arguments given" + return f'{self.msg}: "{self.cmd[0]}"' class ExecutableError(CommandError): - def __init__(self, cwd, cmd, msg): + def __init__(self, cwd: str, cmd: List[str], msg: str): super().__init__(cwd, cmd, msg) class WorkingDirectoryError(CommandError): - def __init__(self, cwd, cmd, msg): + def __init__(self, cwd: str, cmd: List[str], msg: str): super().__init__(cwd, cmd, msg) def __str__(self): - return '{}: "{}"'.format(self.msg, self.cwd) + return f'{self.msg}: "{self.cwd}"' class CommandResultError(CommandError): - def __init__(self, cwd, cmd, returncode, stdout, stderr, msg="Got a non-zero returncode"): + def __init__( + self, + cwd: str, + cmd: List[str], + returncode: str, + stdout: bytes, + stderr: bytes, + msg: str = "Got a non-zero returncode", + ): super().__init__(cwd, cmd, msg) self.returncode = returncode self.stdout = scrub_secrets(stdout.decode("utf-8"), env_secrets()) @@ -472,17 +480,15 @@ def __init__(self, cwd, cmd, returncode, stdout, stderr, msg="Got a non-zero ret self.args = (cwd, self.cmd, returncode, self.stdout, self.stderr, msg) def __str__(self): - return "{} running: {}".format(self.msg, self.cmd) + return f"{self.msg} running: {self.cmd}" class InvalidConnectionException(RuntimeException): - def __init__(self, thread_id, known, node=None): + def __init__(self, thread_id, known: List): self.thread_id = thread_id self.known = known super().__init__( - msg="connection never acquired for thread {}, have {}".format( - self.thread_id, self.known - ) + msg="connection never acquired for thread {self.thread_id}, have {self.known}" ) @@ -507,7 +513,7 @@ class ConnectionException(Exception): # event level exception class EventCompilationException(CompilationException): - def __init__(self, msg, node): + def __init__(self, msg: str, node): self.msg = scrub_secrets(msg, env_secrets()) self.node = node super().__init__(msg=self.msg) @@ -515,7 +521,7 @@ def __init__(self, msg, node): # compilation level exceptions class GraphDependencyNotFound(CompilationException): - def __init__(self, node, dependency): + def __init__(self, node, dependency: str): self.node = node self.dependency = dependency super().__init__(msg=self.get_message()) @@ -543,7 +549,7 @@ def __init__(self, node): class UndefinedCompilation(CompilationException): - def __init__(self, name, node): + def __init__(self, name: str, node): self.name = name self.node = node self.msg = f"{self.name} is undefined" @@ -551,14 +557,14 @@ def __init__(self, name, node): class CaughtMacroExceptionWithNode(CompilationException): - def __init__(self, exc, node): + def __init__(self, exc: Exception, node): self.exc = exc self.node = node super().__init__(msg=str(exc)) class CaughtMacroException(CompilationException): - def __init__(self, exc): + def __init__(self, exc: Exception): self.exc = exc super().__init__(msg=str(exc)) @@ -577,7 +583,7 @@ def get_message(self) -> str: class MissingControlFlowStartTag(CompilationException): - def __init__(self, tag, expected_tag, tag_parser): + def __init__(self, tag, expected_tag: str, tag_parser): self.tag = tag self.expected_tag = expected_tag self.tag_parser = tag_parser @@ -593,7 +599,7 @@ def get_message(self) -> str: class UnexpectedControlFlowEndTag(CompilationException): - def __init__(self, tag, expected_tag, tag_parser): + def __init__(self, tag, expected_tag: str, tag_parser): self.tag = tag self.expected_tag = expected_tag self.tag_parser = tag_parser @@ -609,7 +615,7 @@ def get_message(self) -> str: class UnexpectedMacroEOF(CompilationException): - def __init__(self, expected_name, actual_name): + def __init__(self, expected_name: str, actual_name: str): self.expected_name = expected_name self.actual_name = actual_name super().__init__(msg=self.get_message()) @@ -620,7 +626,7 @@ def get_message(self) -> str: class MacroNamespaceNotString(CompilationException): - def __init__(self, kwarg_type): + def __init__(self, kwarg_type: Any): self.kwarg_type = kwarg_type super().__init__(msg=self.get_message()) @@ -663,7 +669,7 @@ def get_message(self) -> str: class MissingCloseTag(CompilationException): - def __init__(self, block_type_name, linecount): + def __init__(self, block_type_name: str, linecount: int): self.block_type_name = block_type_name self.linecount = linecount super().__init__(msg=self.get_message()) @@ -674,7 +680,7 @@ def get_message(self) -> str: class GitCloningProblem(RuntimeException): - def __init__(self, repo): + def __init__(self, repo: str): self.repo = scrub_secrets(repo, env_secrets()) super().__init__(msg=self.get_message()) @@ -687,7 +693,7 @@ def get_message(self) -> str: class GitCloningError(InternalException): - def __init__(self, repo, revision, error): + def __init__(self, repo: str, revision: str, error: CommandResultError): self.repo = repo self.revision = revision self.error = error @@ -698,7 +704,7 @@ def get_message(self) -> str: if "usage: git" in stderr: stderr = stderr.split("\nusage: git")[0] if re.match("fatal: destination path '(.+)' already exists", stderr): - self.error.cmd = scrub_secrets(str(self.error.cmd), env_secrets()) + self.error.cmd = list(scrub_secrets(str(self.error.cmd), env_secrets())) raise self.error msg = f"Error checking out spec='{self.revision}' for repo {self.repo}\n{stderr}" @@ -706,7 +712,7 @@ def get_message(self) -> str: class GitCheckoutError(InternalException): - def __init__(self, repo, revision, error): + def __init__(self, repo: str, revision: str, error: CommandResultError): self.repo = repo self.revision = revision self.stderr = error.stderr.strip() @@ -718,7 +724,7 @@ def get_message(self) -> str: class InvalidMaterializationArg(CompilationException): - def __init__(self, name, argument): + def __init__(self, name: str, argument: str): self.name = name self.argument = argument super().__init__(msg=self.get_message()) @@ -746,7 +752,7 @@ def get_message(self) -> str: class LoadAgateTableValueError(CompilationException): - def __init__(self, exc, node): + def __init__(self, exc: ValueError, node): self.exc = exc self.node = node msg = str(self.exc) @@ -769,7 +775,7 @@ def __init__(self, node): class PackageNotInDeps(CompilationException): - def __init__(self, package_name, node): + def __init__(self, package_name: str, node): self.package_name = package_name self.node = node msg = f"Node package named {self.package_name} not found!" @@ -777,7 +783,7 @@ def __init__(self, package_name, node): class OperationsCannotRefEphemeralNodes(CompilationException): - def __init__(self, target_name, node): + def __init__(self, target_name: str, node): self.target_name = target_name self.node = node msg = f"Operations can not ref() ephemeral nodes, but {target_name} is ephemeral" @@ -785,7 +791,7 @@ def __init__(self, target_name, node): class InvalidPersistDocsValueType(CompilationException): - def __init__(self, persist_docs): + def __init__(self, persist_docs: Any): self.persist_docs = persist_docs msg = ( "Invalid value provided for 'persist_docs'. Expected dict " @@ -802,7 +808,7 @@ def __init__(self, node): class ConflictingConfigKeys(CompilationException): - def __init__(self, oldkey, newkey, node): + def __init__(self, oldkey: str, newkey: str, node): self.oldkey = oldkey self.newkey = newkey self.node = node @@ -819,7 +825,7 @@ def __init__(self, args, node): class RequiredVarNotFound(CompilationException): - def __init__(self, var_name, merged, node): + def __init__(self, var_name: str, merged: Dict, node): self.var_name = var_name self.merged = merged self.node = node @@ -839,14 +845,14 @@ def get_message(self) -> str: class PackageNotFoundForMacro(CompilationException): - def __init__(self, package_name): + def __init__(self, package_name: str): self.package_name = package_name msg = f"Could not find package '{self.package_name}'" super().__init__(msg=msg) class DisallowSecretEnvVar(ParsingException): - def __init__(self, env_var_name): + def __init__(self, env_var_name: str): self.env_var_name = env_var_name super().__init__(msg=self.get_message()) @@ -859,7 +865,9 @@ def get_message(self) -> str: class InvalidMacroArgType(CompilationException): - def __init__(self, method_name, arg_name, got_value, expected_type, version): + def __init__( + self, method_name: str, arg_name: str, got_value: Any, expected_type: str, version: str + ): self.method_name = method_name self.arg_name = arg_name self.got_value = got_value @@ -878,7 +886,7 @@ def get_message(self) -> str: class InvalidBoolean(CompilationException): - def __init__(self, return_value, macro_name): + def __init__(self, return_value: Any, macro_name: str): self.return_value = return_value self.macro_name = macro_name super().__init__(msg=self.get_message()) @@ -969,7 +977,7 @@ def get_message(self) -> str: class MacroInvalidDispatchArg(CompilationException): - def __init__(self, macro_name): + def __init__(self, macro_name: str): self.macro_name = macro_name super().__init__(msg=self.get_message()) @@ -988,7 +996,7 @@ def get_message(self) -> str: class DuplicateMacroName(CompilationException): - def __init__(self, node_1, node_2, namespace): + def __init__(self, node_1, node_2, namespace: str): self.node_1 = node_1 self.node_2 = node_2 self.namespace = namespace @@ -997,9 +1005,7 @@ def __init__(self, node_1, node_2, namespace): def get_message(self) -> str: duped_name = self.node_1.name if self.node_1.package_name != self.node_2.package_name: - extra = ' ("{}" and "{}" are both in the "{}" namespace)'.format( - self.node_1.package_name, self.node_2.package_name, self.namespace - ) + extra = f' ("{self.node_1.package_name}" and "{self.node_2.package_name}" are both in the "{self.namespace}" namespace)' else: extra = "" @@ -1016,7 +1022,7 @@ def get_message(self) -> str: # parser level exceptions class InvalidDictParse(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: ValidationError, node): self.exc = exc self.node = node msg = self.validator_error_message(exc) @@ -1024,7 +1030,7 @@ def __init__(self, exc, node): class InvalidConfigUpdate(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: ValidationError, node): self.exc = exc self.node = node msg = self.validator_error_message(exc) @@ -1032,7 +1038,7 @@ def __init__(self, exc, node): class PythonParsingException(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: SyntaxError, node): self.exc = exc self.node = node super().__init__(msg=self.get_message()) @@ -1044,13 +1050,13 @@ def get_message(self) -> str: class PythonLiteralEval(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: Exception, node): self.exc = exc self.node = node super().__init__(msg=self.get_message()) def get_message(self) -> str: - msg = self.validator_error_message( + msg = ( f"Error when trying to literal_eval an arg to dbt.ref(), dbt.source(), dbt.config() or dbt.config.get() \n{self.exc}\n" "https://docs.python.org/3/library/ast.html#ast.literal_eval\n" "In dbt python model, `dbt.ref`, `dbt.source`, `dbt.config`, `dbt.config.get` function args only support Python literal structures" @@ -1060,7 +1066,7 @@ def get_message(self) -> str: class InvalidModelConfig(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: ValidationError, node): self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) @@ -1069,9 +1075,9 @@ def __init__(self, exc, node): class YamlParseFailure(ParsingException): def __init__( self, - path, - key, - yaml_data, + path: str, + key: str, + yaml_data: List, cause, ): self.path = path @@ -1092,7 +1098,7 @@ def get_message(self) -> str: class YamlLoadFailure(ParsingException): - def __init__(self, project_name, path, exc): + def __init__(self, project_name: str, path: str, exc: ValidationException): self.project_name = project_name self.path = path self.exc = exc @@ -1107,21 +1113,21 @@ def get_message(self) -> str: class InvalidTestConfig(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: ValidationError, node): self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class InvalidSchemaConfig(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: ValidationError, node): self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) class InvalidSnapshopConfig(ParsingException): - def __init__(self, exc, node): + def __init__(self, exc: ValidationError, node): self.msg = self.validator_error_message(exc) self.node = node super().__init__(msg=self.msg) @@ -1140,14 +1146,14 @@ def __init__(self): class UnexpectedTestNamePattern(CompilationException): - def __init__(self, test_name): + def __init__(self, test_name: str): self.test_name = test_name msg = f"Test name string did not match expected pattern: {self.test_name}" super().__init__(msg=msg) class CustomMacroPopulatingConfigValues(CompilationException): - def __init__(self, target_name, column_name, name, key, err_msg): + def __init__(self, target_name: str, column_name: str, name: str, key: str, err_msg: str): self.target_name = target_name self.column_name = column_name self.name = name @@ -1177,21 +1183,21 @@ def get_message(self) -> str: class TagsNotListOfStrings(CompilationException): - def __init__(self, tags): + def __init__(self, tags: Any): self.tags = tags msg = f"got {self.tags} ({type(self.tags)}) for tags, expected a list of strings" super().__init__(msg=msg) class TagNotString(CompilationException): - def __init__(self, tag): + def __init__(self, tag: Any): self.tag = tag msg = f"got {self.tag} ({type(self.tag)}) for tag, expected a str" super().__init__(msg=msg) class TestNameNotString(ParsingException): - def __init__(self, test_name): + def __init__(self, test_name: Any): self.test_name = test_name super().__init__(msg=self.get_message()) @@ -1202,7 +1208,7 @@ def get_message(self) -> str: class TestArgsNotDict(ParsingException): - def __init__(self, test_args): + def __init__(self, test_args: Any): self.test_args = test_args super().__init__(msg=self.get_message()) @@ -1227,7 +1233,7 @@ def get_message(self) -> str: class TestInvalidType(ParsingException): - def __init__(self, test): + def __init__(self, test: Any): self.test = test super().__init__(msg=self.get_message()) @@ -1238,7 +1244,7 @@ def get_message(self) -> str: # This is triggered across multiple files class EnvVarMissing(ParsingException): - def __init__(self, var): + def __init__(self, var: str): self.var = var super().__init__(msg=self.get_message()) @@ -1361,7 +1367,7 @@ def get_message(self) -> str: class CrossDbReferenceProhibited(CompilationException): - def __init__(self, adapter, exc_msg): + def __init__(self, adapter, exc_msg: str): self.adapter = adapter self.exc_msg = exc_msg super().__init__(msg=self.get_message()) @@ -1372,7 +1378,7 @@ def get_message(self) -> str: class IndexConfigNotDict(CompilationException): - def __init__(self, raw_index): + def __init__(self, raw_index: Any): self.raw_index = raw_index super().__init__(msg=self.get_message()) @@ -1386,7 +1392,7 @@ def get_message(self) -> str: class InvalidIndexConfig(CompilationException): - def __init__(self, exc): + def __init__(self, exc: TypeError): self.exc = exc super().__init__(msg=self.get_message()) @@ -1398,7 +1404,7 @@ def get_message(self) -> str: # adapters exceptions class InvalidMacroResult(CompilationException): - def __init__(self, freshness_macro_name, table): + def __init__(self, freshness_macro_name: str, table): self.freshness_macro_name = freshness_macro_name self.table = table super().__init__(msg=self.get_message()) @@ -1410,7 +1416,7 @@ def get_message(self) -> str: class SnapshotTargetNotSnapshotTable(CompilationException): - def __init__(self, missing): + def __init__(self, missing: List): self.missing = missing super().__init__(msg=self.get_message()) @@ -1422,7 +1428,7 @@ def get_message(self) -> str: class SnapshotTargetIncomplete(CompilationException): - def __init__(self, extra, missing): + def __init__(self, extra: List, missing: List): self.extra = extra self.missing = missing super().__init__(msg=self.get_message()) @@ -1438,7 +1444,7 @@ def get_message(self) -> str: class RenameToNoneAttempted(CompilationException): - def __init__(self, src_name, dst_name, name): + def __init__(self, src_name: str, dst_name: str, name: str): self.src_name = src_name self.dst_name = dst_name self.name = name @@ -1447,21 +1453,21 @@ def __init__(self, src_name, dst_name, name): class NullRelationDropAttempted(CompilationException): - def __init__(self, name): + def __init__(self, name: str): self.name = name self.msg = f"Attempted to drop a null relation for {self.name}" super().__init__(msg=self.msg) class NullRelationCacheAttempted(CompilationException): - def __init__(self, name): + def __init__(self, name: str): self.name = name self.msg = f"Attempted to cache a null relation for {self.name}" super().__init__(msg=self.msg) class InvalidQuoteConfigType(CompilationException): - def __init__(self, quote_config): + def __init__(self, quote_config: Any): self.quote_config = quote_config super().__init__(msg=self.get_message()) @@ -1491,7 +1497,7 @@ def __init__(self, relation): class MaterializationNotAvailable(CompilationException): - def __init__(self, model, adapter_type): + def __init__(self, model, adapter_type: str): self.model = model self.adapter_type = adapter_type super().__init__(msg=self.get_message()) @@ -1503,7 +1509,7 @@ def get_message(self) -> str: class RelationReturnedMultipleResults(CompilationException): - def __init__(self, kwargs, matches): + def __init__(self, kwargs: Mapping[str, Any], matches: List): self.kwargs = kwargs self.matches = matches super().__init__(msg=self.get_message()) @@ -1537,7 +1543,7 @@ def get_message(self) -> str: # adapters exceptions class UnexpectedNull(DatabaseException): - def __init__(self, field_name, source): + def __init__(self, field_name: str, source): self.field_name = field_name self.source = source msg = ( @@ -1548,7 +1554,7 @@ def __init__(self, field_name, source): class UnexpectedNonTimestamp(DatabaseException): - def __init__(self, field_name, source, dt): + def __init__(self, field_name: str, source, dt: Any): self.field_name = field_name self.source = source self.type_name = type(dt).__name__ @@ -1561,7 +1567,7 @@ def __init__(self, field_name, source, dt): # deps exceptions class MultipleVersionGitDeps(DependencyException): - def __init__(self, git, requested): + def __init__(self, git: str, requested): self.git = git self.requested = requested msg = ( @@ -1572,7 +1578,7 @@ def __init__(self, git, requested): class DuplicateProjectDependency(DependencyException): - def __init__(self, project_name): + def __init__(self, project_name: str): self.project_name = project_name msg = ( f'Found duplicate project "{self.project_name}". This occurs when ' @@ -1582,7 +1588,7 @@ def __init__(self, project_name): class DuplicateDependencyToRoot(DependencyException): - def __init__(self, project_name): + def __init__(self, project_name: str): self.project_name = project_name msg = ( "Found a dependency with the same name as the root project " @@ -1604,7 +1610,13 @@ def __init__(self, new, old): class PackageVersionNotFound(DependencyException): - def __init__(self, package_name, version_range, available_versions, should_version_check): + def __init__( + self, + package_name: str, + version_range, + available_versions: List[str], + should_version_check: bool, + ): self.package_name = package_name self.version_range = version_range self.available_versions = available_versions @@ -1634,7 +1646,7 @@ def get_message(self) -> str: class PackageNotFound(DependencyException): - def __init__(self, package_name): + def __init__(self, package_name: str): self.package_name = package_name msg = f"Package {self.package_name} was not found in the package index" super().__init__(msg) @@ -1644,35 +1656,35 @@ def __init__(self, package_name): class ProfileConfigInvalid(DbtProfileError): - def __init__(self, exc): + def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class ProjectContractInvalid(DbtProjectError): - def __init__(self, exc): + def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class ProjectContractBroken(DbtProjectError): - def __init__(self, exc): + def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class ConfigContractBroken(DbtProjectError): - def __init__(self, exc): + def __init__(self, exc: ValidationError): self.exc = exc msg = self.validator_error_message(self.exc) super().__init__(msg=msg) class NonUniquePackageName(CompilationException): - def __init__(self, project_name): + def __init__(self, project_name: str): self.project_name = project_name super().__init__(msg=self.get_message()) @@ -1687,7 +1699,12 @@ def get_message(self) -> str: class UninstalledPackagesFound(CompilationException): - def __init__(self, count_packages_specified, count_packages_installed, packages_install_path): + def __init__( + self, + count_packages_specified: int, + count_packages_installed: int, + packages_install_path: str, + ): self.count_packages_specified = count_packages_specified self.count_packages_installed = count_packages_installed self.packages_install_path = packages_install_path @@ -1712,9 +1729,7 @@ def __init__(self, var_type): def get_message(self) -> str: type_name = self.var_type.__name__ - msg = "The --vars argument must be a YAML dictionary, but was " "of type '{}'".format( - type_name - ) + msg = f"The --vars argument must be a YAML dictionary, but was of type '{type_name}'" return msg @@ -1722,7 +1737,7 @@ def get_message(self) -> str: class DuplicateMacroInPackage(CompilationException): - def __init__(self, macro, macro_mapping): + def __init__(self, macro, macro_mapping: Mapping): self.macro = macro self.macro_mapping = macro_mapping super().__init__(msg=self.get_message()) @@ -1771,7 +1786,7 @@ def get_message(self) -> str: # jinja exceptions class MissingConfig(CompilationException): - def __init__(self, unique_id, name): + def __init__(self, unique_id: str, name: str): self.unique_id = unique_id self.name = name msg = ( @@ -1819,32 +1834,25 @@ def __init__(self, node_1, node_2, duped_name=None): def get_message(self) -> str: msg = ( - 'dbt found two resources with the database representation "{}".\ndbt ' + f'dbt found two resources with the database representation "{self.duped_name}".\ndbt ' "cannot create two resources with identical database representations. " "To fix this,\nchange the configuration of one of these resources:" - "\n- {} ({})\n- {} ({})".format( - self.duped_name, - self.node_1.unique_id, - self.node_1.original_file_path, - self.node_2.unique_id, - self.node_2.original_file_path, - ) + f"\n- {self.node_1.unique_id} ({self.node_1.original_file_path})\n- {self.node_2.unique_id} ({self.node_2.original_file_path})" ) return msg class AmbiguousCatalogMatch(CompilationException): - def __init__(self, unique_id, match_1, match_2): + def __init__(self, unique_id: str, match_1, match_2): self.unique_id = unique_id self.match_1 = match_1 self.match_2 = match_2 super().__init__(msg=self.get_message()) def get_match_string(self, match): - return "{}.{}".format( - match.get("metadata", {}).get("schema"), - match.get("metadata", {}).get("name"), - ) + match_schema = match.get("metadata", {}).get("schema") + match_name = match.get("metadata", {}).get("name") + return f"{match_schema}.{match_name}" def get_message(self) -> str: msg = ( @@ -1859,14 +1867,14 @@ def get_message(self) -> str: class CacheInconsistency(InternalException): - def __init__(self, msg): + def __init__(self, msg: str): self.msg = msg formatted_msg = f"Cache inconsistency detected: {self.msg}" super().__init__(msg=formatted_msg) class NewNameAlreadyInCache(CacheInconsistency): - def __init__(self, old_key, new_key): + def __init__(self, old_key: str, new_key: str): self.old_key = old_key self.new_key = new_key msg = ( @@ -1876,21 +1884,21 @@ def __init__(self, old_key, new_key): class ReferencedLinkNotCached(CacheInconsistency): - def __init__(self, referenced_key): + def __init__(self, referenced_key: str): self.referenced_key = referenced_key msg = f"in add_link, referenced link key {self.referenced_key} not in cache!" super().__init__(msg) class DependentLinkNotCached(CacheInconsistency): - def __init__(self, dependent_key): + def __init__(self, dependent_key: str): self.dependent_key = dependent_key msg = f"in add_link, dependent link key {self.dependent_key} not in cache!" super().__init__(msg) class TruncatedModelNameCausedCollision(CacheInconsistency): - def __init__(self, new_key, relations): + def __init__(self, new_key, relations: Dict): self.new_key = new_key self.relations = relations super().__init__(self.get_message()) @@ -1905,8 +1913,8 @@ def get_message(self) -> str: "\n\nName collisions can occur when the length of two " "models' names approach your database's builtin limit. " "Try restructuring your project such that no two models " - "share the prefix '{}'.".format(truncated_model_name_prefix) - + " Then, clean your warehouse of any removed models." + f"share the prefix '{truncated_model_name_prefix}'. " + "Then, clean your warehouse of any removed models." ) else: message_addendum = "" @@ -1924,7 +1932,7 @@ def __init__(self): # this is part of the context and also raised in dbt.contracts.relation.py class DataclassNotDict(CompilationException): - def __init__(self, obj): + def __init__(self, obj: Any): self.obj = obj super().__init__(msg=self.get_message()) @@ -2030,7 +2038,7 @@ def get_message(self) -> str: class InvalidPropertyYML(CompilationException): - def __init__(self, path, issue): + def __init__(self, path: str, issue: str): self.path = path self.issue = issue super().__init__(msg=self.get_message()) @@ -2045,29 +2053,29 @@ def get_message(self) -> str: class PropertyYMLMissingVersion(InvalidPropertyYML): - def __init__(self, path): + def __init__(self, path: str): self.path = path self.issue = f"the yml property file {self.path} is missing a version tag" - super().__init__() + super().__init__(self.path, self.issue) class PropertyYMLVersionNotInt(InvalidPropertyYML): - def __init__(self, path, version): + def __init__(self, path: str, version: Any): self.path = path self.version = version self.issue = ( "its 'version:' tag must be an integer (e.g. version: 2)." f" {self.version} is not an integer" ) - super().__init__() + super().__init__(self.path, self.issue) class PropertyYMLInvalidTag(InvalidPropertyYML): - def __init__(self, path, version): + def __init__(self, path: str, version: int): self.path = path self.version = version self.issue = f"its 'version:' tag is set to {self.version}. Only 2 is supported" - super().__init__() + super().__init__(self.path, self.issue) class RelationWrongType(CompilationException): @@ -2205,12 +2213,12 @@ def invalid_materialization_argument(name, argument): def bad_package_spec(repo, spec, error_message): - msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message) + msg = f"Error checking out spec='{spec}' for repo {repo}\n{error_message}" raise InternalException(scrub_secrets(msg, env_secrets())) def raise_git_cloning_error(error: CommandResultError) -> NoReturn: - error.cmd = scrub_secrets(str(error.cmd), env_secrets()) + error.cmd = list(scrub_secrets(str(error.cmd), env_secrets())) raise error @@ -2300,8 +2308,7 @@ def raise_unrecognized_credentials_type(typename, supported_types): def raise_patch_targets_not_found(patches): patch_list = "\n\t".join( - "model {} (referenced in path {})".format(p.name, p.original_file_path) - for p in patches.values() + f"model {p.name} (referenced in path {p.original_file_path})" for p in patches.values() ) msg = f"dbt could not find models for the following patches:\n\t{patch_list}" raise CompilationException(msg) From e85fb6762fecaea32e976c4cdd8ce4bce521e321 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 15 Dec 2022 07:32:16 -0600 Subject: [PATCH 22/24] fix type errors --- core/dbt/context/base.py | 7 +++-- core/dbt/exceptions.py | 57 ++++++++++++++++++++++++++++++++------ core/dbt/parser/schemas.py | 15 ++++++---- 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 98efaf69a50..59984cb96ab 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -10,11 +10,12 @@ from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER from dbt.contracts.graph.nodes import Resource from dbt.exceptions import ( - CompilationException, DisallowSecretEnvVar, EnvVarMissing, MacroReturn, RequiredVarNotFound, + SetStrictWrongType, + ZipStrictWrongType, ) from dbt.events.functions import fire_event, get_invocation_id from dbt.events.types import JinjaLogInfo, JinjaLogDebug @@ -492,7 +493,7 @@ def set_strict(value: Iterable[Any]) -> Set[Any]: try: return set(value) except TypeError as e: - raise CompilationException(e) + raise SetStrictWrongType(e) @contextmember("zip") @staticmethod @@ -536,7 +537,7 @@ def zip_strict(*args: Iterable[Any]) -> Iterable[Any]: try: return zip(*args) except TypeError as e: - raise CompilationException(e) + raise ZipStrictWrongType(e) @contextmember @staticmethod diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 0683098d58a..735e36c07bf 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -1,7 +1,7 @@ import builtins import json import re -from typing import Any, Dict, List, Mapping, NoReturn, Optional +from typing import Any, Dict, List, Mapping, NoReturn, Optional, Union # from dbt.contracts.graph import ManifestNode # or ParsedNode? from dbt.dataclass_schema import ValidationError @@ -468,7 +468,7 @@ def __init__( self, cwd: str, cmd: List[str], - returncode: str, + returncode: Union[int, Any], stdout: bytes, stderr: bytes, msg: str = "Got a non-zero returncode", @@ -557,14 +557,14 @@ def __init__(self, name: str, node): class CaughtMacroExceptionWithNode(CompilationException): - def __init__(self, exc: Exception, node): + def __init__(self, exc, node): self.exc = exc self.node = node super().__init__(msg=str(exc)) class CaughtMacroException(CompilationException): - def __init__(self, exc: Exception): + def __init__(self, exc): self.exc = exc super().__init__(msg=str(exc)) @@ -751,6 +751,20 @@ def get_message(self) -> str: # context level exceptions +class ZipStrictWrongType(CompilationException): + def __init__(self, exc): + self.exc = exc + msg = str(self.exc) + super().__init__(msg=msg) + + +class SetStrictWrongType(CompilationException): + def __init__(self, exc): + self.exc = exc + msg = str(self.exc) + super().__init__(msg=msg) + + class LoadAgateTableValueError(CompilationException): def __init__(self, exc: ValueError, node): self.exc = exc @@ -866,7 +880,7 @@ def get_message(self) -> str: class InvalidMacroArgType(CompilationException): def __init__( - self, method_name: str, arg_name: str, got_value: Any, expected_type: str, version: str + self, method_name: str, arg_name: str, got_value: Any, expected_type, version: str ): self.method_name = method_name self.arg_name = arg_name @@ -1072,7 +1086,7 @@ def __init__(self, exc: ValidationError, node): super().__init__(msg=self.msg) -class YamlParseFailure(ParsingException): +class YamlParseListFailure(ParsingException): def __init__( self, path: str, @@ -1097,8 +1111,33 @@ def get_message(self) -> str: return msg +class YamlParseDictFailure(ParsingException): + def __init__( + self, + path: str, + key: str, + yaml_data: Dict[str, Any], + cause, + ): + self.path = path + self.key = key + self.yaml_data = yaml_data + self.cause = cause + super().__init__(msg=self.get_message()) + + def get_message(self) -> str: + if isinstance(self.cause, str): + reason = self.cause + elif isinstance(self.cause, ValidationError): + reason = self.validator_error_message(self.cause) + else: + reason = self.cause.msg + msg = f"Invalid {self.key} config given in {self.path} @ {self.key}: {self.yaml_data} - {reason}" + return msg + + class YamlLoadFailure(ParsingException): - def __init__(self, project_name: str, path: str, exc: ValidationException): + def __init__(self, project_name: Optional[str], path: str, exc: ValidationException): self.project_name = project_name self.path = path self.exc = exc @@ -1153,7 +1192,9 @@ def __init__(self, test_name: str): class CustomMacroPopulatingConfigValues(CompilationException): - def __init__(self, target_name: str, column_name: str, name: str, key: str, err_msg: str): + def __init__( + self, target_name: str, column_name: Optional[str], name: str, key: str, err_msg: str + ): self.target_name = target_name self.column_name = column_name self.name = name diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index fa0c999713b..b5fd8558889 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -64,7 +64,8 @@ PropertyYMLVersionNotInt, ValidationException, YamlLoadFailure, - YamlParseFailure, + YamlParseDictFailure, + YamlParseListFailure, ) from dbt.events.functions import warn_or_error from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroPatchNotFound @@ -606,7 +607,9 @@ def get_key_dicts(self) -> Iterable[Dict[str, Any]]: # check that entry is a dict and that all dict values # are strings if coerce_dict_str(entry) is None: - raise YamlParseFailure(path, self.key, data, "expected a dict with string keys") + raise YamlParseListFailure( + path, self.key, data, "expected a dict with string keys" + ) if "name" not in entry: raise ParsingException("Entry did not contain a name") @@ -653,7 +656,7 @@ def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T: cls.validate(data) return cls.from_dict(data) except (ValidationError, JSONValidationException) as exc: - raise YamlParseFailure(path, self.key, data, exc) + raise YamlParseDictFailure(path, self.key, data, exc) # The other parse method returns TestBlocks. This one doesn't. # This takes the yaml dictionaries in 'sources' keys and uses them @@ -778,7 +781,7 @@ def get_unparsed_target(self) -> Iterable[NonSourceTarget]: self.normalize_docs_attribute(data, path) node = self._target_type().from_dict(data) except (ValidationError, JSONValidationException) as exc: - raise YamlParseFailure(path, self.key, data, exc) + raise YamlParseDictFailure(path, self.key, data, exc) else: yield node @@ -1061,7 +1064,7 @@ def parse(self): UnparsedExposure.validate(data) unparsed = UnparsedExposure.from_dict(data) except (ValidationError, JSONValidationException) as exc: - raise YamlParseFailure(self.yaml.path, self.key, data, exc) + raise YamlParseDictFailure(self.yaml.path, self.key, data, exc) self.parse_exposure(unparsed) @@ -1178,5 +1181,5 @@ def parse(self): unparsed = UnparsedMetric.from_dict(data) except (ValidationError, JSONValidationException) as exc: - raise YamlParseFailure(self.yaml.path, self.key, data, exc) + raise YamlParseDictFailure(self.yaml.path, self.key, data, exc) self.parse_metric(unparsed) From 5b86c8b20b33d4f091e5281f0d52339ca437c25a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 15 Dec 2022 15:38:32 -0600 Subject: [PATCH 23/24] fix type for adapter_response --- core/dbt/task/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index 44d325f6047..bc8f9a2de75 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -400,7 +400,7 @@ def safe_run_hooks( thread_id="main", timing=[], message=f"{hook_type.value} failed, error:\n {exc.msg}", - adapter_response=exc.msg, + adapter_response={}, execution_time=0, failures=1, ) From a6dbe5121d3252ada17c6f16771b6dd766bb7282 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 16 Dec 2022 15:33:57 -0600 Subject: [PATCH 24/24] remove 0.13 version from message --- core/dbt/adapters/base/impl.py | 5 ----- core/dbt/exceptions.py | 13 ++++--------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index 48c4c4f2989..64ebbeac5dd 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -617,7 +617,6 @@ def get_missing_columns( arg_name="from_relation", got_value=from_relation, expected_type=self.Relation, - version="0.13.0", ) if not isinstance(to_relation, self.Relation): @@ -626,7 +625,6 @@ def get_missing_columns( arg_name="to_relation", got_value=to_relation, expected_type=self.Relation, - version="0.13.0", ) from_columns = {col.name: col for col in self.get_columns_in_relation(from_relation)} @@ -652,7 +650,6 @@ def valid_snapshot_target(self, relation: BaseRelation) -> None: arg_name="relation", got_value=relation, expected_type=self.Relation, - version="0.13.0", ) columns = self.get_columns_in_relation(relation) @@ -683,7 +680,6 @@ def expand_target_column_types( arg_name="from_relation", got_value=from_relation, expected_type=self.Relation, - version="0.13.0", ) if not isinstance(to_relation, self.Relation): @@ -692,7 +688,6 @@ def expand_target_column_types( arg_name="to_relation", got_value=to_relation, expected_type=self.Relation, - version="0.13.0", ) self.expand_column_types(from_relation, to_relation) diff --git a/core/dbt/exceptions.py b/core/dbt/exceptions.py index 735e36c07bf..2db130bb44e 100644 --- a/core/dbt/exceptions.py +++ b/core/dbt/exceptions.py @@ -879,20 +879,17 @@ def get_message(self) -> str: class InvalidMacroArgType(CompilationException): - def __init__( - self, method_name: str, arg_name: str, got_value: Any, expected_type, version: str - ): + def __init__(self, method_name: str, arg_name: str, got_value: Any, expected_type): self.method_name = method_name self.arg_name = arg_name self.got_value = got_value self.expected_type = expected_type - self.version = version super().__init__(msg=self.get_message()) def get_message(self) -> str: got_type = type(self.got_value) msg = ( - f"As of {self.version}, 'adapter.{self.method_name}' expects argument " + f"'adapter.{self.method_name}' expects argument " f"'{self.arg_name}' to be of type '{self.expected_type}', instead got " f"{self.got_value} ({got_type})" ) @@ -2319,13 +2316,11 @@ def invalid_bool_error(got_value, macro_name) -> NoReturn: raise InvalidBoolean(return_value=got_value, macro_name=macro_name) -def invalid_type_error( - method_name, arg_name, got_value, expected_type, version="0.13.0" -) -> NoReturn: +def invalid_type_error(method_name, arg_name, got_value, expected_type) -> NoReturn: """Raise a CompilationException when an adapter method available to macros has changed. """ - raise InvalidMacroArgType(method_name, arg_name, got_value, expected_type, version) + raise InvalidMacroArgType(method_name, arg_name, got_value, expected_type) def disallow_secret_env_var(env_var_name) -> NoReturn: