From 5f1ea261337db5989ee12c941521c01956cb9098 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 8 Jul 2019 09:35:30 -0600 Subject: [PATCH] Handle some TODOs remove another to_dict call we do not in fact pass dicts in to Var() Make things that get passed to Var() tell it about their vars finally make empty a property documentation resource type is now a property, not serialized added a Mergeable helper mixin to perform simple merges --- core/dbt/clients/jinja.py | 13 +- core/dbt/context/common.py | 22 +--- core/dbt/contracts/graph/compiled.py | 6 +- core/dbt/contracts/graph/parsed.py | 11 +- core/dbt/contracts/graph/unparsed.py | 18 +-- core/dbt/contracts/project.py | 4 +- core/dbt/contracts/util.py | 16 +++ core/dbt/parser/base.py | 7 +- core/dbt/parser/docs.py | 2 - core/dbt/parser/schemas.py | 10 +- .../test_docs_generate.py | 30 ----- test/unit/test_compiler.py | 9 -- test/unit/test_context.py | 1 - test/unit/test_contracts_graph_unparsed.py | 116 ++++++++++++++++++ test/unit/test_docs_blocks.py | 1 - test/unit/test_manifest.py | 15 +-- test/unit/test_parser.py | 44 ------- 17 files changed, 166 insertions(+), 159 deletions(-) create mode 100644 test/unit/test_contracts_graph_unparsed.py diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index 765aab184a0..4433726c6d6 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -76,7 +76,7 @@ def get_node_template(self, node): template = get_template( string=node.raw_sql, ctx={}, - node=node.to_dict(), + node=node, ) self.file_cache[key] = template @@ -170,7 +170,6 @@ def _is_dunder_name(name): def create_macro_capture_env(node): - # TODO: remove the to_dict() call and change this to use objects class ParserMacroCapture(jinja2.Undefined): """ This class sets up the parser to capture macros. @@ -179,21 +178,21 @@ def __init__(self, hint=None, obj=None, name=None, exc=None): super().__init__(hint=hint, name=name) self.node = node self.name = name - self.package_name = node.get('package_name') + self.package_name = node.package_name # jinja uses these for safety, so we have to override them. # see https://github.com/pallets/jinja/blob/master/jinja2/sandbox.py#L332-L339 # noqa self.unsafe_callable = False self.alters_data = False def __deepcopy__(self, memo): - path = os.path.join(self.node.get('root_path'), - self.node.get('original_file_path')) + path = os.path.join(self.node.root_path, + self.node.original_file_path) logger.debug( 'dbt encountered an undefined variable, "{}" in node {}.{} ' '(source path: {})' - .format(self.name, self.node.get('package_name'), - self.node.get('name'), path)) + .format(self.name, self.node.package_name, + self.node.name, path)) # match jinja's message dbt.exceptions.raise_compiler_error( diff --git a/core/dbt/context/common.py b/core/dbt/context/common.py index afbbd667ff9..c1487c5427c 100644 --- a/core/dbt/context/common.py +++ b/core/dbt/context/common.py @@ -3,7 +3,6 @@ from dbt.adapters.factory import get_adapter from dbt.node_types import NodeType -from dbt.contracts.graph.parsed import ParsedMacro from dbt.include.global_project import PACKAGES from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME @@ -225,28 +224,13 @@ def __init__(self, model, context, overrides): # precedence over context-based var definitions self.overrides = overrides - if isinstance(model, dict) and model.get('unique_id'): - # TODO: is this reachable anymore? - local_vars = model.get('config', {}).get('vars', {}) - self.model_name = model.get('name') - elif isinstance(model, ParsedMacro): - local_vars = {} - self.model_name = model.name - elif hasattr(model, 'name'): - # handle all manner of Node objects - # TODO: maybe we should register them against an abstract base - # class so we can just do an `isinstance` check? - local_vars = model.config.vars - self.model_name = model.name - elif model is None: + if model is None: # during config parsing we have no model and no local vars self.model_name = '' local_vars = {} else: - raise dbt.exceptions.InternalException( - 'Could not call Var() for unknown model {} with type {}' - .format(model, type(model)) - ) + self.model_name = model.name + local_vars = model.local_vars() self.local_vars = dbt.utils.merge(local_vars, overrides) diff --git a/core/dbt/contracts/graph/compiled.py b/core/dbt/contracts/graph/compiled.py index 0059fc535a9..bf0c8d1ea04 100644 --- a/core/dbt/contracts/graph/compiled.py +++ b/core/dbt/contracts/graph/compiled.py @@ -19,7 +19,7 @@ class InjectedCTE(JsonSchemaMixin, Replaceable): sql: Optional[str] = None -class CompiledNodeMixins: +class CompiledNodeMixins(ParsedNodeMixins): def prepend_ctes(self, prepended_ctes): self.extra_ctes_injected = True self.extra_ctes = prepended_ctes @@ -53,7 +53,6 @@ class CompiledNode( HasFqn, CanRef, HasRelationMetadata, - ParsedNodeMixins, CompiledNodeMixins): compiled: bool compiled_sql: Optional[str] @@ -61,7 +60,6 @@ class CompiledNode( extra_ctes: List[InjectedCTE] injected_sql: Optional[str] alias: str - empty: bool tags: List[str] config: NodeConfig docrefs: List[Docref] = field(default_factory=list) @@ -80,7 +78,6 @@ class CompiledTestNode( HasFqn, CanRef, HasRelationMetadata, - ParsedNodeMixins, CompiledNodeMixins): compiled: bool compiled_sql: Optional[str] @@ -89,7 +86,6 @@ class CompiledTestNode( injected_sql: Optional[str] resource_type: TestType alias: str - empty: bool tags: List[str] config: TestConfig docrefs: List[Docref] = field(default_factory=list) diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index b81d9a31166..cb0f9a3dc06 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -10,7 +10,6 @@ UnparsedBaseNode, FreshnessThreshold ) from dbt.contracts.util import Replaceable - from dbt.logger import GLOBAL_LOGGER as logger # noqa from dbt.node_types import ( NodeType, SourceType, SnapshotType, MacroType, TestType @@ -187,6 +186,9 @@ def patch(self, patch): def get_materialization(self): return self.config.materialized + def local_vars(self): + return self.config.vars + # TODO(jeb): hooks should get their own parsed type instead of including # index everywhere! @@ -199,7 +201,6 @@ class ParsedNode( HasRelationMetadata, ParsedNodeMixins): alias: str - empty: bool tags: List[str] config: NodeConfig docrefs: List[Docref] = field(default_factory=list) @@ -225,7 +226,6 @@ class ParsedTestNode( ParsedNodeMixins): resource_type: TestType alias: str - empty: bool tags: List[str] config: TestConfig docrefs: List[Docref] = field(default_factory=list) @@ -239,8 +239,8 @@ class ParsedTestNode( @dataclass(init=False) class _SnapshotConfig(NodeConfig): unique_key: str - target_database: str = None target_schema: str = None + target_database: str = None def __init__( self, @@ -333,6 +333,9 @@ class ParsedMacro(UnparsedMacro): tags: List[str] depends_on: _MacroDependsOn + def local_vars(self): + return {} + @property def generator(self): """ diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 996be7dcab4..189274a544b 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -1,7 +1,5 @@ -from dbt.node_types import UnparsedNodeType, NodeType -from dbt.contracts.util import Replaceable from dbt.node_types import UnparsedNodeType, NodeType, OperationType -from dbt.contracts.util import Replaceable +from dbt.contracts.util import Replaceable, Mergeable from hologram import JsonSchemaMixin from hologram.helpers import StrEnum @@ -23,6 +21,10 @@ class UnparsedBaseNode(JsonSchemaMixin, Replaceable): class HasSQL: raw_sql: str + @property + def empty(self): + return not self.raw_sql.strip() + @dataclass class UnparsedMacro(UnparsedBaseNode, HasSQL): @@ -100,7 +102,7 @@ class FreshnessStatus(StrEnum): @dataclass -class FreshnessThreshold(JsonSchemaMixin, Replaceable): +class FreshnessThreshold(JsonSchemaMixin, Mergeable): warn_after: Optional[Time] = None error_after: Optional[Time] = None @@ -114,7 +116,7 @@ def status(self, age: float) -> FreshnessStatus: @dataclass -class Quoting(JsonSchemaMixin, Replaceable): +class Quoting(JsonSchemaMixin, Mergeable): database: Optional[bool] = None schema: Optional[bool] = None identifier: Optional[bool] = None @@ -152,5 +154,7 @@ class UnparsedDocumentationFile(JsonSchemaMixin, Replaceable): path: str original_file_path: str file_contents: str - # TODO: remove this. - resource_type: StrLiteral(NodeType.Documentation) + + @property + def resource_type(self): + return NodeType.Documentation diff --git a/core/dbt/contracts/project.py b/core/dbt/contracts/project.py index 27373d9de09..24ddfc6ed9c 100644 --- a/core/dbt/contracts/project.py +++ b/core/dbt/contracts/project.py @@ -1,4 +1,4 @@ -from dbt.contracts.util import Replaceable +from dbt.contracts.util import Replaceable, Mergeable from dbt.logger import GLOBAL_LOGGER as logger # noqa from dbt import tracking from dbt.ui import printer @@ -36,7 +36,7 @@ def json_schem(self): @dataclass -class Quoting(JsonSchemaMixin, Replaceable): +class Quoting(JsonSchemaMixin, Mergeable): identifier: Optional[bool] schema: Optional[bool] database: Optional[bool] diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py index 695611f8ea4..f7718997950 100644 --- a/core/dbt/contracts/util.py +++ b/core/dbt/contracts/util.py @@ -8,6 +8,22 @@ def replace(self, **kwargs): return dataclasses.replace(self, **kwargs) +class Mergeable(Replaceable): + def merged(self, *args): + """Perform a shallow merge, where the last non-None write wins. This is + intended to merge dataclasses that are a collection of optional values. + """ + replacements = {} + cls = type(self) + for field in dataclasses.fields(cls): + for arg in args: + value = getattr(arg, field.name) + if value is not None: + replacements[field.name] = value + + return self.replace(**replacements) + + class Writable: def write(self, path: str, omit_none: bool = True): write_json(path, self.to_dict(omit_none=omit_none)) diff --git a/core/dbt/parser/base.py b/core/dbt/parser/base.py index ddadc2a90ca..646288e426e 100644 --- a/core/dbt/parser/base.py +++ b/core/dbt/parser/base.py @@ -170,10 +170,6 @@ def _build_intermediate_node_dict(self, config, node_dict, node_path, config_dict.update(config.config) self._mangle_hooks(config_dict) - empty = ( - 'raw_sql' in node_dict and len(node_dict['raw_sql'].strip()) == 0 - ) - node_dict.update({ 'refs': [], 'sources': [], @@ -182,7 +178,6 @@ def _build_intermediate_node_dict(self, config, node_dict, node_path, 'macros': [], }, 'unique_id': node_path, - 'empty': empty, 'fqn': fqn, 'tags': tags, 'config': config_dict, @@ -212,7 +207,7 @@ def _render_with_context(self, parsed_node, config): config) dbt.clients.jinja.get_rendered( - parsed_node.raw_sql, context, parsed_node.to_dict(), + parsed_node.raw_sql, context, parsed_node, capture_macros=True) def _update_parsed_node_info(self, parsed_node, config): diff --git a/core/dbt/parser/docs.py b/core/dbt/parser/docs.py index 19fdbcc83fb..18db1ab0754 100644 --- a/core/dbt/parser/docs.py +++ b/core/dbt/parser/docs.py @@ -1,5 +1,4 @@ import dbt.exceptions -from dbt.node_types import NodeType from dbt.parser.base import BaseParser from dbt.contracts.graph.unparsed import UnparsedDocumentationFile from dbt.contracts.graph.parsed import ParsedDocumentation @@ -35,7 +34,6 @@ def load_file(cls, package_name, root_dir, relative_dirs): yield UnparsedDocumentationFile( root_path=root_dir, - resource_type=NodeType.Documentation, path=path, original_file_path=original_file_path, package_name=package_name, diff --git a/core/dbt/parser/schemas.py b/core/dbt/parser/schemas.py index f5821e790f4..de8a1efe083 100644 --- a/core/dbt/parser/schemas.py +++ b/core/dbt/parser/schemas.py @@ -15,7 +15,6 @@ import dbt.contracts.project from dbt.contracts.graph.parsed import ColumnInfo, Docref -from dbt.contracts.graph.unparsed import FreshnessThreshold, Quoting from dbt.context.common import generate_config_context from dbt.clients.jinja import get_rendered from dbt.node_types import NodeType @@ -455,14 +454,9 @@ def generate_source_node(self, source, table, path, package_name, root_dir, get_rendered(source_description, context) loaded_at_field = table.loaded_at_field or source.loaded_at_field - # TODO: maybe hologram should do these deep merges + conversions? - freshness = FreshnessThreshold.from_dict(dbt.utils.deep_merge( - source.freshness.to_dict(), table.freshness.to_dict() - )) + freshness = source.freshness.merged(table.freshness) - quoting = Quoting.from_dict(dbt.utils.deep_merge( - source.quoting.to_dict(), table.quoting.to_dict() - )) + quoting = source.quoting.merged(table.quoting) default_database = self.root_project_config.credentials.database return ParsedSourceDefinition( diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index d01f49a039d..8ca69865b78 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -845,7 +845,6 @@ def expected_seeded_manifest(self, model_database=None): 'sources': [], 'depends_on': {'nodes': ['seed.test.seed'], 'macros': []}, 'unique_id': 'model.test.model', - 'empty': False, 'fqn': ['test', 'model'], 'tags': [], 'config': model_config, @@ -891,7 +890,6 @@ def expected_seeded_manifest(self, model_database=None): 'sources': [], 'depends_on': {'nodes': [], 'macros': []}, 'unique_id': 'seed.test.seed', - 'empty': False, 'fqn': ['test', 'seed'], 'tags': [], 'config': { @@ -931,7 +929,6 @@ def expected_seeded_manifest(self, model_database=None): 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', - 'empty': False, 'fqn': ['test', 'schema_test', 'not_null_model_id'], 'name': 'not_null_model_id', 'original_file_path': schema_yml_path, @@ -965,7 +962,6 @@ def expected_seeded_manifest(self, model_database=None): 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', - 'empty': False, 'fqn': ['test', 'schema_test', 'nothing_model_'], 'name': 'nothing_model_', 'original_file_path': schema_yml_path, @@ -1000,7 +996,6 @@ def expected_seeded_manifest(self, model_database=None): 'sources': [], 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', - 'empty': False, 'fqn': ['test', 'schema_test', 'unique_model_id'], 'name': 'unique_model_id', 'original_file_path': schema_yml_path, @@ -1075,7 +1070,6 @@ def expected_postgres_references_manifest(self, model_database=None): }, 'description': '', 'docrefs': [], - 'empty': False, 'fqn': ['test', 'ephemeral_copy'], 'name': 'ephemeral_copy', 'original_file_path': self.dir('ref_models/ephemeral_copy.sql'), @@ -1138,7 +1132,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'documentation_package': '' } ], - 'empty': False, 'fqn': ['test', 'ephemeral_summary'], 'name': 'ephemeral_summary', 'original_file_path': self.dir('ref_models/ephemeral_summary.sql'), @@ -1203,7 +1196,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'documentation_package': '' } ], - 'empty': False, 'fqn': ['test', 'view_summary'], 'name': 'view_summary', 'original_file_path': self.dir('ref_models/view_summary.sql'), @@ -1241,7 +1233,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'depends_on': {'macros': [], 'nodes': []}, 'description': '', 'docrefs': [], - 'empty': False, 'fqn': ['test', 'seed'], 'name': 'seed', 'original_file_path': self.dir('seed/seed.csv'), @@ -1309,7 +1300,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'resource_type': 'docs', 'root_path': OneOf(self.test_root_dir, self.initial_dir), 'unique_id': 'test.ephemeral_summary' }, @@ -1320,7 +1310,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'resource_type': 'docs', 'root_path': OneOf(self.test_root_dir, self.initial_dir), 'unique_id': 'test.source_info', }, @@ -1331,7 +1320,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'resource_type': 'docs', 'root_path': OneOf(self.test_root_dir, self.initial_dir), 'unique_id': 'test.summary_count' }, @@ -1342,7 +1330,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'resource_type': 'docs', 'root_path': OneOf(self.test_root_dir, self.initial_dir), 'unique_id': 'test.summary_first_name' }, @@ -1353,7 +1340,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'resource_type': 'docs', 'root_path': OneOf(self.test_root_dir, self.initial_dir), 'unique_id': 'test.table_info' }, @@ -1367,7 +1353,6 @@ def expected_postgres_references_manifest(self, model_database=None): 'original_file_path': docs_path, 'package_name': 'test', 'path': 'docs.md', - 'resource_type': 'docs', 'root_path': OneOf(self.test_root_dir, self.initial_dir), 'unique_id': 'test.view_summary' }, @@ -1419,7 +1404,6 @@ def expected_bigquery_complex_manifest(self): }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'empty': False, 'fqn': ['test', 'clustered'], 'name': 'clustered', 'original_file_path': clustered_sql_path, @@ -1476,7 +1460,6 @@ def expected_bigquery_complex_manifest(self): }, 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, - 'empty': False, 'fqn': ['test', 'multi_clustered'], 'name': 'multi_clustered', 'original_file_path': multi_clustered_sql_path, @@ -1534,7 +1517,6 @@ def expected_bigquery_complex_manifest(self): 'macros': [], 'nodes': ['model.test.nested_table'] }, - 'empty': False, 'fqn': ['test', 'nested_view'], 'name': 'nested_view', 'original_file_path': nested_view_sql_path, @@ -1592,7 +1574,6 @@ def expected_bigquery_complex_manifest(self): 'macros': [], 'nodes': [] }, - 'empty': False, 'fqn': ['test', 'nested_table'], 'name': 'nested_table', 'original_file_path': nested_table_sql_path, @@ -1625,7 +1606,6 @@ def expected_bigquery_complex_manifest(self): 'macros': [], }, 'unique_id': 'seed.test.seed', - 'empty': False, 'fqn': ['test', 'seed'], 'tags': [], 'config': { @@ -1692,7 +1672,6 @@ def expected_redshift_incremental_view_manifest(self): "macros": [], }, "unique_id": "model.test.model", - "empty": False, "fqn": ["test", "model"], "tags": [], "config": { @@ -1751,7 +1730,6 @@ def expected_redshift_incremental_view_manifest(self): "macros": [], }, "unique_id": "seed.test.seed", - "empty": False, "fqn": ["test", "seed"], "tags": [], "config": { @@ -1890,7 +1868,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, }, 'description': 'The test model', 'docrefs': [], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'model'], @@ -1943,7 +1920,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'depends_on': {'macros': [], 'nodes': []}, 'description': '', 'docrefs': [], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], @@ -1996,7 +1972,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'docrefs': [], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'not_null_model_id'], @@ -2048,7 +2023,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'docrefs': [], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'nothing_model_'], @@ -2100,7 +2074,6 @@ def expected_run_results(self, quote_schema=True, quote_model=False, 'depends_on': {'macros': [], 'nodes': ['model.test.model']}, 'description': '', 'docrefs': [], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'schema_test', 'unique_model_id'], @@ -2209,7 +2182,6 @@ def expected_postgres_references_run_results(self): 'documentation_package': '' } ], - 'empty': False, 'extra_ctes': [ {'id': 'model.test.ephemeral_copy', 'sql': cte_sql}, ], @@ -2301,7 +2273,6 @@ def expected_postgres_references_run_results(self): 'documentation_package': '' } ], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'view_summary'], @@ -2358,7 +2329,6 @@ def expected_postgres_references_run_results(self): 'depends_on': {'macros': [], 'nodes': []}, 'description': '', 'docrefs': [], - 'empty': False, 'extra_ctes': [], 'extra_ctes_injected': True, 'fqn': ['test', 'seed'], diff --git a/test/unit/test_compiler.py b/test/unit/test_compiler.py index 7a192b1472a..96184b82eb5 100644 --- a/test/unit/test_compiler.py +++ b/test/unit/test_compiler.py @@ -62,7 +62,6 @@ def test__prepend_ctes__already_has_cte(self): resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root_project', 'view'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -89,7 +88,6 @@ def test__prepend_ctes__already_has_cte(self): resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root_project', 'ephemeral'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -142,7 +140,6 @@ def test__prepend_ctes__no_ctes(self): resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root_project', 'view'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -169,7 +166,6 @@ def test__prepend_ctes__no_ctes(self): resource_type=NodeType.Model, unique_id='model.root.view_no_cte', fqn=['root_project', 'view_no_cte'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -230,7 +226,6 @@ def test__prepend_ctes(self): resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root_project', 'view'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -255,7 +250,6 @@ def test__prepend_ctes(self): resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root_project', 'ephemeral'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -309,7 +303,6 @@ def test__prepend_ctes__multiple_levels(self): resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root_project', 'view'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -334,7 +327,6 @@ def test__prepend_ctes__multiple_levels(self): resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root_project', 'ephemeral'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], @@ -359,7 +351,6 @@ def test__prepend_ctes__multiple_levels(self): resource_type=NodeType.Model, unique_id='model.root.ephemeral_level_two', fqn=['root_project', 'ephemeral_level_two'], - empty=False, package_name='root', root_path='/usr/src/app', refs=[], diff --git a/test/unit/test_context.py b/test/unit/test_context.py index 15eaa137646..2032d25cc4a 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -17,7 +17,6 @@ def setUp(self): resource_type='model', unique_id='model.root.model_one', fqn=['root', 'model_one'], - empty=False, package_name='root', original_file_path='model_one.sql', root_path='/usr/src/app', diff --git a/test/unit/test_contracts_graph_unparsed.py b/test/unit/test_contracts_graph_unparsed.py new file mode 100644 index 00000000000..45e1d92ebbd --- /dev/null +++ b/test/unit/test_contracts_graph_unparsed.py @@ -0,0 +1,116 @@ +import pytest + +from hologram import ValidationError + +from dbt.contracts.graph.unparsed import ( + UnparsedNode, UnparsedRunHook, UnparsedMacro +) +from dbt.node_types import NodeType + + +def test_unparsed_node_ok(): + node_dict = { + 'name': 'foo', + 'root_path': '/root/', + 'resource_type': NodeType.Model, + 'path': '/root/x/path.sql', + 'original_file_path': '/root/path.sql', + 'package_name': 'test', + 'raw_sql': 'select * from {{ ref("thing") }}', + } + node = UnparsedNode( + package_name='test', + root_path='/root/', + path='/root/x/path.sql', + original_file_path='/root/path.sql', + raw_sql='select * from {{ ref("thing") }}', + name='foo', + resource_type=NodeType.Model, + ) + assert UnparsedNode.from_dict(node_dict) == node + assert node.to_dict() == node_dict + assert node.empty is False + with pytest.raises(ValidationError): + UnparsedRunHook.from_dict(node_dict) + with pytest.raises(ValidationError): + UnparsedMacro.from_dict(node_dict) + + +def test_empty_unparsed_node(): + node_dict = { + 'name': 'foo', + 'root_path': '/root/', + 'resource_type': NodeType.Model, + 'path': '/root/x/path.sql', + 'original_file_path': '/root/path.sql', + 'package_name': 'test', + 'raw_sql': ' \n', + } + node = UnparsedNode( + package_name='test', + root_path='/root/', + path='/root/x/path.sql', + original_file_path='/root/path.sql', + raw_sql=' \n', + name='foo', + resource_type=NodeType.Model, + ) + assert UnparsedNode.from_dict(node_dict) == node + assert node.to_dict() == node_dict + assert node.empty is True + + +def test_unparsed_node_bad_type(): + node_dict = { + 'name': 'foo', + 'root_path': '/root/', + 'resource_type': NodeType.Source, # not valid! + 'path': '/root/x/path.sql', + 'original_file_path': '/root/path.sql', + 'package_name': 'test', + 'raw_sql': 'select * from {{ ref("thing") }}', + } + with pytest.raises(ValidationError): + UnparsedNode.from_dict(node_dict) + + +def test_unparsed_run_hook_ok(): + node_dict = { + 'name': 'foo', + 'root_path': 'test/dbt_project.yml', + 'resource_type': NodeType.Operation, + 'path': '/root/dbt_project.yml', + 'original_file_path': '/root/dbt_project.yml', + 'package_name': 'test', + 'raw_sql': 'GRANT select on dbt_postgres', + 'index': 4 + } + node = UnparsedRunHook( + package_name='test', + root_path='test/dbt_project.yml', + path='/root/dbt_project.yml', + original_file_path='/root/dbt_project.yml', + raw_sql='GRANT select on dbt_postgres', + name='foo', + resource_type=NodeType.Operation, + index=4, + ) + assert UnparsedRunHook.from_dict(node_dict) == node + assert node.to_dict() == node_dict + with pytest.raises(ValidationError): + UnparsedNode.from_dict(node_dict) + + +def test_unparsed_run_hook_bad_type(): + node_dict = { + 'name': 'foo', + 'root_path': 'test/dbt_project.yml', + 'resource_type': NodeType.Model, # invalid + 'path': '/root/dbt_project.yml', + 'original_file_path': '/root/dbt_project.yml', + 'package_name': 'test', + 'raw_sql': 'GRANT select on dbt_postgres', + 'index': 4 + } + with pytest.raises(ValidationError): + UnparsedRunHook.from_dict(node_dict) diff --git a/test/unit/test_docs_blocks.py b/test/unit/test_docs_blocks.py index a28eb1e5fcf..213d745d958 100644 --- a/test/unit/test_docs_blocks.py +++ b/test/unit/test_docs_blocks.py @@ -119,7 +119,6 @@ def test_load_file(self, system): def test_parse(self): docfile = UnparsedDocumentationFile( root_path=self.root_path, - resource_type=NodeType.Documentation, path='test_file.md', original_file_path=self.testfile_path, package_name='some_package', diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index cb0f948d537..ed47a5f990f 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -14,7 +14,7 @@ REQUIRED_PARSED_NODE_KEYS = frozenset({ - 'alias', 'empty', 'tags', 'config', 'unique_id', 'refs', 'sources', + 'alias', 'tags', 'config', 'unique_id', 'refs', 'sources', 'depends_on', 'database', 'schema', 'name', 'resource_type', 'package_name', 'root_path', 'path', 'original_file_path', 'raw_sql', 'docrefs', 'description', 'columns', 'fqn', 'build_path', 'patch_path', @@ -54,7 +54,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.snowplow.events', fqn=['snowplow', 'events'], - empty=False, package_name='snowplow', refs=[], sources=[], @@ -74,7 +73,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.events', fqn=['root', 'events'], - empty=False, package_name='root', refs=[], sources=[], @@ -94,7 +92,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.dep', fqn=['root', 'dep'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -114,7 +111,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.nested', fqn=['root', 'nested'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -134,7 +130,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.sibling', fqn=['root', 'sibling'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -154,7 +149,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.multi', fqn=['root', 'multi'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -330,7 +324,6 @@ def test_get_resource_fqns(self): resource_type='seed', unique_id='seed.root.seed', fqn=['root', 'seed'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -386,7 +379,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.snowplow.events', fqn=['snowplow', 'events'], - empty=False, package_name='snowplow', refs=[], sources=[], @@ -411,7 +403,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.events', fqn=['root', 'events'], - empty=False, package_name='root', refs=[], sources=[], @@ -436,7 +427,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.dep', fqn=['root', 'dep'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -456,7 +446,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.nested', fqn=['root', 'nested'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -476,7 +465,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.sibling', fqn=['root', 'sibling'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -496,7 +484,6 @@ def setUp(self): resource_type=NodeType.Model, unique_id='model.root.multi', fqn=['root', 'multi'], - empty=False, package_name='root', refs=[['events']], sources=[], diff --git a/test/unit/test_parser.py b/test/unit/test_parser.py index a8b51a84275..8640a4a4bd1 100644 --- a/test/unit/test_parser.py +++ b/test/unit/test_parser.py @@ -246,7 +246,6 @@ def setUp(self): unique_id='test.root.source_accepted_values_my_source_my_table_id__a__b', fqn=['root', 'schema_test', 'source_accepted_values_my_source_my_table_id__a__b'], - empty=False, package_name='root', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), @@ -270,7 +269,6 @@ def setUp(self): resource_type=NodeType.Test, unique_id='test.root.source_not_null_my_source_my_table_id', fqn=['root', 'schema_test', 'source_not_null_my_source_my_table_id'], - empty=False, package_name='root', root_path=get_os_path('/usr/src/app'), refs=[], @@ -294,7 +292,6 @@ def setUp(self): unique_id='test.root.source_relationships_my_source_my_table_id__id__ref_model_two_', # noqa fqn=['root', 'schema_test', 'source_relationships_my_source_my_table_id__id__ref_model_two_'], - empty=False, package_name='root', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), @@ -317,7 +314,6 @@ def setUp(self): resource_type=NodeType.Test, unique_id='test.snowplow.source_some_test_my_source_my_table_value', fqn=['snowplow', 'schema_test', 'source_some_test_my_source_my_table_value'], - empty=False, package_name='snowplow', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), @@ -339,7 +335,6 @@ def setUp(self): resource_type=NodeType.Test, unique_id='test.root.source_unique_my_source_my_table_id', fqn=['root', 'schema_test', 'source_unique_my_source_my_table_id'], - empty=False, package_name='root', root_path=get_os_path('/usr/src/app'), refs=[], @@ -366,7 +361,6 @@ def setUp(self): unique_id='test.root.accepted_values_model_one_id__a__b', fqn=['root', 'schema_test', 'accepted_values_model_one_id__a__b'], - empty=False, package_name='root', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), @@ -390,7 +384,6 @@ def setUp(self): resource_type=NodeType.Test, unique_id='test.root.not_null_model_one_id', fqn=['root', 'schema_test', 'not_null_model_one_id'], - empty=False, package_name='root', root_path=get_os_path('/usr/src/app'), refs=[['model_one']], @@ -414,7 +407,6 @@ def setUp(self): unique_id='test.root.relationships_model_one_id__id__ref_model_two_', # noqa fqn=['root', 'schema_test', 'relationships_model_one_id__id__ref_model_two_'], - empty=False, package_name='root', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), @@ -437,7 +429,6 @@ def setUp(self): resource_type=NodeType.Test, unique_id='test.snowplow.some_test_model_one_value', fqn=['snowplow', 'schema_test', 'some_test_model_one_value'], - empty=False, package_name='snowplow', original_file_path='test_one.yml', root_path=get_os_path('/usr/src/app'), @@ -459,7 +450,6 @@ def setUp(self): resource_type=NodeType.Test, unique_id='test.root.unique_model_one_id', fqn=['root', 'schema_test', 'unique_model_one_id'], - empty=False, package_name='root', root_path=get_os_path('/usr/src/app'), refs=[['model_one']], @@ -1001,7 +991,6 @@ def test__single_model(self): resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], - empty=False, package_name='root', original_file_path='model_one.sql', root_path=get_os_path('/usr/src/app'), @@ -1060,7 +1049,6 @@ def test__single_model__nested_configuration(self): resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'nested', 'path', 'model_one'], - empty=False, package_name='root', original_file_path='nested/path/model_one.sql', root_path=get_os_path('/usr/src/app'), @@ -1108,7 +1096,6 @@ def test__empty_model(self): resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], - empty=True, package_name='root', refs=[], sources=[], @@ -1163,7 +1150,6 @@ def test__simple_dependency(self): resource_type=NodeType.Model, unique_id='model.root.base', fqn=['root', 'base'], - empty=False, package_name='root', refs=[], sources=[], @@ -1186,7 +1172,6 @@ def test__simple_dependency(self): resource_type=NodeType.Model, unique_id='model.root.events_tx', fqn=['root', 'events_tx'], - empty=False, package_name='root', refs=[['base']], sources=[], @@ -1268,7 +1253,6 @@ def test__multiple_dependencies(self): resource_type=NodeType.Model, unique_id='model.root.events', fqn=['root', 'events'], - empty=False, package_name='root', refs=[], sources=[], @@ -1291,7 +1275,6 @@ def test__multiple_dependencies(self): resource_type=NodeType.Model, unique_id='model.root.sessions', fqn=['root', 'sessions'], - empty=False, package_name='root', refs=[], sources=[], @@ -1314,7 +1297,6 @@ def test__multiple_dependencies(self): resource_type=NodeType.Model, unique_id='model.root.events_tx', fqn=['root', 'events_tx'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -1337,7 +1319,6 @@ def test__multiple_dependencies(self): resource_type=NodeType.Model, unique_id='model.root.sessions_tx', fqn=['root', 'sessions_tx'], - empty=False, package_name='root', refs=[['sessions']], sources=[], @@ -1360,7 +1341,6 @@ def test__multiple_dependencies(self): resource_type=NodeType.Model, unique_id='model.root.multi', fqn=['root', 'multi'], - empty=False, package_name='root', refs=[['sessions_tx'], ['events_tx']], sources=[], @@ -1445,7 +1425,6 @@ def test__multiple_dependencies__packages(self): resource_type=NodeType.Model, unique_id='model.snowplow.events', fqn=['snowplow', 'events'], - empty=False, package_name='snowplow', refs=[], sources=[], @@ -1468,7 +1447,6 @@ def test__multiple_dependencies__packages(self): resource_type=NodeType.Model, unique_id='model.snowplow.sessions', fqn=['snowplow', 'sessions'], - empty=False, package_name='snowplow', refs=[], sources=[], @@ -1491,7 +1469,6 @@ def test__multiple_dependencies__packages(self): resource_type=NodeType.Model, unique_id='model.snowplow.events_tx', fqn=['snowplow', 'events_tx'], - empty=False, package_name='snowplow', refs=[['events']], sources=[], @@ -1514,7 +1491,6 @@ def test__multiple_dependencies__packages(self): resource_type=NodeType.Model, unique_id='model.snowplow.sessions_tx', fqn=['snowplow', 'sessions_tx'], - empty=False, package_name='snowplow', refs=[['sessions']], sources=[], @@ -1537,7 +1513,6 @@ def test__multiple_dependencies__packages(self): resource_type=NodeType.Model, unique_id='model.root.multi', fqn=['root', 'multi'], - empty=False, package_name='root', refs=[['snowplow', 'sessions_tx'], ['snowplow', 'events_tx']], @@ -1567,7 +1542,6 @@ def test__process_refs__packages(self): resource_type=NodeType.Model, unique_id='model.snowplow.events', fqn=['snowplow', 'events'], - empty=False, package_name='snowplow', refs=[], sources=[], @@ -1587,7 +1561,6 @@ def test__process_refs__packages(self): resource_type=NodeType.Model, unique_id='model.root.events', fqn=['root', 'events'], - empty=False, package_name='root', refs=[], sources=[], @@ -1607,7 +1580,6 @@ def test__process_refs__packages(self): resource_type=NodeType.Model, unique_id='model.root.dep', fqn=['root', 'dep'], - empty=False, package_name='root', refs=[['events']], sources=[], @@ -1643,7 +1615,6 @@ def test__process_refs__packages(self): 'resource_type': 'model', 'unique_id': 'model.snowplow.events', 'fqn': ['snowplow', 'events'], - 'empty': False, 'package_name': 'snowplow', 'docrefs': [], 'refs': [], @@ -1672,7 +1643,6 @@ def test__process_refs__packages(self): 'resource_type': 'model', 'unique_id': 'model.root.events', 'fqn': ['root', 'events'], - 'empty': False, 'package_name': 'root', 'docrefs': [], 'refs': [], @@ -1701,7 +1671,6 @@ def test__process_refs__packages(self): 'resource_type': 'model', 'unique_id': 'model.root.dep', 'fqn': ['root', 'dep'], - 'empty': False, 'package_name': 'root', 'docrefs': [], 'refs': [['events']], @@ -1757,7 +1726,6 @@ def test__in_model_config(self): resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], - empty=False, package_name='root', refs=[], sources=[], @@ -1834,7 +1802,6 @@ def test__root_project_config(self): resource_type=NodeType.Model, unique_id='model.root.table', fqn=['root', 'table'], - empty=False, package_name='root', refs=[], sources=[], @@ -1857,7 +1824,6 @@ def test__root_project_config(self): resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root', 'ephemeral'], - empty=False, package_name='root', refs=[], sources=[], @@ -1880,7 +1846,6 @@ def test__root_project_config(self): resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root', 'view'], - empty=False, package_name='root', refs=[], sources=[], @@ -2022,7 +1987,6 @@ def test__other_project_config(self): resource_type=NodeType.Model, unique_id='model.root.table', fqn=['root', 'table'], - empty=False, package_name='root', refs=[], sources=[], @@ -2045,7 +2009,6 @@ def test__other_project_config(self): resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root', 'ephemeral'], - empty=False, package_name='root', refs=[], sources=[], @@ -2068,7 +2031,6 @@ def test__other_project_config(self): resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root', 'view'], - empty=False, package_name='root', refs=[], sources=[], @@ -2091,7 +2053,6 @@ def test__other_project_config(self): resource_type=NodeType.Model, unique_id='model.snowplow.multi_sort', fqn=['snowplow', 'views', 'multi_sort'], - empty=False, package_name='snowplow', refs=[], sources=[], @@ -2123,7 +2084,6 @@ def test__other_project_config(self): depends_on=DependsOn(), config=disabled_config, tags=[], - empty=False, alias='disabled', unique_id='model.snowplow.disabled', fqn=['snowplow', 'disabled'], @@ -2144,7 +2104,6 @@ def test__other_project_config(self): depends_on=DependsOn(), config=sort_config, tags=[], - empty=False, alias='package', unique_id='model.snowplow.package', fqn=['snowplow', 'views', 'package'], @@ -2181,7 +2140,6 @@ def test__simple_data_test(self): resource_type=NodeType.Test, unique_id='test.root.no_events', fqn=['root', 'no_events'], - empty=False, package_name='root', refs=[['base']], sources=[], @@ -2300,7 +2258,6 @@ def test__simple_macro_used_in_model(self): resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], - empty=False, package_name='root', original_file_path='model_one.sql', root_path=get_os_path('/usr/src/app'), @@ -2347,7 +2304,6 @@ def test__macro_no_explicit_project_used_in_model(self): resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], - empty=False, package_name='root', root_path=get_os_path('/usr/src/app'), refs=[],