Merge pull request #1589 from fishtown-analytics/feature/dataclasses-…

…jsonschema-types

dataclasses-jsonschema types

CHANGELOG.md

@@ -1,3 +1,8 @@
+## dbt ? - Louisa May Alcott
+
+### Breaking changes
+ - the undocumented "graph" variable was removed from the parsing context ([#1589](https://github.com/fishtown-analytics/dbt/pull/1589))
+
 ## dbt 0.14.0 - Wilt Chamberlain (July 10, 2019)
 
 ### Overview

core/dbt/adapters/base/connections.py

@@ -5,23 +5,26 @@
 
 import dbt.exceptions
 import dbt.flags
-from dbt.api import APIObject
 from dbt.contracts.connection import Connection
+from dbt.contracts.util import Replaceable
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.utils import translate_aliases
 
+from hologram.helpers import ExtensibleJsonSchemaMixin
 
-class Credentials(APIObject):
-    """Common base class for credentials. This is not valid to instantiate"""
-    SCHEMA = NotImplemented
-    # map credential aliases to their canonical names.
-    ALIASES = {}
+from dataclasses import dataclass, field
+from typing import Any, ClassVar, Dict, Tuple
 
-    def __init__(self, **kwargs):
-        renamed = self.translate_aliases(kwargs)
-        super().__init__(**renamed)
 
-    @property
+@dataclass
+class Credentials(
+    ExtensibleJsonSchemaMixin,
+    Replaceable,
+    metaclass=abc.ABCMeta
+):
+    _ALIASES: ClassVar[Dict[str, str]] = field(default={}, init=False)
+
+    @abc.abstractproperty
     def type(self):
         raise NotImplementedError(
             'type not implemented for base credentials class'
@@ -30,37 +33,34 @@ def type(self):
     def connection_info(self):
         """Return an ordered iterator of key/value pairs for pretty-printing.
         """
+        as_dict = self.to_dict()
         for key in self._connection_keys():
-            if key in self._contents:
-                yield key, self._contents[key]
+            if key in as_dict:
+                yield key, as_dict[key]
 
-    def _connection_keys(self):
-        """The credential object keys that should be printed to users in
-        'dbt debug' output. This is specific to each adapter.
-        """
+    @abc.abstractmethod
+    def _connection_keys(self) -> Tuple[str, ...]:
         raise NotImplementedError
 
-    def incorporate(self, **kwargs):
-        # implementation note: we have to do this here, or
-        # incorporate(alias_name=...) will result in duplicate keys in the
-        # merged dict that APIObject.incorporate() creates.
-        renamed = self.translate_aliases(kwargs)
-        return super().incorporate(**renamed)
+    @classmethod
+    def from_dict(cls, data):
+        data = cls.translate_aliases(data)
+        return super().from_dict(data)
+
+    @classmethod
+    def translate_aliases(cls, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        return translate_aliases(kwargs, cls._ALIASES)
 
-    def serialize(self, with_aliases=False):
-        serialized = super().serialize()
+    def to_dict(self, omit_none=True, validate=False, with_aliases=False):
+        serialized = super().to_dict(omit_none=omit_none, validate=validate)
         if with_aliases:
             serialized.update({
                 new_name: serialized[canonical_name]
-                for new_name, canonical_name in self.ALIASES.items()
+                for new_name, canonical_name in self._ALIASES.items()
                 if canonical_name in serialized
             })
         return serialized
 
-    @classmethod
-    def translate_aliases(cls, kwargs):
-        return translate_aliases(kwargs, cls.ALIASES)
-
 
 class BaseConnectionManager(metaclass=abc.ABCMeta):
     """Methods to implement:

core/dbt/adapters/base/relation.py

@@ -220,7 +220,7 @@ def quoted(self, identifier):
     def create_from_source(cls, source, **kwargs):
         quote_policy = dbt.utils.deep_merge(
             cls.DEFAULTS['quote_policy'],
-            source.quoting,
+            source.quoting.to_dict(),
             kwargs.get('quote_policy', {})
         )
         return cls.create(
@@ -240,9 +240,9 @@ def create_from_node(cls, config, node, table_name=None, quote_policy=None,
         quote_policy = dbt.utils.merge(config.quoting, quote_policy)
 
         return cls.create(
-            database=node.get('database'),
-            schema=node.get('schema'),
-            identifier=node.get('alias'),
+            database=node.database,
+            schema=node.schema,
+            identifier=node.alias,
             table_name=table_name,
             quote_policy=quote_policy,
             **kwargs)

core/dbt/adapters/sql/connections.py

@@ -112,7 +112,7 @@ def begin(self):
         if connection.transaction_open is True:
             raise dbt.exceptions.InternalException(
                 'Tried to begin a new transaction on connection "{}", but '
-                'it already had one open!'.format(connection.get('name')))
+                'it already had one open!'.format(connection.name))
 
         self.add_begin_query()
 

core/dbt/clients/jinja.py

@@ -68,15 +68,15 @@ def __init__(self):
         self.file_cache = {}
 
     def get_node_template(self, node):
-        key = (node['package_name'], node['original_file_path'])
+        key = (node.package_name, node.original_file_path)
 
         if key in self.file_cache:
             return self.file_cache[key]
 
         template = get_template(
-            string=node.get('raw_sql'),
+            string=node.raw_sql,
             ctx={},
-            node=node
+            node=node,
         )
         self.file_cache[key] = template
 
@@ -92,7 +92,7 @@ def clear(self):
 def macro_generator(node):
     def apply_context(context):
         def call(*args, **kwargs):
-            name = node.get('name')
+            name = node.name
             template = template_cache.get_node_template(node)
             module = template.make_module(context, False, context)
 
@@ -178,21 +178,21 @@ def __init__(self, hint=None, obj=None, name=None, exc=None):
             super().__init__(hint=hint, name=name)
             self.node = node
             self.name = name
-            self.package_name = node.get('package_name')
+            self.package_name = node.package_name
             # jinja uses these for safety, so we have to override them.
             # see https://github.com/pallets/jinja/blob/master/jinja2/sandbox.py#L332-L339 # noqa
             self.unsafe_callable = False
             self.alters_data = False
 
         def __deepcopy__(self, memo):
-            path = os.path.join(self.node.get('root_path'),
-                                self.node.get('original_file_path'))
+            path = os.path.join(self.node.root_path,
+                                self.node.original_file_path)
 
             logger.debug(
                 'dbt encountered an undefined variable, "{}" in node {}.{} '
                 '(source path: {})'
-                .format(self.name, self.node.get('package_name'),
-                        self.node.get('name'), path))
+                .format(self.name, self.node.package_name,
+                        self.node.name, path))
 
             # match jinja's message
             dbt.exceptions.raise_compiler_error(

core/dbt/compilation.py

@@ -15,13 +15,21 @@
 import dbt.flags
 import dbt.loader
 import dbt.config
-from dbt.contracts.graph.compiled import CompiledNode
+from dbt.contracts.graph.compiled import InjectedCTE, CompiledNode, \
+    CompiledTestNode
 
 from dbt.logger import GLOBAL_LOGGER as logger
 
 graph_file_name = 'graph.gpickle'
 
 
+def _compiled_type_for(model):
+    if model.resource_type == NodeType.Test:
+        return CompiledTestNode
+    else:
+        return CompiledNode
+
+
 def print_compile_stats(stats):
     names = {
         NodeType.Model: 'models',
@@ -44,9 +52,9 @@ def print_compile_stats(stats):
 
 
 def _add_prepended_cte(prepended_ctes, new_cte):
-    for dct in prepended_ctes:
-        if dct['id'] == new_cte['id']:
-            dct['sql'] = new_cte['sql']
+    for cte in prepended_ctes:
+        if cte.id == new_cte.id:
+            cte.sql = new_cte.sql
             return
     prepended_ctes.append(new_cte)
 
@@ -67,20 +75,19 @@ def recursively_prepend_ctes(model, manifest):
         return (model, model.extra_ctes, manifest)
 
     if dbt.flags.STRICT_MODE:
-        # ensure that the cte we're adding to is compiled
-        CompiledNode(**model.serialize())
+        assert isinstance(model, (CompiledNode, CompiledTestNode))
 
     prepended_ctes = []
 
     for cte in model.extra_ctes:
-        cte_id = cte['id']
+        cte_id = cte.id
         cte_to_add = manifest.nodes.get(cte_id)
         cte_to_add, new_prepended_ctes, manifest = recursively_prepend_ctes(
             cte_to_add, manifest)
         _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
-        new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.get('name'))
+        new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.name)
         sql = ' {} as (\n{}\n)'.format(new_cte_name, cte_to_add.compiled_sql)
-        _add_prepended_cte(prepended_ctes, {'id': cte_id, 'sql': sql})
+        _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte_id, sql=sql))
 
     model.prepend_ctes(prepended_ctes)
 
@@ -101,7 +108,7 @@ def compile_node(self, node, manifest, extra_context=None):
         if extra_context is None:
             extra_context = {}
 
-        logger.debug("Compiling {}".format(node.get('unique_id')))
+        logger.debug("Compiling {}".format(node.unique_id))
 
         data = node.to_dict()
         data.update({
@@ -111,14 +118,14 @@ def compile_node(self, node, manifest, extra_context=None):
             'extra_ctes': [],
             'injected_sql': None,
         })
-        compiled_node = CompiledNode(**data)
+        compiled_node = _compiled_type_for(node).from_dict(data)
 
         context = dbt.context.runtime.generate(
             compiled_node, self.config, manifest)
         context.update(extra_context)
 
         compiled_node.compiled_sql = dbt.clients.jinja.get_rendered(
-            node.get('raw_sql'),
+            node.raw_sql,
             context,
             node)
 

core/dbt/config/__init__.py

@@ -1,5 +1,5 @@
 # all these are just exports, they need "noqa" so flake8 will not complain.
 from .renderer import ConfigRenderer  # noqa
-from .profile import Profile, UserConfig, PROFILES_DIR  # noqa
+from .profile import Profile, PROFILES_DIR, read_user_config  # noqa
 from .project import Project  # noqa
 from .runtime import RuntimeConfig  # noqa