Support for v1/v2 config blocks

- Backwards compatiblity shims from v2 to v1 configs
- config merging for v2
- compatibility shim for parsing/contexts
- defer var lookups as late as possible
- fixed ContextConfigType to be a proper Union of the two context config types
- Fix adapter configs to be proper dataclasses
- fix unused config paths error
- make v2 config parsing not alter its source data

core/dbt/adapters/base/__init__.py

@@ -10,5 +10,5 @@
     SchemaSearchMap,
 )
 from dbt.adapters.base.column import Column  # noqa
-from dbt.adapters.base.impl import BaseAdapter  # noqa
+from dbt.adapters.base.impl import AdapterConfig, BaseAdapter  # noqa
 from dbt.adapters.base.plugin import AdapterPlugin  # noqa

core/dbt/config/project.py

@@ -1,7 +1,9 @@
 from copy import deepcopy
 from dataclasses import dataclass, field
 from itertools import chain
-from typing import List, Dict, Any, Optional, TypeVar, Union, Tuple, Callable
+from typing import (
+    List, Dict, Any, Optional, TypeVar, Union, Tuple, Callable, Mapping
+)
 from typing_extensions import Protocol
 
 import hashlib
@@ -20,7 +22,7 @@
 from dbt.semver import VersionSpecifier
 from dbt.semver import versions_compatible
 from dbt.version import get_installed_version
-from dbt.utils import deep_map
+from dbt.utils import deep_map, MultiDict
 from dbt.legacy_config_updater import ConfigUpdater, IsFQNResource
 
 from dbt.contracts.project import (
@@ -222,7 +224,7 @@ class VarProvider(Protocol):
     """Var providers are tied to a particular Project."""
     def vars_for(
         self, node: IsFQNResource, adapter_type: str
-    ) -> Dict[str, Any]:
+    ) -> Mapping[str, Any]:
         raise NotImplementedError(
             f'vars_for not implemented for {type(self)}!'
         )
@@ -247,7 +249,7 @@ def __init__(
 
     def vars_for(
         self, node: IsFQNResource, adapter_type: str
-    ) -> Dict[str, Any]:
+    ) -> Mapping[str, Any]:
         updater = ConfigUpdater(adapter_type)
         return updater.get_project_config(node, self).get('vars', {})
 
@@ -267,11 +269,13 @@ def __init__(
 
     def vars_for(
         self, node: IsFQNResource, adapter_type: str
-    ) -> Dict[str, Any]:
-        # in v2, vars are only scoped to the project.
-        updater = ConfigUpdater(adapter_type)
-        node_project = self.vars.get(node.package_name, {})
-        return updater.get_project_vars(node_project)
+    ) -> Mapping[str, Any]:
+        # in v2, vars are only either project or globally scoped
+
+        merged = MultiDict([self.vars])
+        if node.package_name in self.vars:
+            merged.add(self.vars.get(node.package_name, {}))
+        return merged
 
     def to_dict(self):
         return self.vars
@@ -613,3 +617,53 @@ def validate_version(self):
                 ]
             )
             raise DbtProjectError(msg)
+
+    def as_v1(self):
+        if self.config_version == 1:
+            return self
+
+        dct = self.to_project_config()
+
+        mutated = deepcopy(dct)
+        # remove sources, it doesn't exist
+        mutated.pop('sources', None)
+
+        common_config_keys = ['models', 'seeds', 'snapshots']
+
+        if 'vars' in dct and isinstance(dct['vars'], dict):
+            # stuff any 'vars' entries into the old-style
+            # models/seeds/snapshots dicts
+            for project_name, items in dct['vars'].items():
+                for cfgkey in ['models', 'seeds', 'snapshots']:
+                    if project_name not in mutated[cfgkey]:
+                        mutated[cfgkey][project_name] = {}
+                    project_type_cfg = mutated[cfgkey][project_name]
+                    if 'vars' not in project_type_cfg:
+                        project_type_cfg['vars'] = {}
+                    mutated[cfgkey][project_name]['vars'].update(items)
+            # remove this from the v1 form
+            mutated.pop('vars')
+        # ok, now we want to look through all the existing cfgkeys and mirror
+        # it, except anything under 'config' gets included directly.
+        for cfgkey in common_config_keys:
+            if cfgkey not in dct:
+                continue
+
+            mutated[cfgkey] = _flatten_config(dct[cfgkey])
+        mutated['config-version'] = 1
+        project = Project.from_project_config(mutated)
+        project.packages = self.packages
+        return project
+
+
+def _flatten_config(dct: Dict[str, Any]):
+    result = {}
+    for key, value in dct.items():
+        if isinstance(value, dict):
+            if key == 'config':
+                result.update(value)
+            else:
+                result[key] = _flatten_config(value)
+        else:
+            result[key] = value
+    return result

core/dbt/config/runtime.py

@@ -49,7 +49,11 @@ def __post_init__(self):
 
     @classmethod
     def from_parts(
-        cls, project: Project, profile: Profile, args: Any,
+        cls,
+        project: Project,
+        profile: Profile,
+        args: Any,
+        dependencies: Optional[Mapping[str, Project]] = None,
     ) -> 'RuntimeConfig':
         """Instantiate a RuntimeConfig from its components.
 
@@ -100,6 +104,7 @@ def from_parts(
             credentials=profile.credentials,
             args=args,
             cli_vars=cli_vars,
+            dependencies=dependencies,
         )
 
     def new_project(self, project_root: str) -> 'RuntimeConfig':
@@ -195,15 +200,27 @@ def get_metadata(self) -> ManifestMetadata:
             adapter_type=self.credentials.type
         )
 
-    def _get_config_paths(
+    def _get_v2_config_paths(
         self,
-        config: Dict[str, Any],
-        path: FQNPath = (),
-        paths: Optional[MutableSet[FQNPath]] = None,
+        config,
+        path: FQNPath,
+        paths: MutableSet[FQNPath],
     ) -> PathSet:
-        if paths is None:
-            paths = set()
+        for key, value in config.items():
+            if isinstance(value, dict):
+                if key == 'config':
+                    paths.add(path)
+                else:
+                    self._get_v2_config_paths(value, path + (key,), paths)
 
+        return frozenset(paths)
+
+    def _get_v1_config_paths(
+        self,
+        config: Dict[str, Any],
+        path: FQNPath,
+        paths: MutableSet[FQNPath],
+    ) -> PathSet:
         keys = ConfigUpdater(self.credentials.type).ConfigKeys
 
         for key, value in config.items():
@@ -212,13 +229,27 @@ def _get_config_paths(
                     if path not in paths:
                         paths.add(path)
                 else:
-                    self._get_config_paths(value, path + (key,), paths)
+                    self._get_v1_config_paths(value, path + (key,), paths)
             else:
                 if path not in paths:
                     paths.add(path)
 
         return frozenset(paths)
 
+    def _get_config_paths(
+        self,
+        config: Dict[str, Any],
+        path: FQNPath = (),
+        paths: Optional[MutableSet[FQNPath]] = None,
+    ) -> PathSet:
+        if paths is None:
+            paths = set()
+
+        if self.config_version == 2:
+            return self._get_v2_config_paths(config, path, paths)
+        else:
+            return self._get_v1_config_paths(config, path, paths)
+
     def get_resource_config_paths(self) -> Dict[str, PathSet]:
         """Return a dictionary with 'seeds' and 'models' keys whose values are
         lists of lists of strings, where each inner list of strings represents
@@ -309,6 +340,17 @@ def _get_project_directories(self) -> Iterator[Path]:
                 if path.is_dir() and not path.name.startswith('__'):
                     yield path
 
+    def as_v1(self):
+        if self.config_version == 1:
+            return self
+
+        return self.from_parts(
+            project=Project.as_v1(self),
+            profile=self,
+            args=self.args,
+            dependencies=self.dependencies,
+        )
+
 
 class UnsetCredentials(Credentials):
     def __init__(self):
@@ -386,7 +428,11 @@ def to_target_dict(self):
 
     @classmethod
     def from_parts(
-        cls, project: Project, profile: Any, args: Any,
+        cls,
+        project: Project,
+        profile: Profile,
+        args: Any,
+        dependencies: Optional[Mapping[str, Project]] = None,
     ) -> 'RuntimeConfig':
         """Instantiate a RuntimeConfig from its components.
 
@@ -431,6 +477,7 @@ def from_parts(
             credentials=UnsetCredentials(),
             args=args,
             cli_vars=cli_vars,
+            dependencies=dependencies,
         )
 
     @classmethod

core/dbt/context/base.py

@@ -1,7 +1,7 @@
 import json
 import os
 from typing import (
-    Any, Dict, NoReturn, Optional
+    Any, Dict, NoReturn, Optional, Mapping
 )
 
 from dbt import flags
@@ -100,16 +100,16 @@ class Var:
 
     def __init__(
         self,
-        context: Dict[str, Any],
-        cli_vars: Dict[str, Any],
+        context: Mapping[str, Any],
+        cli_vars: Mapping[str, Any],
         node: Optional[CompiledResource] = None
     ) -> None:
-        self.context: Dict[str, Any] = context
-        self.cli_vars: Dict[str, Any] = cli_vars
+        self.context: Mapping[str, Any] = context
+        self.cli_vars: Mapping[str, Any] = cli_vars
         self.node: Optional[CompiledResource] = node
-        self.merged: Dict[str, Any] = self._generate_merged()
+        self.merged: Mapping[str, Any] = self._generate_merged()
 
-    def _generate_merged(self) -> Dict[str, Any]:
+    def _generate_merged(self) -> Mapping[str, Any]:
         return self.cli_vars
 
     @property
@@ -120,7 +120,8 @@ def node_name(self):
             return '<Configuration>'
 
     def get_missing_var(self, var_name):
-        pretty_vars = json.dumps(self.merged, sort_keys=True, indent=4)
+        dct = {k: self.merged[k] for k in self.merged}
+        pretty_vars = json.dumps(dct, sort_keys=True, indent=4)
         msg = self.UndefinedVarError.format(
             var_name, self.node_name, pretty_vars
         )