Merge pull request #2675 from fishtown-analytics/feature/python-adapt…

…er-macro

Port the adapter macro to python

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## dbt 0.18.0 (Release TBD)
 
+### Breaking changes
+- `adapter_macro` is no longer a macro, instead it is a builtin context method. Any custom macros that intercepted it by going through `context['dbt']` will need to instead access it via `context['builtins']` ([#2302](https://github.com/fishtown-analytics/dbt/issues/2302), [#2673](https://github.com/fishtown-analytics/dbt/pull/2673))
+
 ## dbt 0.18.0b2 (July 30, 2020)
 
 ### Features

core/dbt/adapters/base/query_headers.py

@@ -3,7 +3,7 @@
 
 from dbt.clients.jinja import QueryStringGenerator
 
-from dbt.context.configured import generate_query_header_context
+from dbt.context.manifest import generate_query_header_context
 from dbt.contracts.connection import AdapterRequiredConfig, QueryComment
 from dbt.contracts.graph.compiled import CompileResultNode
 from dbt.contracts.graph.manifest import Manifest

core/dbt/adapters/factory.py

@@ -1,7 +1,7 @@
 import threading
 from pathlib import Path
 from importlib import import_module
-from typing import Type, Dict, Any, List, Optional
+from typing import Type, Dict, Any, List, Optional, Set
 
 from dbt.exceptions import RuntimeException, InternalException
 from dbt.include.global_project import (
@@ -124,15 +124,15 @@ def cleanup_connections(self):
             for adapter in self.adapters.values():
                 adapter.cleanup_connections()
 
-    def get_adapter_package_names(self, name: Optional[str]) -> List[str]:
+    def get_adapter_plugins(self, name: Optional[str]) -> List[AdapterPlugin]:
+        """Iterate over the known adapter plugins. If a name is provided,
+        iterate in dependency order over the named plugin and its dependencies.
+        """
         if name is None:
-            # the important thing is that the global project is last.
-            return sorted(
-                self.packages,
-                key=lambda k: k == GLOBAL_PROJECT_NAME
-            )
-        package_names: List[str] = []
-        # slice into a list instead of using a set + pop(), to preserve order
+            return list(self.plugins.values())
+
+        plugins: List[AdapterPlugin] = []
+        seen: Set[str] = set()
         plugin_names: List[str] = [name]
         while plugin_names:
             plugin_name = plugin_names[0]
@@ -143,12 +143,19 @@ def get_adapter_package_names(self, name: Optional[str]) -> List[str]:
                 raise InternalException(
                     f'No plugin found for {plugin_name}'
                 ) from None
-            package_names.append(plugin.project_name)
+            plugins.append(plugin)
+            seen.add(plugin_name)
             if plugin.dependencies is None:
                 continue
             for dep in plugin.dependencies:
-                if dep not in package_names:
+                if dep not in seen:
                     plugin_names.append(dep)
+        return plugins
+
+    def get_adapter_package_names(self, name: Optional[str]) -> List[str]:
+        package_names: List[str] = [
+            p.project_name for p in self.get_adapter_plugins(name)
+        ]
         package_names.append(GLOBAL_PROJECT_NAME)
         return package_names
 
@@ -164,6 +171,9 @@ def get_include_paths(self, name: Optional[str]) -> List[Path]:
             paths.append(path)
         return paths
 
+    def get_adapter_type_names(self, name: Optional[str]) -> List[str]:
+        return [p.adapter.type() for p in self.get_adapter_plugins(name)]
+
 
 FACTORY: AdapterContainer = AdapterContainer()
 
@@ -211,3 +221,7 @@ def get_include_paths(name: Optional[str]) -> List[Path]:
 
 def get_adapter_package_names(name: Optional[str]) -> List[str]:
     return FACTORY.get_adapter_package_names(name)
+
+
+def get_adapter_type_names(name: Optional[str]) -> List[str]:
+    return FACTORY.get_adapter_type_names(name)

core/dbt/context/configured.py

@@ -1,16 +1,11 @@
-from typing import Any, Dict, Iterable, Union, Optional, List
+from typing import Any, Dict
 
-from dbt.clients.jinja import MacroGenerator, MacroStack
 from dbt.contracts.connection import AdapterRequiredConfig
-from dbt.contracts.graph.manifest import Manifest
-from dbt.contracts.graph.parsed import ParsedMacro
-from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME
 from dbt.node_types import NodeType
 from dbt.utils import MultiDict
 
 from dbt.context.base import contextproperty, Var
 from dbt.context.target import TargetContext
-from dbt.exceptions import raise_duplicate_macro_name
 
 
 class ConfiguredContext(TargetContext):
@@ -81,150 +76,6 @@ def var(self) -> ConfiguredVar:
         )
 
 
-FlatNamespace = Dict[str, MacroGenerator]
-NamespaceMember = Union[FlatNamespace, MacroGenerator]
-FullNamespace = Dict[str, NamespaceMember]
-
-
-class MacroNamespace:
-    def __init__(
-        self,
-        root_package: str,
-        search_package: str,
-        thread_ctx: MacroStack,
-        internal_packages: List[str],
-        node: Optional[Any] = None,
-    ) -> None:
-        self.root_package = root_package
-        self.search_package = search_package
-        self.internal_package_names = set(internal_packages)
-        self.internal_package_names_order = internal_packages
-        self.globals: FlatNamespace = {}
-        self.locals: FlatNamespace = {}
-        self.internal_packages: Dict[str, FlatNamespace] = {}
-        self.packages: Dict[str, FlatNamespace] = {}
-        self.thread_ctx = thread_ctx
-        self.node = node
-
-    def _add_macro_to(
-        self,
-        heirarchy: Dict[str, FlatNamespace],
-        macro: ParsedMacro,
-        macro_func: MacroGenerator,
-    ):
-        if macro.package_name in heirarchy:
-            namespace = heirarchy[macro.package_name]
-        else:
-            namespace = {}
-            heirarchy[macro.package_name] = namespace
-
-        if macro.name in namespace:
-            raise_duplicate_macro_name(
-                macro_func.macro, macro, macro.package_name
-            )
-        heirarchy[macro.package_name][macro.name] = macro_func
-
-    def add_macro(self, macro: ParsedMacro, ctx: Dict[str, Any]):
-        macro_name: str = macro.name
-
-        macro_func: MacroGenerator = MacroGenerator(
-            macro, ctx, self.node, self.thread_ctx
-        )
-
-        # internal macros (from plugins) will be processed separately from
-        # project macros, so store them in a different place
-        if macro.package_name in self.internal_package_names:
-            self._add_macro_to(self.internal_packages, macro, macro_func)
-        else:
-            self._add_macro_to(self.packages, macro, macro_func)
-
-            if macro.package_name == self.search_package:
-                self.locals[macro_name] = macro_func
-            elif macro.package_name == self.root_package:
-                self.globals[macro_name] = macro_func
-
-    def add_macros(self, macros: Iterable[ParsedMacro], ctx: Dict[str, Any]):
-        for macro in macros:
-            self.add_macro(macro, ctx)
-
-    def get_macro_dict(self) -> FullNamespace:
-        root_namespace: FullNamespace = {}
-
-        # add everything in the 'dbt' namespace to the root namespace
-        # overwriting any duplicates. Iterate in reverse-order because the
-        # packages that are first in the list are the ones we want to "win".
-        global_project_namespace = {}
-        for pkg in reversed(self.internal_package_names_order):
-            macros = self.internal_packages.get(pkg, {})
-            global_project_namespace.update(macros)
-            # these can then be overwitten by globals/locals
-            root_namespace.update(macros)
-
-        root_namespace[GLOBAL_PROJECT_NAME] = global_project_namespace
-        root_namespace.update(self.packages)
-        root_namespace.update(self.globals)
-        root_namespace.update(self.locals)
-
-        return root_namespace
-
-
-class ManifestContext(ConfiguredContext):
-    """The Macro context has everything in the target context, plus the macros
-    in the manifest.
-
-    The given macros can override any previous context values, which will be
-    available as if they were accessed relative to the package name.
-    """
-    def __init__(
-        self,
-        config: AdapterRequiredConfig,
-        manifest: Manifest,
-        search_package: str,
-    ) -> None:
-        super().__init__(config)
-        self.manifest = manifest
-        self.search_package = search_package
-        self.macro_stack = MacroStack()
-
-    def _get_namespace(self):
-        # avoid an import loop
-        from dbt.adapters.factory import get_adapter_package_names
-        internal_packages = get_adapter_package_names(
-            self.config.credentials.type
-        )
-        return MacroNamespace(
-            self.config.project_name,
-            self.search_package,
-            self.macro_stack,
-            internal_packages,
-            None,
-        )
-
-    def get_macros(self) -> Dict[str, Any]:
-        nsp = self._get_namespace()
-        nsp.add_macros(self.manifest.macros.values(), self._ctx)
-        return nsp.get_macro_dict()
-
-    def to_dict(self) -> Dict[str, Any]:
-        dct = super().to_dict()
-        dct.update(self.get_macros())
-        return dct
-
-
-class QueryHeaderContext(ManifestContext):
-    def __init__(
-        self, config: AdapterRequiredConfig, manifest: Manifest
-    ) -> None:
-        super().__init__(config, manifest, config.project_name)
-
-
-def generate_query_header_context(
-    config: AdapterRequiredConfig, manifest: Manifest
-):
-    ctx = QueryHeaderContext(config, manifest)
-    return ctx.to_dict()
-
-
 def generate_schema_yml(
     config: AdapterRequiredConfig, project_name: str
 ) -> Dict[str, Any]: