Merge pull request #223 from jwills/jwills_smarter_python_module_stuff

Add an option to the profile for specifying additional entries for sys.path

dbt/adapters/duckdb/credentials.py

@@ -120,6 +120,12 @@ class DuckDBCredentials(Credentials):
     # local DuckDB files, but this is a way to override that behavior)
     keep_open: bool = False
 
+    # A list of paths to Python modules that should be loaded into the
+    # running Python environment when dbt is invoked; this is useful for
+    # loading custom dbt-duckdb plugins or locally defined modules that
+    # provide helper functions for dbt Python models.
+    module_paths: Optional[List[str]] = None
+
     @classmethod
     def __pre_deserialize__(cls, data: Dict[Any, Any]) -> Dict[Any, Any]:
         data = super().__pre_deserialize__(data)

dbt/adapters/duckdb/environments/__init__.py

@@ -1,6 +1,7 @@
 import abc
 import importlib.util
 import os
+import sys
 import tempfile
 from typing import Dict
 from typing import Optional
@@ -39,6 +40,12 @@ class Environment(abc.ABC):
     def __init__(self, creds: DuckDBCredentials):
         self._creds = creds
 
+        # Add any module paths to the Python path for the environment
+        if creds.module_paths:
+            for path in creds.module_paths:
+                if path not in sys.path:
+                    sys.path.append(path)
+
     @property
     def creds(self) -> DuckDBCredentials:
         return self._creds

dbt/adapters/duckdb/plugins/__init__.py

@@ -1,4 +1,5 @@
 import importlib
+import os
 from typing import Any
 from typing import Dict
 from typing import Optional
@@ -23,6 +24,11 @@ class BasePlugin:
     contains a name and its configuration.
     """
 
+    # A set of built-in plugins that are included with dbt-duckdb.
+    _BUILTIN = set(
+        [x.split(".")[0] for x in os.listdir(os.path.dirname(__file__)) if "_" not in x]
+    )
+
     @classmethod
     def create(
         cls,
@@ -43,9 +49,7 @@ def create(
         if not isinstance(module, str):
             raise TypeError("Module name must be a string.")
 
-        if "." not in module:
-            # if the module does not have a dot, assume it is a builtin
-            # plugin module and prepend the path name
+        if module in cls._BUILTIN:
             name = module
             module = f"dbt.adapters.duckdb.plugins.{module}"
         else: