Basic local FS functionality

core/dbt/clients/git.py

@@ -2,7 +2,7 @@
 import os.path
 
 from dbt.clients.system import run_cmd
-from dbt.clients.storage_adapter import StorageAdapter as SA
+from dbt.clients.storage import adapter as SA
 from dbt.logger import GLOBAL_LOGGER as logger
 import dbt.exceptions
 from packaging import version
@@ -43,7 +43,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec
         run_cmd(os.path.join(cwd, dirname or ''), ['git', 'sparse-checkout', 'set', subdirectory])
 
     if remove_git_dir:
-        SA.adapter.rmdir(os.path.join(dirname, '.git'))
+        SA.rmdir(os.path.join(dirname, '.git'))
 
     return result
 

core/dbt/clients/storage.py

@@ -0,0 +1,27 @@
+from importlib import import_module
+from os import getenv
+from dbt.logger import GLOBAL_LOGGER as logger
+from .storage_adapter_type import StorageAdapterType
+from typing import cast
+
+# TODO:
+# ensure configured adapter has the correct module signature
+# provide better not-ready error messagin
+
+# get configured storage adapter
+_module_to_load = getenv(
+    "DBT_STORAGE_ADAPTER", "dbt.adapters.internal_storage.local_filesystem"
+)
+
+# load module if it exists
+try:
+    _adapter = cast(StorageAdapterType, import_module(_module_to_load))
+    logger.debug(f"Storage adapter {_module_to_load} loaded")
+except ModuleNotFoundError:
+    logger.error(f"Storage adapter {_module_to_load} not found")
+
+# run ready check
+if not _adapter.ready_check():
+    logger.error(f"Storage Adapter{_module_to_load} not ready")
+else:
+    adapter = _adapter

core/dbt/clients/storage_adapter_type.py

@@ -0,0 +1,37 @@
+from typing import Any, Dict, Union
+from typing_extensions import Literal
+
+
+class StorageAdapterType:
+    """Class that defines type hinting for storage adapters.
+
+    This is needed because importlib (used in the storage client) and mypy aren't exactly friends.
+    N.B: https://stackoverflow.com/questions/48976499/mypy-importlib-module-functions
+
+    """
+
+    @staticmethod
+    def ready_check() -> bool:
+        pass
+
+    @staticmethod
+    def read(path: str, strip: bool = ...) -> str:
+        pass
+
+    @staticmethod
+    def write(
+        path: str, content: Union[str, Dict[str, Any], None], overwrite: bool = ...
+    ) -> bool:
+        pass
+
+    @staticmethod
+    def delete(path: str) -> bool:
+        pass
+
+    @staticmethod
+    def find() -> None:
+        pass
+
+    @staticmethod
+    def info(path: str) -> Union[dict, Literal[False]]:
+        pass

core/dbt/compilation.py

@@ -8,7 +8,7 @@
 from dbt import flags
 from dbt.adapters.factory import get_adapter
 from dbt.clients import jinja
-from dbt.clients.storage_adapter import StorageAdapter as SA
+from dbt.clients.storage import adapter as SA
 from dbt.context.providers import generate_runtime_model
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.compiled import (
@@ -153,8 +153,8 @@ def __init__(self, config):
         self.config = config
 
     def initialize(self):
-        SA.adapter.make_directory(self.config.target_path)
-        SA.adapter.make_directory(self.config.modules_path)
+        SA.write(self.config.target_path, None)
+        SA.write(self.config.modules_path, None)
 
     # creates a ModelContext which is converted to
     # a dict for jinja rendering of SQL

core/dbt/config/profile.py

@@ -4,7 +4,7 @@
 
 from dbt.dataclass_schema import ValidationError
 
-from dbt.clients.storage_adapter import StorageAdapter as SA
+from dbt.clients.storage import adapter as SA
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import Credentials, HasCredentials
 from dbt.contracts.project import ProfileConfig, UserConfig
@@ -52,7 +52,7 @@ def read_profile(profiles_dir: str) -> Dict[str, Any]:
     contents = None
     if os.path.isfile(path):
         try:
-            contents = SA.adapter.load_file_contents(path, strip=False)  # type: ignore
+            contents = SA.read(path, strip=False)
             yaml_content = load_yaml_text(contents)
             if not yaml_content:
                 msg = f'The profiles.yml file at {path} is empty'

core/dbt/config/project.py

@@ -9,7 +9,8 @@
 import hashlib
 import os
 
-from dbt.clients.storage_adapter import StorageAdapter as SA
+from dbt.clients.storage import adapter as SA
+from dbt.clients import system
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import QueryComment
 from dbt.exceptions import DbtProjectError
@@ -75,16 +76,16 @@ class IsFQNResource(Protocol):
 
 
 def _load_yaml(path):
-    contents = SA.adapter.load_file_contents(path)
+    contents = SA.read(path)
     return load_yaml_text(contents)
 
 
 def package_data_from_root(project_root):
-    package_filepath = SA.adapter.resolve_path_from_base(
+    package_filepath = system.resolve_path_from_base(
         'packages.yml', project_root
     )
 
-    if SA.adapter.path_exists(package_filepath):
+    if SA.info(package_filepath):
         packages_dict = _load_yaml(package_filepath)
     else:
         packages_dict = None
@@ -147,7 +148,7 @@ def _raw_project_from(project_root: str) -> Dict[str, Any]:
     project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml')
 
     # get the project.yml contents
-    if not SA.adapter.path_exists(project_yaml_filepath):  # type: ignore # noqa
+    if not SA.info(project_yaml_filepath):
         raise DbtProjectError(
             'no dbt_project.yml found at expected path {}'
             .format(project_yaml_filepath)

core/dbt/config/selectors.py

@@ -7,7 +7,8 @@
 
 from .renderer import SelectorRenderer
 
-from dbt.clients.storage_adapter import StorageAdapter as SA
+from dbt.clients.storage import adapter as SA
+from dbt.clients import system
 from dbt.contracts.selection import SelectorFile
 from dbt.exceptions import DbtSelectorsError, RuntimeException
 from dbt.graph import parse_from_selectors_definition, SelectionSpec
@@ -71,7 +72,7 @@ def from_path(
         cls, path: Path, renderer: SelectorRenderer,
     ) -> 'SelectorConfig':
         try:
-            data = load_yaml_text(SA.adapter.load_file_contents(str(path)))  # type: ignore
+            data = load_yaml_text(SA.read(str(path)))  # type: ignore
         except (ValidationError, RuntimeException) as exc:
             raise DbtSelectorsError(
                 f'Could not read selector file: {exc}',
@@ -87,12 +88,12 @@ def from_path(
 
 
 def selector_data_from_root(project_root: str) -> Dict[str, Any]:
-    selector_filepath = SA.adapter.resolve_path_from_base(  # type: ignore
+    selector_filepath = system.resolve_path_from_base(  # type: ignore
         'selectors.yml', project_root
     )
 
-    if SA.adapter.path_exists(selector_filepath):  # type: ignore
-        selectors_dict = load_yaml_text(SA.adapter.load_file_contents(selector_filepath))  # type: ignore # noqa
+    if SA.info(selector_filepath):
+        selectors_dict = load_yaml_text(SA.read(selector_filepath))  # type: ignore # noqa
     else:
         selectors_dict = None
     return selectors_dict

core/dbt/deps/base.py

@@ -4,7 +4,7 @@
 from contextlib import contextmanager
 from typing import List, Optional, Generic, TypeVar
 
-from dbt.clients import system
+from dbt.clients.storage import adapter as SA
 from dbt.contracts.project import ProjectPackageMetadata
 from dbt.logger import GLOBAL_LOGGER as logger
 
@@ -30,13 +30,13 @@ def downloads_directory():
         DOWNLOADS_PATH = tempfile.mkdtemp(prefix='dbt-downloads-')
         remove_downloads = True
 
-    system.make_directory(DOWNLOADS_PATH)
+    SA.write(DOWNLOADS_PATH, None)
     logger.debug("Set downloads directory='{}'".format(DOWNLOADS_PATH))
 
     yield DOWNLOADS_PATH
 
     if remove_downloads:
-        system.rmtree(DOWNLOADS_PATH)
+        SA.delete(DOWNLOADS_PATH)
         DOWNLOADS_PATH = None
 
 

core/dbt/deps/registry.py

@@ -3,6 +3,7 @@
 
 from dbt import semver
 from dbt.clients import registry, system
+from dbt.clients.storage import adapter as SA
 from dbt.contracts.project import (
     RegistryPackageMetadata,
     RegistryPackage,
@@ -58,7 +59,7 @@ def install(self, project, renderer):
         tar_path = os.path.realpath(
             os.path.join(get_downloads_path(), tar_name)
         )
-        system.make_directory(os.path.dirname(tar_path))
+        SA.write(os.path.dirname(tar_path), None)
 
         download_url = metadata.downloads.tarball
         system.download(download_url, tar_path)

core/dbt/parser/manifest.py

@@ -20,7 +20,7 @@
 from dbt.node_types import NodeType
 from dbt.clients.jinja import get_rendered, MacroStack
 from dbt.clients.jinja_static import statically_extract_macro_calls
-from dbt.clients.system import make_directory
+from dbt.clients.storage import adapter as SA
 from dbt.config import Project, RuntimeConfig
 from dbt.context.docs import generate_runtime_docs
 from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
@@ -435,9 +435,7 @@ def write_manifest_for_partial_parse(self):
                             PARTIAL_PARSE_FILE_NAME)
         try:
             manifest_msgpack = self.manifest.to_msgpack()
-            make_directory(os.path.dirname(path))
-            with open(path, 'wb') as fp:
-                fp.write(manifest_msgpack)
+            SA.write(path, manifest_msgpack)
         except Exception:
             raise
 

core/dbt/parser/read_files.py

@@ -1,4 +1,4 @@
-from dbt.clients.system import load_file_contents
+from dbt.clients.storage import adapter as SA
 from dbt.contracts.files import (
     FilePath, ParseFileType, SourceFile, FileHash, AnySourceFile, SchemaSourceFile
 )
@@ -12,7 +12,7 @@
 def load_source_file(
         path: FilePath, parse_file_type: ParseFileType,
         project_name: str) -> AnySourceFile:
-    file_contents = load_file_contents(path.absolute_path, strip=False)
+    file_contents = SA.read(path.absolute_path, strip=False)
     checksum = FileHash.from_contents(file_contents)
     sf_cls = SchemaSourceFile if parse_file_type == ParseFileType.Schema else SourceFile
     source_file = sf_cls(path=path, checksum=checksum,
@@ -54,7 +54,7 @@ def load_seed_source_file(match: FilePath, project_name) -> SourceFile:
         # We don't want to calculate a hash of this file. Use the path.
         source_file = SourceFile.big_seed(match)
     else:
-        file_contents = load_file_contents(match.absolute_path, strip=False)
+        file_contents = SA.read(match.absolute_path, strip=False)
         checksum = FileHash.from_contents(file_contents)
         source_file = SourceFile(path=match, checksum=checksum)
         source_file.contents = ''

core/dbt/task/debug.py

@@ -7,6 +7,7 @@
 from dbt.logger import GLOBAL_LOGGER as logger
 import dbt.clients.system
 import dbt.exceptions
+from dbt.clients.storage import adapter as SA
 from dbt.adapters.factory import get_adapter, register_adapter
 from dbt.config import Project, Profile, PROFILES_DIR
 from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
@@ -255,7 +256,7 @@ def _load_profile(self):
 
         try:
             raw_profile_data = load_yaml_text(
-                dbt.clients.system.load_file_contents(self.profile_path)
+                SA.read(self.profile_path)
             )
         except Exception:
             pass  # we'll report this when we try to load the profile for real

core/dbt/task/deps.py

@@ -9,7 +9,7 @@
 from dbt.deps.resolver import resolve_packages
 
 from dbt.logger import GLOBAL_LOGGER as logger
-from dbt.clients import system
+from dbt.clients.storage import adapter as SA
 
 from dbt.task.base import BaseTask, move_to_nearest_project_dir
 
@@ -43,7 +43,7 @@ def track_package_install(
         )
 
     def run(self):
-        system.make_directory(self.config.modules_path)
+        SA.write(self.config.modules_path, None)
         packages = self.config.packages.packages
         if not packages:
             logger.info('Warning: No packages were found in packages.yml')

core/dbt/task/init.py

@@ -3,6 +3,7 @@
 
 import dbt.config
 import dbt.clients.system
+from dbt.clients.storage import adapter as SA
 from dbt.version import _get_adapter_plugin_names
 from dbt.adapters.factory import load_plugin, get_include_paths
 
@@ -50,7 +51,7 @@ def create_profiles_dir(self, profiles_dir):
         if not os.path.exists(profiles_dir):
             msg = "Creating dbt configuration folder at {}"
             logger.info(msg.format(profiles_dir))
-            dbt.clients.system.make_directory(profiles_dir)
+            SA.write(profiles_dir, None)
             return True
         return False