diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index e575a9d82e0..7c1cc592530 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -260,6 +260,10 @@ def load_macro_manifest(self) -> Manifest: self._macro_manifest_lazy = manifest return self._macro_manifest_lazy + def clear_macro_manifest(self): + if self._macro_manifest_lazy is not None: + self._macro_manifest_lazy = None + ### # Caching methods ### diff --git a/core/dbt/config/runtime.py b/core/dbt/config/runtime.py index e11471c9830..184e2732762 100644 --- a/core/dbt/config/runtime.py +++ b/core/dbt/config/runtime.py @@ -355,6 +355,9 @@ def load_dependencies(self) -> Mapping[str, 'RuntimeConfig']: self.dependencies = all_projects return self.dependencies + def clear_dependencies(self): + self.dependencies = None + def load_projects( self, paths: Iterable[Path] ) -> Iterator[Tuple[str, 'RuntimeConfig']]: diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index a580f6dd3d0..41f67bd637a 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -195,6 +195,8 @@ def load(self, macro_manifest: Manifest): old_results = self.read_parse_results() if old_results is not None: logger.debug('Got an acceptable cached parse result') + + self.results.macros.update(macro_manifest.macros) self.results.files.update(macro_manifest.files) diff --git a/core/dbt/perf_utils.py b/core/dbt/perf_utils.py index 4e8f74033f4..782859a88a3 100644 --- a/core/dbt/perf_utils.py +++ b/core/dbt/perf_utils.py @@ -7,7 +7,11 @@ from dbt.config import RuntimeConfig -def get_full_manifest(config: RuntimeConfig) -> Manifest: +def get_full_manifest( + config: RuntimeConfig, + *, + reset: bool = False, +) -> Manifest: """Load the full manifest, using the adapter's internal manifest if it exists to skip parsing internal (dbt + plugins) macros a second time. @@ -15,6 +19,10 @@ def get_full_manifest(config: RuntimeConfig) -> Manifest: attached to the adapter for any methods that need it. """ adapter = get_adapter(config) # type: ignore + if reset: + config.clear_dependencies() + adapter.clear_macro_manifest() + internal: Manifest = adapter.load_macro_manifest() return load_manifest( diff --git a/core/dbt/rpc/task_manager.py b/core/dbt/rpc/task_manager.py index 83ab7c0eabc..1ab24c395f5 100644 --- a/core/dbt/rpc/task_manager.py +++ b/core/dbt/rpc/task_manager.py @@ -8,6 +8,7 @@ import dbt.exceptions import dbt.flags as flags +from dbt.adapters.factory import get_adapter, reset_adapters, register_adapter from dbt.contracts.graph.manifest import Manifest from dbt.contracts.rpc import ( LastParse, @@ -126,6 +127,8 @@ def reload_manifest(self) -> bool: def reload_config(self): config = self.config.from_args(self.args) self.config = config + reset_adapters() + register_adapter(config) return config def add_request(self, request_handler: TaskHandlerProtocol): @@ -184,7 +187,7 @@ def set_parsing(self) -> bool: return True def parse_manifest(self) -> None: - self.manifest = get_full_manifest(self.config) + self.manifest = get_full_manifest(self.config, reset=True) def set_compile_exception(self, exc, logs=List[LogMessage]) -> None: assert self.last_parse.state == ManifestStatus.Compiling, \ @@ -227,6 +230,7 @@ def get_handler( return None task = self.rpc_task(method) + return task def task_table(self) -> List[TaskRow]: diff --git a/core/dbt/task/rpc/cli.py b/core/dbt/task/rpc/cli.py index e39d82ead92..a981783a0cf 100644 --- a/core/dbt/task/rpc/cli.py +++ b/core/dbt/task/rpc/cli.py @@ -104,7 +104,7 @@ def handle_request(self) -> Result: if dumped != self.args.vars: self.real_task.args.vars = dumped if isinstance(self.real_task, RemoteManifestMethod): - self.real_task.manifest = get_full_manifest(self.config) + self.real_task.manifest = get_full_manifest(self.config, reset=True) # we parsed args from the cli, so we're set on that front return self.real_task.handle_request() diff --git a/core/dbt/utils.py b/core/dbt/utils.py index 8f446cab665..aac730ebc51 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -320,7 +320,7 @@ def default(self, obj): if hasattr(obj, 'to_dict'): # if we have a to_dict we should try to serialize the result of # that! - obj = obj.to_dict() + return obj.to_dict() return super().default(obj)