Skip to content

Commit

Permalink
Rename ContextConfig ConfigBuilder
Browse files Browse the repository at this point in the history
  • Loading branch information
gshank committed Dec 3, 2024
1 parent 3f7ee0e commit c16cbe0
Show file tree
Hide file tree
Showing 14 changed files with 187 additions and 200 deletions.
6 changes: 0 additions & 6 deletions core/dbt/artifacts/resources/v1/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,6 @@ def final_validate(self):
if self.materialized and self.materialized != "snapshot":
raise ValidationError("A snapshot must have a materialized value of 'snapshot'")

# Called by "calculate_node_config_dict" in ContextConfigGenerator
def finalize_and_validate(self):
data = self.to_dict(omit_none=True)
self.validate(data)
return self.from_dict(data)


@dataclass
class Snapshot(CompiledResource):
Expand Down
17 changes: 7 additions & 10 deletions core/dbt/context/context_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def fix_hooks(config_dict: Dict[str, Any]):
config_dict[key] = [hooks.get_hook_dict(h) for h in config_dict[key]]


class BaseContextConfigGenerator(Generic[T]):
class BaseConfigGenerator(Generic[T]):
def __init__(self, active_project: RuntimeConfig):
self._active_project = active_project

Expand Down Expand Up @@ -103,7 +103,7 @@ def generate_node_config(
) -> Dict[str, Any]: ...


class ContextConfigGenerator(BaseContextConfigGenerator[C]):
class RenderedConfigGenerator(BaseConfigGenerator[C]):
def __init__(self, active_project: RuntimeConfig):
self._active_project = active_project

Expand Down Expand Up @@ -198,7 +198,7 @@ def translate_hook_names(self, project_dict):
project_dict["post-hook"] = project_dict.pop("post_hook")
return project_dict

# ContextConfigGenerator
# RenderedConfigGenerator
def generate_node_config(
self,
config_call_dict: Dict[str, Any],
Expand All @@ -219,9 +219,7 @@ def generate_node_config(
)
fix_hooks(config_dict)
try:
# Call "finalize_and_validate" on the config obj
config_cls.validate(config_dict)
# return finalized.to_dict(omit_none=True)
config_obj = config_cls.from_dict(config_dict)
return config_obj
except ValidationError as exc:
Expand All @@ -230,7 +228,7 @@ def generate_node_config(
raise SchemaConfigError(exc, node=config_obj) from exc


class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]):
class UnrenderedConfigGenerator(BaseConfigGenerator[Dict[str, Any]]):
def get_model_configs(self, project: Project, resource_type: NodeType) -> Dict[str, Any]:
unrendered = project.unrendered.project_dict
if resource_type == NodeType.Seed:
Expand Down Expand Up @@ -267,7 +265,7 @@ def merge_config_dicts(
patch_config_dict: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
# Note: This method returns a Dict[str, Any]. This is a duplicate of
# of ContextConfigGenerator.generate_node_config, but calls methods
# of RenderedConfigGenerator.generate_node_config, but calls methods
# that deal with dictionaries instead of config object.
# Additions to one method, should probably also go in the other.

Expand Down Expand Up @@ -332,7 +330,7 @@ def _update_from_config(
return result_dict


class ContextConfig:
class ConfigBuilder:
def __init__(
self,
active_project: RuntimeConfig,
Expand All @@ -355,15 +353,14 @@ def add_unrendered_config_call(self, opts: Dict[str, Any]) -> None:
# Cannot perform complex merge behaviours on unrendered configs as they may not be appropriate types.
self._unrendered_config_call_dict.update(opts)

# ContextConfig
def build_config_dict(
self,
*,
rendered: bool = True,
patch_config_dict: Optional[dict] = None,
) -> Dict[str, Any]:
if rendered:
config_generator = ContextConfigGenerator(self._active_project) # type: ignore[var-annotated]
config_generator = RenderedConfigGenerator(self._active_project) # type: ignore[var-annotated]
config_call_dict = self._config_call_dict
else: # unrendered
config_generator = UnrenderedConfigGenerator(self._active_project) # type: ignore[assignment]
Expand Down
40 changes: 20 additions & 20 deletions core/dbt/context/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
from dbt.constants import DEFAULT_ENV_PLACEHOLDER
from dbt.context.base import Var, contextmember, contextproperty
from dbt.context.configured import FQNLookup
from dbt.context.context_config import ContextConfig
from dbt.context.context_config import ConfigBuilder
from dbt.context.exceptions_jinja import wrapped_exports
from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
from dbt.context.macros import MacroNamespace, MacroNamespaceBuilder
Expand Down Expand Up @@ -366,14 +366,14 @@ def __call__(self, *args: str) -> MetricReference:


class Config(Protocol):
def __init__(self, model, context_config: Optional[ContextConfig]): ...
def __init__(self, model, config_builder: Optional[ConfigBuilder]): ...


# Implementation of "config(..)" calls in models
class ParseConfigObject(Config):
def __init__(self, model, context_config: Optional[ContextConfig]):
def __init__(self, model, config_builder: Optional[ConfigBuilder]):
self.model = model
self.context_config = context_config
self.config_builder = config_builder

def _transform_config(self, config):
for oldkey in ("pre_hook", "post_hook"):
Expand All @@ -394,19 +394,19 @@ def __call__(self, *args, **kwargs):

opts = self._transform_config(opts)

# it's ok to have a parse context with no context config, but you must
# it's ok to have a parse context with no config builder, but you must
# not call it!
if self.context_config is None:
raise DbtRuntimeError("At parse time, did not receive a context config")
if self.config_builder is None:
raise DbtRuntimeError("At parse time, did not receive a config builder")

# Track unrendered opts to build parsed node unrendered_config later on
if get_flags().state_modified_compare_more_unrendered_values:
unrendered_config = statically_parse_unrendered_config(self.model.raw_code)
if unrendered_config:
self.context_config.add_unrendered_config_call(unrendered_config)
self.config_builder.add_unrendered_config_call(unrendered_config)

# Use rendered opts to populate context_config
self.context_config.add_config_call(opts)
# Use rendered opts to populate config builder
self.config_builder.add_config_call(opts)
return ""

def set(self, name, value):
Expand All @@ -426,7 +426,7 @@ def persist_column_docs(self) -> bool:


class RuntimeConfigObject(Config):
def __init__(self, model, context_config: Optional[ContextConfig] = None):
def __init__(self, model, config_builder: Optional[ConfigBuilder] = None):
self.model = model
# we never use or get a config, only the parser cares

Expand Down Expand Up @@ -886,7 +886,7 @@ def __init__(
config: RuntimeConfig,
manifest: Manifest,
provider: Provider,
context_config: Optional[ContextConfig],
config_builder: Optional[ConfigBuilder],
) -> None:
if provider is None:
raise DbtInternalError(f"Invalid provider given to context: {provider}")
Expand All @@ -895,7 +895,7 @@ def __init__(
self.model: Union[Macro, ManifestNode] = model
super().__init__(config, manifest, model.package_name)
self.sql_results: Dict[str, Optional[AttrDict]] = {}
self.context_config: Optional[ContextConfig] = context_config
self.config_builder: Optional[ConfigBuilder] = config_builder
self.provider: Provider = provider
self.adapter = get_adapter(self.config)
# The macro namespace is used in creating the DatabaseWrapper
Expand Down Expand Up @@ -1164,7 +1164,7 @@ def ctx_config(self) -> Config:
{%- set unique_key = config.require('unique_key') -%}
...
""" # noqa
return self.provider.Config(self.model, self.context_config)
return self.provider.Config(self.model, self.config_builder)

@contextproperty()
def execute(self) -> bool:
Expand Down Expand Up @@ -1688,12 +1688,12 @@ def generate_parser_model_context(
model: ManifestNode,
config: RuntimeConfig,
manifest: Manifest,
context_config: ContextConfig,
config_builder: ConfigBuilder,
) -> Dict[str, Any]:
# The __init__ method of ModelContext also initializes
# a ManifestContext object which creates a MacroNamespaceBuilder
# which adds every macro in the Manifest.
ctx = ModelContext(model, config, manifest, ParseProvider(), context_config)
ctx = ModelContext(model, config, manifest, ParseProvider(), config_builder)
# The 'to_dict' method in ManifestContext moves all of the macro names
# in the macro 'namespace' up to top level keys
return ctx.to_dict()
Expand Down Expand Up @@ -1895,14 +1895,14 @@ def __init__(
config: RuntimeConfig,
manifest: Manifest,
provider: Provider,
context_config: Optional[ContextConfig],
config_builder: Optional[ConfigBuilder],
macro_resolver: MacroResolver,
) -> None:
# this must be before super init so that macro_resolver exists for
# build_namespace
self.macro_resolver = macro_resolver
self.thread_ctx = MacroStack()
super().__init__(model, config, manifest, provider, context_config)
super().__init__(model, config, manifest, provider, config_builder)
self._build_test_namespace()
# We need to rebuild this because it's already been built by
# the ProviderContext with the wrong namespace.
Expand Down Expand Up @@ -1974,10 +1974,10 @@ def generate_test_context(
model: ManifestNode,
config: RuntimeConfig,
manifest: Manifest,
context_config: ContextConfig,
config_builder: ConfigBuilder,
macro_resolver: MacroResolver,
) -> Dict[str, Any]:
ctx = TestContext(model, config, manifest, ParseProvider(), context_config, macro_resolver)
ctx = TestContext(model, config, manifest, ParseProvider(), config_builder, macro_resolver)
# The 'to_dict' method in ManifestContext moves all of the macro names
# in the macro 'namespace' up to top level keys
return ctx.to_dict()
72 changes: 35 additions & 37 deletions core/dbt/parser/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from dbt.artifacts.resources import Contract
from dbt.clients.jinja import MacroGenerator, get_rendered
from dbt.config import RuntimeConfig
from dbt.context.context_config import ContextConfig
from dbt.context.context_config import ConfigBuilder
from dbt.context.providers import (
generate_generate_name_macro_context,
generate_parser_model_context,
Expand Down Expand Up @@ -209,7 +209,7 @@ def _create_parsetime_node(
self,
block: ConfiguredBlockType,
path: str,
config: ContextConfig,
config_builder: ConfigBuilder,
fqn: List[str],
name=None,
**kwargs,
Expand Down Expand Up @@ -239,7 +239,7 @@ def _create_parsetime_node(
"raw_code": block.contents,
"language": language,
"unique_id": self.generate_unique_id(name),
"config": self.config_dict(config),
"config": self.config_dict(config_builder),
"checksum": block.file.checksum.to_dict(omit_none=True),
}
dct.update(kwargs)
Expand All @@ -257,14 +257,18 @@ def _create_parsetime_node(
)
raise DictParseError(exc, node=node)

def _context_for(self, parsed_node: FinalNode, config: ContextConfig) -> Dict[str, Any]:
return generate_parser_model_context(parsed_node, self.root_project, self.manifest, config)
def _context_for(
self, parsed_node: FinalNode, config_builder: ConfigBuilder
) -> Dict[str, Any]:
return generate_parser_model_context(
parsed_node, self.root_project, self.manifest, config_builder
)

def render_with_context(self, parsed_node: FinalNode, config: ContextConfig):
# Given the parsed node and a ContextConfig to use during parsing,
def render_with_context(self, parsed_node: FinalNode, config_builder: ConfigBuilder):
# Given the parsed node and a ConfigBuilder to use during parsing,
# render the node's sql with macro capture enabled.
# Note: this mutates the config object when config calls are rendered.
context = self._context_for(parsed_node, config)
context = self._context_for(parsed_node, config_builder)

# this goes through the process of rendering, but just throws away
# the rendered result. The "macro capture" is the point?
Expand Down Expand Up @@ -308,17 +312,17 @@ def update_parsed_node_relation_names(
def update_parsed_node_config(
self,
parsed_node: FinalNode,
context_config: ContextConfig,
config_builder: ConfigBuilder,
context=None,
patch_config_dict=None,
patch_file_id=None,
) -> None:
"""Given the ContextConfig used for parsing and the parsed node,
"""Given the ConfigBuilder used for parsing and the parsed node,
generate and set the true values to use, overriding the temporary parse
values set in _build_intermediate_parsed_node.
"""

# build_config_dict takes the config_call_dict in the ContextConfig object
# build_config_dict takes the config_call_dict in the ConfigBuilder object
# and calls calculate_node_config to combine dbt_project configs and
# config calls from SQL files, plus patch configs (from schema files)
# This normalize the config for a model node due #8520; should be improved latter
Expand All @@ -330,7 +334,7 @@ def update_parsed_node_config(
):
if "materialized" not in patch_config_dict:
patch_config_dict["materialized"] = "table"
config_dict = context_config.build_config_dict(patch_config_dict=patch_config_dict)
config_dict = config_builder.build_config_dict(patch_config_dict=patch_config_dict)

# Set tags on node provided in config blocks. Tags are additive, so even if
# config has been built before, we don't have to reset tags in the parsed_node.
Expand Down Expand Up @@ -396,12 +400,12 @@ def update_parsed_node_config(

# unrendered_config is used to compare the original database/schema/alias
# values and to handle 'same_config' and 'same_contents' calls
parsed_node.unrendered_config = context_config.build_config_dict(
parsed_node.unrendered_config = config_builder.build_config_dict(
rendered=False, patch_config_dict=patch_config_dict
)

parsed_node.config_call_dict = context_config._config_call_dict
parsed_node.unrendered_config_call_dict = context_config._unrendered_config_call_dict
parsed_node.config_call_dict = config_builder._config_call_dict
parsed_node.unrendered_config_call_dict = config_builder._unrendered_config_call_dict

# do this once before we parse the node database/schema/alias, so
# parsed_node.config is what it would be if they did nothing
Expand All @@ -421,36 +425,30 @@ def update_parsed_node_config(
if not hooks:
return
if not context:
context = self._context_for(parsed_node, context_config)
context = self._context_for(parsed_node, config_builder)
for hook in hooks:
get_rendered(hook.sql, context, parsed_node, capture_macros=True)

def initial_config(self, fqn: List[str]) -> ContextConfig:
config_version = min([self.project.config_version, self.root_project.config_version])
if config_version == 2:
return ContextConfig(
self.root_project,
fqn,
self.resource_type,
self.project.project_name,
)
else:
raise DbtInternalError(
f"Got an unexpected project version={config_version}, expected 2"
)
def initial_config_builder(self, fqn: List[str]) -> ConfigBuilder:
return ConfigBuilder(
self.root_project,
fqn,
self.resource_type,
self.project.project_name,
)

def config_dict(
self,
context_config: ContextConfig,
config_builder: ConfigBuilder,
) -> Dict[str, Any]:
config_dict = context_config.build_config_dict()
config_dict = config_builder.build_config_dict()
self._mangle_hooks(config_dict)
return config_dict

def render_update(self, node: FinalNode, config: ContextConfig) -> None:
def render_update(self, node: FinalNode, config_builder: ConfigBuilder) -> None:
try:
context = self.render_with_context(node, config)
self.update_parsed_node_config(node, config, context=context)
context = self.render_with_context(node, config_builder)
self.update_parsed_node_config(node, config_builder, context=context)
except ValidationError as exc:
# we got a ValidationError - probably bad types in config()
raise ConfigUpdateError(exc, node=node) from exc
Expand All @@ -465,15 +463,15 @@ def parse_node(self, block: ConfiguredBlockType) -> FinalNode:
compiled_path: str = self.get_compiled_path(block)
fqn = self.get_fqn(compiled_path, block.name)

config: ContextConfig = self.initial_config(fqn)
config_builder: ConfigBuilder = self.initial_config_builder(fqn)

node = self._create_parsetime_node(
block=block,
path=compiled_path,
config=config,
config_builder=config_builder,
fqn=fqn,
)
self.render_update(node, config)
self.render_update(node, config_builder)
self.add_result_node(block, node)
return node

Expand Down
Loading

0 comments on commit c16cbe0

Please sign in to comment.