From c981c70243937d543299ff79227560df4fba6e01 Mon Sep 17 00:00:00 2001 From: Devon Fulcher <24593113+DevonFulcher@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:27:41 -0600 Subject: [PATCH 1/2] Meta config for dimensions measures and entities --- core/dbt/contracts/graph/unparsed.py | 3 + core/dbt/parser/schema_yaml_readers.py | 28 +++++- tests/functional/semantic_models/fixtures.py | 88 +++++++++++++++++++ .../test_semantic_model_configs.py | 88 +++++++++++++++++++ 4 files changed, 204 insertions(+), 3 deletions(-) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index ab17ced5db9..8199ac4e9a2 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -668,6 +668,7 @@ class UnparsedEntity(dbtClassMixin): label: Optional[str] = None role: Optional[str] = None expr: Optional[str] = None + config: Dict[str, Any] = field(default_factory=dict) @dataclass @@ -688,6 +689,7 @@ class UnparsedMeasure(dbtClassMixin): non_additive_dimension: Optional[UnparsedNonAdditiveDimension] = None agg_time_dimension: Optional[str] = None create_metric: bool = False + config: Dict[str, Any] = field(default_factory=dict) @dataclass @@ -705,6 +707,7 @@ class UnparsedDimension(dbtClassMixin): is_partition: bool = False type_params: Optional[UnparsedDimensionTypeParams] = None expr: Optional[str] = None + config: Dict[str, Any] = field(default_factory=dict) @dataclass diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index aca239db153..4598dc200a4 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from typing import Any, Dict, List, Optional, Union from dbt.artifacts.resources import ( @@ -21,6 +22,7 @@ WhereFilter, WhereFilterIntersection, ) +from dbt.artifacts.resources.v1.semantic_model import SemanticLayerElementConfig from dbt.clients.jinja import get_rendered from dbt.context.context_config import ( BaseContextConfigGenerator, @@ -536,6 +538,7 @@ def _get_dimensions(self, unparsed_dimensions: List[UnparsedDimension]) -> List[ type_params=self._get_dimension_type_params(unparsed=unparsed.type_params), expr=unparsed.expr, metadata=None, # TODO: requires a fair bit of parsing context + config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})), ) ) return dimensions @@ -551,6 +554,7 @@ def _get_entities(self, unparsed_entities: List[UnparsedEntity]) -> List[Entity] label=unparsed.label, role=unparsed.role, expr=unparsed.expr, + config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})), ) ) @@ -583,6 +587,7 @@ def _get_measures(self, unparsed_measures: List[UnparsedMeasure]) -> List[Measur unparsed.non_additive_dimension ), agg_time_dimension=unparsed.agg_time_dimension, + config=SemanticLayerElementConfig(meta=unparsed.config.get("meta", {})), ) ) return measures @@ -638,6 +643,10 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None: fqn = self.schema_parser.get_fqn_prefix(path) fqn.append(unparsed.name) + entities = self._get_entities(unparsed.entities) + measures = self._get_measures(unparsed.measures) + dimensions = self._get_dimensions(unparsed.dimensions) + config = self._generate_semantic_model_config( target=unparsed, fqn=fqn, @@ -645,6 +654,19 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None: rendered=True, ) + # Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs + elements: Sequence[Union[Dimension, Entity, Measure]] = [ + *dimensions, + *entities, + *measures, + ] + for element in elements: + if config is not None: + if element.config is None: + element.config = SemanticLayerElementConfig(meta=config.meta) + else: + element.config.meta = {**config.get("meta", {}), **element.config.meta} + config = config.finalize_and_validate() unrendered_config = self._generate_semantic_model_config( @@ -666,9 +688,9 @@ def parse_semantic_model(self, unparsed: UnparsedSemanticModel) -> None: path=path, resource_type=NodeType.SemanticModel, unique_id=unique_id, - entities=self._get_entities(unparsed.entities), - measures=self._get_measures(unparsed.measures), - dimensions=self._get_dimensions(unparsed.dimensions), + entities=entities, + measures=measures, + dimensions=dimensions, defaults=unparsed.defaults, primary_entity=unparsed.primary_entity, config=config, diff --git a/tests/functional/semantic_models/fixtures.py b/tests/functional/semantic_models/fixtures.py index 7788067e91d..8b15e97b99b 100644 --- a/tests/functional/semantic_models/fixtures.py +++ b/tests/functional/semantic_models/fixtures.py @@ -403,3 +403,91 @@ type_params: measure: sum_of_things """ + +semantic_model_dimensions_entities_measures_meta_config = """ +version: 2 + +semantic_models: + - name: semantic_people + label: "Semantic People" + model: ref('people') + dimensions: + - name: favorite_color + label: "Favorite Color" + type: categorical + config: + meta: + dimension: one + - name: created_at + label: "Created At" + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + label: "Years Tenure" + agg: SUM + expr: tenure + config: + meta: + measure: two + - name: people + label: "People" + agg: count + expr: id + entities: + - name: id + label: "Primary ID" + type: primary + config: + meta: + entity: three + defaults: + agg_time_dimension: created_at +""" + +semantic_model_meta_clobbering_yml = """ +version: 2 + +semantic_models: + - name: semantic_people + label: "Semantic People" + model: ref('people') + config: + meta: + model_level: "should_be_inherited" + component_level: "should_be_overridden" + dimensions: + - name: favorite_color + label: "Favorite Color" + type: categorical + config: + meta: + component_level: "dimension_override" + - name: created_at + label: "Created At" + type: TIME + type_params: + time_granularity: day + measures: + - name: years_tenure + label: "Years Tenure" + agg: SUM + expr: tenure + config: + meta: + component_level: "measure_override" + - name: people + label: "People" + agg: count + expr: id + entities: + - name: id + label: "Primary ID" + type: primary + config: + meta: + component_level: "entity_override" + defaults: + agg_time_dimension: created_at +""" diff --git a/tests/functional/semantic_models/test_semantic_model_configs.py b/tests/functional/semantic_models/test_semantic_model_configs.py index cac38e92bd3..c036101f201 100644 --- a/tests/functional/semantic_models/test_semantic_model_configs.py +++ b/tests/functional/semantic_models/test_semantic_model_configs.py @@ -11,6 +11,8 @@ metricflow_time_spine_sql, models_people_metrics_yml, models_people_sql, + semantic_model_dimensions_entities_measures_meta_config, + semantic_model_meta_clobbering_yml, semantic_model_people_yml, ) @@ -225,3 +227,89 @@ def test_meta_config(self, project): sm_node = manifest.semantic_models[sm_id] meta_expected = {"my_meta": "testing", "my_other_meta": "testing more"} assert sm_node.config.meta == meta_expected + + +# test meta configs on semantic model components (dimensions, measures, entities) +class TestMetaConfigForComponents: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_dimensions_entities_measures_meta_config, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + def test_component_meta_configs(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + sm_id = "semantic_model.test.semantic_people" + assert sm_id in manifest.semantic_models + sm_node = manifest.semantic_models[sm_id] + + # Check dimension meta config + favorite_color_dim = next(d for d in sm_node.dimensions if d.name == "favorite_color") + assert favorite_color_dim.config.meta == {"dimension": "one"} + + # Check measure meta config + years_tenure_measure = next(m for m in sm_node.measures if m.name == "years_tenure") + assert years_tenure_measure.config.meta == {"measure": "two"} + + # Check entity meta config + id_entity = next(e for e in sm_node.entities if e.name == "id") + assert id_entity.config.meta == {"entity": "three"} + + +# test meta config clobbering behavior between semantic model and component levels +class TestMetaConfigClobbering: + @pytest.fixture(scope="class") + def models(self): + return { + "people.sql": models_people_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "semantic_models.yml": semantic_model_meta_clobbering_yml, + "people_metrics.yml": models_people_metrics_yml, + "groups.yml": groups_yml, + } + + def test_meta_config_clobbering(self, project): + run_dbt(["parse"]) + manifest = get_manifest(project.project_root) + sm_id = "semantic_model.test.semantic_people" + assert sm_id in manifest.semantic_models + sm_node = manifest.semantic_models[sm_id] + + # Check semantic model level meta config + assert sm_node.config.meta == { + "model_level": "should_be_inherited", + "component_level": "should_be_overridden", + } + + # Check dimension inherits model-level meta and overrides component-level meta + favorite_color_dim = next(d for d in sm_node.dimensions if d.name == "favorite_color") + assert favorite_color_dim.config.meta == { + "model_level": "should_be_inherited", + "component_level": "dimension_override", + } + + # Check measure inherits model-level meta and overrides component-level meta + years_tenure_measure = next(m for m in sm_node.measures if m.name == "years_tenure") + assert years_tenure_measure.config.meta == { + "model_level": "should_be_inherited", + "component_level": "measure_override", + } + + # Check entity inherits model-level meta and overrides component-level meta + id_entity = next(e for e in sm_node.entities if e.name == "id") + assert id_entity.config.meta == { + "model_level": "should_be_inherited", + "component_level": "entity_override", + } + + # Check component without meta config still inherits model-level meta + created_at_dim = next(d for d in sm_node.dimensions if d.name == "created_at") + assert created_at_dim.config.meta == { + "model_level": "should_be_inherited", + "component_level": "should_be_overridden", + } From 52a4b666dd565dce9905fffe906385a8268d8d7c Mon Sep 17 00:00:00 2001 From: Devon Fulcher <24593113+DevonFulcher@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:28:35 -0600 Subject: [PATCH 2/2] changie --- .changes/unreleased/Features-20250106-132829.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20250106-132829.yaml diff --git a/.changes/unreleased/Features-20250106-132829.yaml b/.changes/unreleased/Features-20250106-132829.yaml new file mode 100644 index 00000000000..739933745da --- /dev/null +++ b/.changes/unreleased/Features-20250106-132829.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Meta config for dimensions measures and entities +time: 2025-01-06T13:28:29.176439-06:00 +custom: + Author: DevonFulcher + Issue: None