diff --git a/olivertwist/rules/no_references_outside_of_its_staging_area.py b/olivertwist/rules/no_references_outside_of_its_staging_area.py index a05fcad..b357437 100644 --- a/olivertwist/rules/no_references_outside_of_its_staging_area.py +++ b/olivertwist/rules/no_references_outside_of_its_staging_area.py @@ -17,16 +17,16 @@ def no_references_outside_of_its_staging_area( manifest: Manifest, ) -> Tuple[List[Node], List[Node]]: - def detect_staging_references_another_staging_model_in_a_different_area(node: Node): - invalid_refs = [ + def staging_depends_on_staging_in_another_area(node: Node): + different_staging_area_refs = [ p for p in manifest.graph.predecessors(node.id) if manifest.get_node(p).is_staging if not manifest.get_node(p).area == node.area ] - return node.is_staging and len(list(invalid_refs)) > 0 + return node.is_staging and len(list(different_staging_area_refs)) > 0 passes, failures = partition( - detect_staging_references_another_staging_model_in_a_different_area, manifest.nodes() + staging_depends_on_staging_in_another_area, manifest.nodes() ) return list(passes), list(failures) diff --git a/olivertwist/rules/no_references_to_marts_from_staging.py b/olivertwist/rules/no_references_to_marts_from_staging.py index 01618bd..151c9a7 100644 --- a/olivertwist/rules/no_references_to_marts_from_staging.py +++ b/olivertwist/rules/no_references_to_marts_from_staging.py @@ -17,7 +17,7 @@ def no_references_to_marts_from_staging( manifest: Manifest, ) -> Tuple[List[Node], List[Node]]: - def detect_staging_depends_on_mart(node: Node): + def staging_depends_on_mart(node: Node): mart_refs = [ p for p in manifest.graph.predecessors(node.id) @@ -26,6 +26,6 @@ def detect_staging_depends_on_mart(node: Node): return node.is_staging and len(list(mart_refs)) > 0 passes, failures = partition( - detect_staging_depends_on_mart, manifest.nodes() + staging_depends_on_mart, manifest.nodes() ) return list(passes), list(failures) diff --git a/olivertwist/rules/no_references_to_source_from_marts.py b/olivertwist/rules/no_references_to_source_from_marts.py new file mode 100644 index 0000000..416ad90 --- /dev/null +++ b/olivertwist/rules/no_references_to_source_from_marts.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +"""Staging models should have a single source. + +Copyright (C) 2020, Auto Trader UK +Created 22. Dec 2020 08:40 + +""" + +from typing import List, Tuple + +from olivertwist.manifest import Manifest, Node +from olivertwist.ruleengine.rule import rule +from olivertwist.rules.utils import partition + + +@rule(id="no-references-to-source-from-marts", name="No references to source from marts") +def no_references_to_source_from_marts( + manifest: Manifest, +) -> Tuple[List[Node], List[Node]]: + def mart_depends_on_source(node: Node): + source_refs = [ + p + for p in manifest.graph.predecessors(node.id) + if manifest.get_node(p).is_source + ] + return node.is_mart and len(list(source_refs)) > 0 + + passes, failures = partition( + mart_depends_on_source, manifest.nodes() + ) + return list(passes), list(failures) diff --git a/tests/rules/test_no_references_to_source_from_marts.py b/tests/rules/test_no_references_to_source_from_marts.py new file mode 100644 index 0000000..d97ef46 --- /dev/null +++ b/tests/rules/test_no_references_to_source_from_marts.py @@ -0,0 +1,51 @@ +import pytest + +from olivertwist.manifest import Manifest +from olivertwist.rules.no_references_to_source_from_marts import ( + no_references_to_source_from_marts, +) + + +@pytest.fixture +def manifest() -> Manifest: + return Manifest( + { + "nodes": { + "source_1": { + "unique_id": "source_1", + "resource_type": "source", + "fqn": ["foo", "bar"], + }, + "staging_1": { + "unique_id": "staging_1", + "resource_type": "model", + "fqn": ["foo", "staging", "bar"], + }, + "mart_1": { + "unique_id": "mart_1", + "resource_type": "model", + "fqn": ["foo", "marts", "bar"], + }, + }, + "child_map": { + "source_1": ["mart_1"], + "staging_1": [], + "mart_1": [], + }, + "disabled": [], + "sources": { + "source_1": {} + }, + } + + ) + + +def test_no_references_to_source_from_marts(manifest): + passes, failures = no_references_to_source_from_marts(manifest) + + pass_ids = [p.id for p in passes] + failure_ids = [f.id for f in failures] + + assert pass_ids == ["source_1", "staging_1"] + assert failure_ids == ["mart_1"]