feature/mx-1744 combine ruleset and extracted item ingestion (#249)

### Changes

- update mex-common to 0.50.0
- GraphConnector.ingest now accepts rule-set requests as well
- BREAKING: GraphConnector.ingest returns a list of ingested models,
instead of ids
- POST /ingest now accepts rule-set requests as well as extracted items
- BREAKING: POST /ingest returns a container of ingested models, instead
of ids

### Removed

- remove GraphConnector.create_rule_set, in favor of combined ingest
method
- remove unused ingest_extracted_items_into_graph helper
- remove unused BulkIngestRequest and BulkIngestResponse

CHANGELOG.md

@@ -11,14 +11,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changes
 
-- update mex-common to 0.48.0
+- update mex-common to 0.50.0
+- GraphConnector.ingest now accepts rule-set requests as well
+- BREAKING: GraphConnector.ingest returns a list of ingested models, instead of ids
+- POST /ingest now accepts rule-set requests as well as extracted items
+- BREAKING: POST /ingest returns a container of ingested models, instead of ids
 
 ### Deprecated
 
 ### Removed
 
 - remove backend settings that were just duplicating common settings
 - removed BackendIdentityProvider enum, because it is now included in common
+- remove GraphConnector.create_rule_set, in favor of combined ingest method
+- remove unused ingest_extracted_items_into_graph helper
+- remove unused BulkIngestRequest and BulkIngestResponse
 
 ### Fixed
 

mex/backend/graph/connector.py

@@ -34,7 +34,7 @@
     RULE_MODEL_CLASSES_BY_NAME,
     AnyExtractedModel,
     AnyRuleModel,
-    AnyRuleSetRequest,
+    AnyRuleSetResponse,
     BasePrimarySource,
     ExtractedPrimarySource,
 )
@@ -138,9 +138,9 @@ def _seed_indices(self) -> Result:
             },
         )
 
-    def _seed_data(self) -> list[Identifier]:
+    def _seed_data(self) -> None:
         """Ensure the primary source `mex` is seeded and linked to itself."""
-        return self.ingest([cast(ExtractedPrimarySource, MEX_EXTRACTED_PRIMARY_SOURCE)])
+        self.ingest([cast(ExtractedPrimarySource, MEX_EXTRACTED_PRIMARY_SOURCE)])
 
     def close(self) -> None:
         """Close the connector's underlying requests session."""
@@ -511,31 +511,9 @@ def _merge_edges(
             raise InconsistentGraphError(msg)
         return result
 
-    def create_rule_set(
-        self,
-        model: AnyRuleSetRequest,
-        stable_target_id: Identifier,
-    ) -> None:
-        """Create a new rule set to be applied to one merged item.
-
-        This is a two-step process: first the rule and merged items are created
-        along with their nested objects (like Text and Link); then all edges that
-        represent references (like hadPrimarySource, parentUnit, etc.) are added to
-        the graph in a second step.
-
-        Args:
-            model: A rule-set model
-            stable_target_id: Identifier of the merged item
-        """
-        for rule in (model.additive, model.subtractive, model.preventive):
-            self._merge_item(rule, stable_target_id)
-            self._merge_edges(
-                rule,
-                stable_target_id,
-                extra_refs={"stableTargetId": stable_target_id},
-            )
-
-    def ingest(self, models: list[AnyExtractedModel]) -> list[Identifier]:
+    def ingest(
+        self, models: list[AnyExtractedModel | AnyRuleSetResponse]
+    ) -> list[AnyExtractedModel | AnyRuleSetResponse]:
         """Ingest a list of models into the graph as nodes and connect all edges.
 
         This is a two-step process: first all extracted and merged items are created
@@ -550,12 +528,28 @@ def ingest(self, models: list[AnyExtractedModel]) -> list[Identifier]:
             List of identifiers of the ingested models
         """
         for model in models:
-            self._merge_item(model, model.stableTargetId, identifier=model.identifier)
+            if isinstance(model, AnyRuleSetResponse):
+                for rule in (model.additive, model.subtractive, model.preventive):
+                    self._merge_item(rule, model.stableTargetId)
+            else:
+                self._merge_item(
+                    model, model.stableTargetId, identifier=model.identifier
+                )
 
         for model in models:
-            self._merge_edges(model, model.stableTargetId, identifier=model.identifier)
+            if isinstance(model, AnyRuleSetResponse):
+                for rule in (model.additive, model.subtractive, model.preventive):
+                    self._merge_edges(
+                        rule,
+                        model.stableTargetId,
+                        extra_refs={"stableTargetId": model.stableTargetId},
+                    )
+            else:
+                self._merge_edges(
+                    model, model.stableTargetId, identifier=model.identifier
+                )
 
-        return [m.identifier for m in models]
+        return models
 
     def flush(self) -> None:
         """Flush the database (only in debug mode)."""

mex/backend/ingest/main.py

@@ -1,13 +1,22 @@
 from fastapi import APIRouter
 from starlette import status
 
-from mex.backend.ingest.helpers import ingest_extracted_items_into_graph
-from mex.backend.ingest.models import BulkIngestRequest, BulkIngestResponse
+from mex.backend.graph.connector import GraphConnector
+from mex.common.models import (
+    AnyExtractedModel,
+    AnyRuleSetResponse,
+    ItemsContainer,
+)
 
 router = APIRouter()
 
 
 @router.post("/ingest", status_code=status.HTTP_201_CREATED, tags=["extractors"])
-def ingest_extracted_items(request: BulkIngestRequest) -> BulkIngestResponse:
-    """Ingest a batch of extracted items and return their identifiers."""
-    return ingest_extracted_items_into_graph(request.items)
+def ingest(
+    request: ItemsContainer[AnyExtractedModel | AnyRuleSetResponse],
+) -> ItemsContainer[AnyExtractedModel | AnyRuleSetResponse]:
+    """Ingest a batch of extracted items or rule-sets and return them."""
+    connector = GraphConnector.get()
+    return ItemsContainer[AnyExtractedModel | AnyRuleSetResponse](
+        items=connector.ingest(request.items)
+    )

mex/backend/rules/helpers.py

@@ -64,7 +64,15 @@ def create_and_get_rule_set(
         stable_target_id = Identifier.generate()
 
     connector = GraphConnector.get()
-    connector.create_rule_set(rule_set, stable_target_id)
+    response_class_name = ensure_postfix(rule_set.stemType, "RuleSetResponse")
+    response_class = RULE_SET_RESPONSE_CLASSES_BY_NAME[response_class_name]
+    rule_set_response = response_class(
+        additive=rule_set.additive,
+        preventive=rule_set.preventive,
+        subtractive=rule_set.subtractive,
+        stableTargetId=stable_target_id,
+    )
+    connector.ingest([rule_set_response])
     rule_types = [
         rule_set.additive.entityType,
         rule_set.subtractive.entityType,

pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "fastapi>=0.115,<1",
     "httpx>=0.27,<1",
     "jinja2>=3,<4",
-    "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.48.0",
+    "mex-common @ git+https://github.com/robert-koch-institut/mex-common.git@0.50.0",
     "neo4j>=5,<6",
     "pydantic>=2,<3",
     "starlette>=0.41,<1",

tests/conftest.py

@@ -364,6 +364,15 @@ def organizational_unit_rule_set_request(
     return OrganizationalUnitRuleSetRequest(additive=additive_organizational_unit)
 
 
+@pytest.fixture
+def organizational_unit_rule_set_response(
+    additive_organizational_unit: AdditiveOrganizationalUnit,
+) -> OrganizationalUnitRuleSetResponse:
+    return OrganizationalUnitRuleSetResponse(
+        additive=additive_organizational_unit, stableTargetId=Identifier.generate(42)
+    )
+
+
 @pytest.fixture
 def load_dummy_rule_set(
     organizational_unit_rule_set_request: OrganizationalUnitRuleSetRequest,