diff --git a/core/database_arango.py b/core/database_arango.py index 066aea462..37de574b9 100644 --- a/core/database_arango.py +++ b/core/database_arango.py @@ -115,9 +115,7 @@ def connect( self.db.collection("indicators").add_persistent_index( fields=["name", "type"], unique=True ) - self.db.collection("dfiq").add_persistent_index( - fields=["dfiq_id", "type"], unique=True - ) + self.db.collection("dfiq").add_persistent_index(fields=["uuid"], unique=True) def clear(self, truncate=True): if not self.db: diff --git a/core/schemas/dfiq.py b/core/schemas/dfiq.py index 00078628d..a4ebcb9c1 100644 --- a/core/schemas/dfiq.py +++ b/core/schemas/dfiq.py @@ -102,7 +102,8 @@ class DFIQBase(YetiModel, database_arango.ArangoYetiConnector): _root_type: Literal["dfiq"] = "dfiq" name: str = Field(min_length=1) - dfiq_id: str = Field(min_length=1) + uuid: str # = Field(default_factory=lambda: str(uuid.uuid4())) + dfiq_id: str | None = None dfiq_version: str = Field(min_length=1) dfiq_tags: list[str] | None = None contributors: list[str] | None = None @@ -142,7 +143,7 @@ def parse_yaml(cls, yaml_string: str) -> dict[str, Any]: if "id" not in yaml_data: raise ValueError(f"Invalid DIFQ YAML (missing 'id' attribute): {yaml_data}") - if not re.match("^\d+\.\d+\.\d+$", str(yaml_data.get("dfiq_version", ""))): + if not re.match(r"^\d+\.\d+\.\d+$", str(yaml_data.get("dfiq_version", ""))): raise ValueError(f"Invalid DFIQ version: {yaml_data['dfiq_version']}") return yaml_data @@ -156,27 +157,31 @@ def to_yaml(self) -> str: dump = self.model_dump( exclude={"created", "modified", "id", "root_type", "dfiq_yaml"} ) - dump.pop("internal") dump["type"] = dump["type"].removeprefix("DFIQType.") dump["display_name"] = dump.pop("name") dump["tags"] = dump.pop("dfiq_tags") dump["id"] = dump.pop("dfiq_id") + dump["uuid"] = dump.pop("uuid") if dump["contributors"] is None: dump.pop("contributors") return yaml.dump(dump) def update_parents(self) -> None: intended_parent_ids = None - if hasattr(self, "parent_ids"): + if getattr(self, "parent_ids", []): intended_parent_ids = self.parent_ids - elif self.type == DFIQType.approach: - intended_parent_ids = [self.dfiq_id.split(".")[0]] + elif self.type == DFIQType.approach and self.parent_id: + intended_parent_ids = [self.parent_id] else: return - intended_parents = [ - DFIQBase.find(dfiq_id=parent_id) for parent_id in intended_parent_ids - ] + intended_parents = [] + for parent_id in intended_parent_ids: + parent = DFIQBase.find(dfiq_id=parent_id) + if not parent: + parent = DFIQBase.find(uuid=parent_id) + intended_parents.append(parent) + if not all(intended_parents): raise ValueError( f"Missing parent(s) {intended_parent_ids} for {self.dfiq_id}" @@ -190,7 +195,9 @@ def update_parents(self) -> None: continue if rel.target != self.extended_id: continue - if vertices[rel.source].dfiq_id not in intended_parent_ids: + if ( + vertices[rel.source].dfiq_id and vertices[rel.source].uuid + ) not in intended_parent_ids: rel.delete() for parent in intended_parents: @@ -209,19 +216,20 @@ def from_yaml(cls: Type["DFIQScenario"], yaml_string: str) -> "DFIQScenario": if yaml_data["type"] != "scenario": raise ValueError(f"Invalid type for DFIQ scenario: {yaml_data['type']}") # use re.match to check that DFIQ Ids for scenarios start with S[0-1]\d+ - if not re.match(r"^S[0-1]\d+$", yaml_data["id"] or ""): + if yaml_data.get("id") and not re.match(r"^S[0-1]\d+$", yaml_data["id"] or ""): raise ValueError( f"Invalid DFIQ ID for scenario: {yaml_data['id']}. Must be in the format S[0-1]\d+" ) return cls( name=yaml_data["display_name"], description=yaml_data["description"], + uuid=yaml_data["uuid"], dfiq_id=yaml_data["id"], dfiq_version=yaml_data["dfiq_version"], dfiq_tags=yaml_data.get("tags"), contributors=yaml_data.get("contributors"), dfiq_yaml=yaml_string, - internal=yaml_data["id"][1] == "0", + internal=yaml_data.get("internal", True), ) @@ -237,7 +245,7 @@ def from_yaml(cls: Type["DFIQFacet"], yaml_string: str) -> "DFIQFacet": yaml_data = cls.parse_yaml(yaml_string) if yaml_data["type"] != "facet": raise ValueError(f"Invalid type for DFIQ facet: {yaml_data['type']}") - if not re.match(r"^F[0-1]\d+$", yaml_data["id"] or ""): + if yaml_data.get("id") and not re.match(r"^F[0-1]\d+$", yaml_data["id"] or ""): raise ValueError( f"Invalid DFIQ ID for facet: {yaml_data['id']}. Must be in the format F[0-1]\d+" ) @@ -245,13 +253,14 @@ def from_yaml(cls: Type["DFIQFacet"], yaml_string: str) -> "DFIQFacet": return cls( name=yaml_data["display_name"], description=yaml_data.get("description"), + uuid=yaml_data["uuid"], dfiq_id=yaml_data["id"], dfiq_version=yaml_data["dfiq_version"], dfiq_tags=yaml_data.get("tags"), contributors=yaml_data.get("contributors"), parent_ids=yaml_data["parent_ids"], dfiq_yaml=yaml_string, - internal=yaml_data["id"][1] == "0", + internal=yaml_data.get("internal", True), ) @@ -267,7 +276,7 @@ def from_yaml(cls: Type["DFIQQuestion"], yaml_string: str) -> "DFIQQuestion": yaml_data = cls.parse_yaml(yaml_string) if yaml_data["type"] != "question": raise ValueError(f"Invalid type for DFIQ question: {yaml_data['type']}") - if not re.match(r"^Q[0-1]\d+$", yaml_data["id"] or ""): + if yaml_data.get("id") and not re.match(r"^Q[0-1]\d+$", yaml_data["id"] or ""): raise ValueError( f"Invalid DFIQ ID for question: {yaml_data['id']}. Must be in the format Q[0-1]\d+" ) @@ -275,13 +284,14 @@ def from_yaml(cls: Type["DFIQQuestion"], yaml_string: str) -> "DFIQQuestion": return cls( name=yaml_data["display_name"], description=yaml_data.get("description"), + uuid=yaml_data["uuid"], dfiq_id=yaml_data["id"], dfiq_version=yaml_data["dfiq_version"], dfiq_tags=yaml_data.get("tags"), contributors=yaml_data.get("contributors"), parent_ids=yaml_data["parent_ids"], dfiq_yaml=yaml_string, - internal=yaml_data["id"][1] == "0", + internal=yaml_data.get("internal", True), ) @@ -313,8 +323,7 @@ class DFIQProcessors(BaseModel): class DFIQApproachDescription(BaseModel): - summary: str = Field(min_length=1) - details: str = Field(min_length=1) + details: str = "" references: list[str] = [] references_internal: list[str] | None = None @@ -336,13 +345,14 @@ class DFIQApproach(DFIQBase): description: DFIQApproachDescription view: DFIQApproachView type: Literal[DFIQType.approach] = DFIQType.approach + parent_id: str | None = None @classmethod def from_yaml(cls: Type["DFIQApproach"], yaml_string: str) -> "DFIQApproach": yaml_data = cls.parse_yaml(yaml_string) if yaml_data["type"] != "approach": raise ValueError(f"Invalid type for DFIQ approach: {yaml_data['type']}") - if not re.match(r"^Q[0-1]\d+\.\d+$", yaml_data["id"]): + if yaml_data.get("id") and not re.match(r"^Q[0-1]\d+\.\d+$", yaml_data["id"]): raise ValueError( f"Invalid DFIQ ID for approach: {yaml_data['id']}. Must be in the format Q[0-1]\d+.\d+" ) @@ -355,17 +365,18 @@ def from_yaml(cls: Type["DFIQApproach"], yaml_string: str) -> "DFIQApproach": f"Invalid DFIQ view for approach (has to be an object): {yaml_data['view']}" ) - internal = bool(re.match(r"^Q[0-1]\d+\.0\d+$", yaml_data["id"])) return cls( name=yaml_data["display_name"], description=DFIQApproachDescription(**yaml_data["description"]), view=DFIQApproachView(**yaml_data["view"]), + uuid=yaml_data["uuid"], dfiq_id=yaml_data["id"], dfiq_version=yaml_data["dfiq_version"], dfiq_tags=yaml_data.get("tags"), + parent_id=yaml_data.get("parent_id"), contributors=yaml_data.get("contributors"), dfiq_yaml=yaml_string, - internal=internal, + internal=yaml_data.get("internal", True), ) diff --git a/core/web/apiv2/dfiq.py b/core/web/apiv2/dfiq.py index bf1febb73..4d15bef32 100644 --- a/core/web/apiv2/dfiq.py +++ b/core/web/apiv2/dfiq.py @@ -56,17 +56,43 @@ class DFIQSearchResponse(BaseModel): total: int +class DFIQConfigResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + approach_data_sources: list[str] + approach_analysis_step_types: list[str] + + # API endpoints router = APIRouter() +@router.get("/config") +async def config() -> DFIQConfigResponse: + all_approaches = dfiq.DFIQApproach.list() + + data_sources = set() + analysis_step_types = set() + + for approach in all_approaches: + data_sources.update({data.type for data in approach.view.data}) + for processor in approach.view.processors: + for analysis in processor.analysis: + analysis_step_types.update({step.type for step in analysis.steps}) + + return DFIQConfigResponse( + approach_data_sources=sorted(list(data_sources)), + approach_analysis_step_types=sorted(list(analysis_step_types)), + ) + + @router.post("/from_archive") async def from_archive(archive: UploadFile) -> dict[str, int]: """Uncompresses a ZIP archive and processes the DFIQ content inside it.""" - tempdir = tempfile.TemporaryDirectory() - contents = await archive.read() - ZipFile(BytesIO(contents)).extractall(path=tempdir.name) - total_added = dfiq.read_from_data_directory(tempdir.name) + with tempfile.TemporaryDirectory() as tempdir: + contents = await archive.read() + ZipFile(BytesIO(contents)).extractall(path=tempdir) + total_added = dfiq.read_from_data_directory(tempdir) return {"total_added": total_added} @@ -78,13 +104,20 @@ async def new_from_yaml(request: NewDFIQRequest) -> dfiq.DFIQTypes: except ValueError as error: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(error)) - # Ensure there is not an object with the same ID: - if dfiq.DFIQBase.find(dfiq_id=new.dfiq_id): + # Ensure there is not an object with the same ID or UUID + + if new.dfiq_id and dfiq.DFIQBase.find(dfiq_id=new.dfiq_id): raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"DFIQ with id {new.dfiq_id} already exists", ) + if dfiq.DFIQBase.find(uuid=new.uuid): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"DFIQ with uuid {new.uuid} already exists", + ) + new = new.save() try: @@ -115,19 +148,19 @@ async def to_archive(request: DFIQSearchRequest) -> FileResponse: aliases=request.filter_aliases, ) - tempdir = tempfile.TemporaryDirectory() - for obj in dfiq_objects: - with open(f"{tempdir.name}/{obj.dfiq_id}.yaml", "w") as f: - f.write(obj.to_yaml()) + with tempfile.TemporaryDirectory() as tempdir: + for obj in dfiq_objects: + with open(f"{tempdir}/{obj.dfiq_id}.yaml", "w") as f: + f.write(obj.to_yaml()) - with tempfile.NamedTemporaryFile(delete=False) as archive: - with ZipFile(archive, "w") as zipf: - for obj in dfiq_objects: - subdir = "internal" if obj.internal else "public" - zipf.write( - f"{tempdir.name}/{obj.dfiq_id}.yaml", - f"{subdir}/{obj.type}/{obj.dfiq_id}.yaml", - ) + with tempfile.NamedTemporaryFile(delete=False) as archive: + with ZipFile(archive, "w") as zipf: + for obj in dfiq_objects: + subdir = "internal" if obj.internal else "public" + zipf.write( + f"{tempdir}/{obj.dfiq_id}.yaml", + f"{subdir}/{obj.type}/{obj.dfiq_id}.yaml", + ) return FileResponse(archive.name, media_type="application/zip", filename="dfiq.zip") @@ -144,7 +177,7 @@ async def validate_dfiq_yaml(request: DFIQValidateRequest) -> DFIQValidateRespon except KeyError as error: return DFIQValidateResponse(valid=False, error=f"Invalid DFIQ type: {error}") - if request.check_id and dfiq.DFIQBase.find(dfiq_id=obj.dfiq_id): + if request.check_id and obj.dfiq_id and dfiq.DFIQBase.find(dfiq_id=obj.dfiq_id): return DFIQValidateResponse( valid=False, error=f"DFIQ with id {obj.dfiq_id} already exists" ) diff --git a/plugins/feeds/public/artifacts.py b/plugins/feeds/public/artifacts.py index c730e864c..7dffde670 100644 --- a/plugins/feeds/public/artifacts.py +++ b/plugins/feeds/public/artifacts.py @@ -30,11 +30,11 @@ def run(self): logging.info("No response: skipping ForensicArtifact update") return - tempdir = tempfile.TemporaryDirectory() - ZipFile(BytesIO(response.content)).extractall(path=tempdir.name) - artifacts_datadir = os.path.join( - tempdir.name, "artifacts-main", "artifacts", "data" - ) + with tempfile.TemporaryDirectory() as tempdir: + ZipFile(BytesIO(response.content)).extractall(path=tempdir) + artifacts_datadir = os.path.join( + tempdir, "artifacts-main", "artifacts", "data" + ) data_files_glob = glob.glob(os.path.join(artifacts_datadir, "*.yaml")) artifacts_dict = {} diff --git a/plugins/feeds/public/attack.py b/plugins/feeds/public/attack.py index 56e502fdd..dfd1ac72a 100644 --- a/plugins/feeds/public/attack.py +++ b/plugins/feeds/public/attack.py @@ -255,6 +255,7 @@ def run(self): ) rel_count += 1 logging.info("Processed %s relationships", rel_count) + tempdir.cleanup() taskmanager.TaskManager.register_task(MitreAttack) diff --git a/plugins/feeds/public/dfiq.py b/plugins/feeds/public/dfiq.py index 61c5c7a00..1b596379b 100644 --- a/plugins/feeds/public/dfiq.py +++ b/plugins/feeds/public/dfiq.py @@ -19,15 +19,15 @@ class DFIQFeed(task.FeedTask): def run(self): response = self._make_request( - "https://github.com/google/dfiq/archive/refs/heads/main.zip" + "https://github.com/tomchop/dfiq/archive/refs/heads/dfiq1.1.zip" ) if not response: logging.info("No response: skipping DFIQ update") return - tempdir = tempfile.TemporaryDirectory() - ZipFile(BytesIO(response.content)).extractall(path=tempdir.name) - dfiq.read_from_data_directory(tempdir.name) + with tempfile.TemporaryDirectory() as tempdir: + ZipFile(BytesIO(response.content)).extractall(path=tempdir) + dfiq.read_from_data_directory(tempdir) extra_dirs = yeti_config.get("dfiq", "extra_dirs") if not extra_dirs: diff --git a/tests/apiv2/dfiq.py b/tests/apiv2/dfiq.py index b59a48389..550db7802 100644 --- a/tests/apiv2/dfiq.py +++ b/tests/apiv2/dfiq.py @@ -25,6 +25,38 @@ def setUp(self) -> None: ).json() client.headers = {"Authorization": "Bearer " + token_data["access_token"]} + def test_config(self) -> None: + dfiq.DFIQQuestion( + name="mock_question", + dfiq_id="Q1020", + uuid="bd46ce6e-c933-46e5-960c-36945aaef401", + dfiq_version="1.1.0", + description="desc", + parent_ids=["F1005"], + dfiq_yaml="mock", + ).save() + + with open("tests/dfiq_test_data/Q1020.10.yaml", "r") as f: + yaml_string = f.read() + + response = client.post( + "/api/v2/dfiq/from_yaml", + json={ + "dfiq_yaml": yaml_string, + "dfiq_type": dfiq.DFIQType.approach, + }, + ) + self.assertEqual(response.status_code, 200, response.json()) + + response = client.get("/api/v2/dfiq/config") + data = response.json() + + self.assertEqual(response.status_code, 200, data) + self.assertEqual(data["approach_data_sources"], ["artifact", "description"]) + self.assertEqual( + data["approach_analysis_step_types"], ["opensearch-query", "pandas"] + ) + def test_new_dfiq_scenario(self) -> None: with open("tests/dfiq_test_data/S1003.yaml", "r") as f: yaml_string = f.read() @@ -42,7 +74,7 @@ def test_new_dfiq_scenario(self) -> None: self.assertIsNotNone(data["created"]) self.assertEqual(data["name"], "scenario1") self.assertEqual(data["dfiq_id"], "S1003") - self.assertEqual(data["dfiq_version"], "1.0.0") + self.assertEqual(data["dfiq_version"], "1.1.0") self.assertEqual(data["description"], "Long description 1\n") self.assertEqual(data["type"], dfiq.DFIQType.scenario) self.assertEqual(data["dfiq_tags"], ["Tag1", "Tag2", "Tag3"]) @@ -51,7 +83,8 @@ def test_new_dfiq_facet(self) -> None: scenario = dfiq.DFIQScenario( name="mock_scenario", dfiq_id="S1003", - dfiq_version="1.0.0", + uuid="fake_scenario_uuid", + dfiq_version="1.1.0", description="desc", dfiq_yaml="mock", ).save() @@ -72,7 +105,7 @@ def test_new_dfiq_facet(self) -> None: self.assertIsNotNone(data["created"]) self.assertEqual(data["name"], "facet1") self.assertEqual(data["dfiq_id"], "F1005") - self.assertEqual(data["dfiq_version"], "1.0.0") + self.assertEqual(data["dfiq_version"], "1.1.0") self.assertEqual(data["description"], "Long description of facet1\n") self.assertEqual(data["type"], dfiq.DFIQType.facet) self.assertEqual(data["dfiq_tags"], ["Web Browser"]) @@ -89,7 +122,8 @@ def test_new_dfiq_question(self) -> None: facet = dfiq.DFIQFacet( name="mock_facet", dfiq_id="F1005", - dfiq_version="1.0.0", + uuid="fake_facet_uuid", + dfiq_version="1.1.0", description="desc", parent_ids=["S1003"], dfiq_yaml="mock", @@ -111,7 +145,7 @@ def test_new_dfiq_question(self) -> None: self.assertIsNotNone(data["created"]) self.assertEqual(data["name"], "What is a question?") self.assertEqual(data["dfiq_id"], "Q1020") - self.assertEqual(data["dfiq_version"], "1.0.0") + self.assertEqual(data["dfiq_version"], "1.1.0") self.assertEqual(data["description"], None) self.assertEqual(data["type"], dfiq.DFIQType.question) self.assertEqual(data["dfiq_tags"], ["Web Browser"]) @@ -128,7 +162,8 @@ def test_new_dfiq_approach(self) -> None: question = dfiq.DFIQQuestion( name="mock_question", dfiq_id="Q1020", - dfiq_version="1.0.0", + uuid="fake_question_uuid", + dfiq_version="1.1.0", description="desc", parent_ids=["F1005"], dfiq_yaml="mock", @@ -150,7 +185,7 @@ def test_new_dfiq_approach(self) -> None: self.assertIsNotNone(data["created"]) self.assertEqual(data["name"], "Approach1") self.assertEqual(data["dfiq_id"], "Q1020.10") - self.assertEqual(data["dfiq_version"], "1.0.0") + self.assertEqual(data["dfiq_version"], "1.1.0") self.assertEqual(data["description"]["summary"], "Description for approach") self.assertEqual(data["type"], dfiq.DFIQType.approach) self.assertEqual(data["dfiq_tags"], ["Lots", "Of", "Tags"]) @@ -166,7 +201,8 @@ def test_dfiq_patch_updates_parents(self) -> None: scenario1 = dfiq.DFIQScenario( name="mock_scenario", dfiq_id="S1003", - dfiq_version="1.0.0", + uuid="fake_scenario_uuid1", + dfiq_version="1.1.0", description="desc", dfiq_yaml="mock", ).save() @@ -174,7 +210,8 @@ def test_dfiq_patch_updates_parents(self) -> None: scenario2 = dfiq.DFIQScenario( name="mock_scenario2", dfiq_id="S1222", - dfiq_version="1.0.0", + uuid="fake_scenario_uuid2", + dfiq_version="1.1.0", description="desc", dfiq_yaml="mock", ).save() @@ -182,7 +219,8 @@ def test_dfiq_patch_updates_parents(self) -> None: facet = dfiq.DFIQFacet( name="mock_facet", dfiq_id="F1005", - dfiq_version="1.0.0", + uuid="fake_facet_uuid", + dfiq_version="1.1.0", description="desc", parent_ids=["S1003"], dfiq_yaml="mock", @@ -217,7 +255,8 @@ def test_dfiq_patch_approach_updates_parents(self) -> None: dfiq.DFIQScenario( name="mock_scenario", dfiq_id="S1003", - dfiq_version="1.0.0", + uuid="fake_scenario_uuid", + dfiq_version="1.1.0", description="desc", dfiq_yaml="mock", ).save() @@ -225,25 +264,28 @@ def test_dfiq_patch_approach_updates_parents(self) -> None: dfiq.DFIQFacet( name="mock_facet", dfiq_id="F1005", - dfiq_version="1.0.0", + uuid="fake_facet_uuid", + dfiq_version="1.1.0", description="desc", - parent_ids=["S1003"], + parent_ids=["fake_scenario_uuid"], dfiq_yaml="mock", ).save() question1 = dfiq.DFIQQuestion( name="mock_question", dfiq_id="Q1020", - dfiq_version="1.0.0", + uuid="bd46ce6e-c933-46e5-960c-36945aaef401", + dfiq_version="1.1.0", description="desc", - parent_ids=["F1005"], + parent_ids=["fake_facet_uuid"], dfiq_yaml="mock", ).save() question2 = dfiq.DFIQQuestion( name="mock_question2", + uuid="fake_question_uuid_2", dfiq_id="Q1022", - dfiq_version="1.0.0", + dfiq_version="1.1.0", description="desc", parent_ids=["F1005"], dfiq_yaml="mock", @@ -261,7 +303,7 @@ def test_dfiq_patch_approach_updates_parents(self) -> None: self.assertEqual(edges[0][0].description, "Uses DFIQ approach") self.assertEqual(total, 1) - approach.dfiq_id = "Q1022.10" + approach.parent_id = "fake_question_uuid_2" response = client.patch( f"/api/v2/dfiq/{approach.id}", json={ @@ -272,12 +314,14 @@ def test_dfiq_patch_approach_updates_parents(self) -> None: ) data = response.json() self.assertEqual(response.status_code, 200, data) - self.assertEqual(data["dfiq_id"], "Q1022.10") + self.assertEqual(data["dfiq_id"], "Q1020.10") self.assertEqual(data["id"], approach.id) vertices, edges, total = approach.neighbors() self.assertEqual(len(vertices), 1) self.assertEqual(vertices[f"dfiq/{question2.id}"].dfiq_id, "Q1022") + self.assertEqual(vertices[f"dfiq/{question2.id}"].uuid, "fake_question_uuid_2") + self.assertEqual(edges[0][0].type, "approach") self.assertEqual(edges[0][0].description, "Uses DFIQ approach") self.assertEqual(total, 1) @@ -285,16 +329,16 @@ def test_dfiq_patch_approach_updates_parents(self) -> None: def test_dfiq_patch_approach_updates_indicators(self) -> None: dfiq.DFIQScenario( name="mock_scenario", - dfiq_id="S1003", - dfiq_version="1.0.0", + uuid="fake_scenario_uuid", + dfiq_version="1.1.0", description="desc", dfiq_yaml="mock", ).save() dfiq.DFIQFacet( name="mock_facet", - dfiq_id="F1005", - dfiq_version="1.0.0", + uuid="fake_facet_uuid", + dfiq_version="1.1.0", description="desc", parent_ids=["S1003"], dfiq_yaml="mock", @@ -302,8 +346,8 @@ def test_dfiq_patch_approach_updates_indicators(self) -> None: dfiq.DFIQQuestion( name="mock_question", - dfiq_id="Q1020", - dfiq_version="1.0.0", + uuid="bd46ce6e-c933-46e5-960c-36945aaef401", + dfiq_version="1.1.0", description="desc", parent_ids=["F1005"], dfiq_yaml="mock", @@ -352,16 +396,16 @@ def test_dfiq_patch_approach_updates_indicators(self) -> None: def test_dfiq_post_approach(self): dfiq.DFIQScenario( name="mock_scenario", - dfiq_id="S1003", - dfiq_version="1.0.0", + uuid="fake_scenario_uuid", + dfiq_version="1.1.0", description="desc", dfiq_yaml="mock", ).save() dfiq.DFIQFacet( name="mock_facet", - dfiq_id="F1005", - dfiq_version="1.0.0", + uuid="fake_facet_uuid", + dfiq_version="1.1.0", description="desc", parent_ids=["S1003"], dfiq_yaml="mock", @@ -369,8 +413,8 @@ def test_dfiq_post_approach(self): dfiq.DFIQQuestion( name="mock_question", - dfiq_id="Q1020", - dfiq_version="1.0.0", + uuid="bd46ce6e-c933-46e5-960c-36945aaef401", + dfiq_version="1.1.0", description="desc", parent_ids=["F1005"], dfiq_yaml="mock", @@ -395,6 +439,7 @@ def test_dfiq_post_approach(self): self.assertEqual(len(vertices), 1) approach.delete() + # Repeat the action, updating indicators response = client.post( "/api/v2/dfiq/from_yaml", json={ @@ -438,7 +483,12 @@ def test_wrong_parent_approach(self) -> None: ) data = response.json() self.assertEqual(response.status_code, 400, data) - self.assertEqual(data, {"detail": "Missing parent(s) ['Q1020'] for Q1020.10"}) + self.assertEqual( + data, + { + "detail": "Missing parent(s) ['bd46ce6e-c933-46e5-960c-36945aaef401'] for Q1020.10" + }, + ) def test_valid_dfiq_yaml(self) -> None: with open("tests/dfiq_test_data/S1003.yaml", "r") as f: @@ -501,14 +551,46 @@ def test_valid_dfiq_yaml(self) -> None: self.assertEqual(response.status_code, 200, data) self.assertEqual(data["valid"], True) - def test_upload_dfiq_archive(self): - zip_archive = open("tests/dfiq_test_data/dfiq_test_data.zip", "rb") + def test_standalone_question_creation(self): + with open("tests/dfiq_test_data/Q1020_no_parents.yaml", "r") as f: + yaml_string = f.read() + + response = client.post( + "/api/v2/dfiq/from_yaml", + json={ + "dfiq_yaml": yaml_string, + "dfiq_type": dfiq.DFIQType.question, + }, + ) + data = response.json() + self.assertEqual(response.status_code, 200, data) + self.assertIsNotNone(data["id"]) + self.assertEqual(data["parent_ids"], []) + + def test_standalone_approach_creation(self): + with open("tests/dfiq_test_data/Q1020.10_no_parent.yaml", "r") as f: + yaml_string = f.read() + response = client.post( - "/api/v2/dfiq/from_archive", - files={"archive": ("test_archive.zip", zip_archive, "application/zip")}, + "/api/v2/dfiq/from_yaml", + json={ + "dfiq_yaml": yaml_string, + "dfiq_type": dfiq.DFIQType.approach, + }, ) data = response.json() self.assertEqual(response.status_code, 200, data) + self.assertIsNotNone(data["id"]) + self.assertIsNone(data["parent_id"]) + + def test_upload_dfiq_archive(self): + with open("tests/dfiq_test_data/dfiq_test_data.zip", "rb") as zip_archive: + response = client.post( + "/api/v2/dfiq/from_archive", + files={"archive": ("test_archive.zip", zip_archive, "application/zip")}, + ) + data = response.json() + self.assertEqual(response.status_code, 200, data) self.assertEqual(data, {"total_added": 4}) def test_to_archive(self): diff --git a/tests/dfiq_test_data/DFIQ_Scenario_no_id.yaml b/tests/dfiq_test_data/DFIQ_Scenario_no_id.yaml new file mode 100644 index 000000000..9b3cb8ef4 --- /dev/null +++ b/tests/dfiq_test_data/DFIQ_Scenario_no_id.yaml @@ -0,0 +1,13 @@ +--- +display_name: scenario1 +type: scenario +description: > + Long description 1 +id: +uuid: 2ee16263-56f8-49a5-9b33-d1a2dd8b829c +internal: false +dfiq_version: 1.1.0 +tags: + - Tag1 + - Tag2 + - Tag3 diff --git a/tests/dfiq_test_data/F1005.yaml b/tests/dfiq_test_data/F1005.yaml index 4ea3a0f71..ee5de55f6 100644 --- a/tests/dfiq_test_data/F1005.yaml +++ b/tests/dfiq_test_data/F1005.yaml @@ -4,7 +4,9 @@ type: facet description: > Long description of facet1 id: F1005 -dfiq_version: 1.0.0 +uuid: b2bab31f-1670-4297-8cb1-685747a13468 +internal: false +dfiq_version: 1.1.0 tags: - Web Browser parent_ids: diff --git a/tests/dfiq_test_data/Q1020.10.yaml b/tests/dfiq_test_data/Q1020.10.yaml index 530022870..c3612db80 100644 --- a/tests/dfiq_test_data/Q1020.10.yaml +++ b/tests/dfiq_test_data/Q1020.10.yaml @@ -2,13 +2,15 @@ display_name: Approach1 type: approach id: Q1020.10 -dfiq_version: 1.0.0 +uuid: 292500f7-9d54-40ca-8254-34821e9b5c4e +parent_id: bd46ce6e-c933-46e5-960c-36945aaef401 +internal: false +dfiq_version: 1.1.0 tags: - Lots - Of - Tags description: - summary: Description for approach details: > Details for approach references: diff --git a/tests/dfiq_test_data/Q1020.10_no_indicators.yaml b/tests/dfiq_test_data/Q1020.10_no_indicators.yaml index 293f05efe..3d0b53f70 100644 --- a/tests/dfiq_test_data/Q1020.10_no_indicators.yaml +++ b/tests/dfiq_test_data/Q1020.10_no_indicators.yaml @@ -2,13 +2,15 @@ display_name: Approach1 type: approach id: Q1020.10 -dfiq_version: 1.0.0 +uuid: fcbdb313-424a-436e-a877-130aeba3f134 +parent_id: bd46ce6e-c933-46e5-960c-36945aaef401 +internal: false +dfiq_version: 1.1.0 tags: - Lots - Of - Tags description: - summary: Description for approach details: > Details for approach references: diff --git a/tests/dfiq_test_data/Q1020.10_no_parent.yaml b/tests/dfiq_test_data/Q1020.10_no_parent.yaml new file mode 100644 index 000000000..8a401fa38 --- /dev/null +++ b/tests/dfiq_test_data/Q1020.10_no_parent.yaml @@ -0,0 +1,60 @@ +--- +display_name: Approach1 +type: approach +id: Q1020.10 +uuid: 292500f7-9d54-40ca-8254-34821e9b5c4e +parent_id: +internal: false +dfiq_version: 1.1.0 +tags: + - Lots + - Of + - Tags +description: + details: > + Details for approach + references: + - "ref1" + - "ref2" + references_internal: null +view: + data: + - type: artifact + value: RandomArtifact + - type: description + value: Random description + notes: + covered: + - Covered1 + - Covered2 + - Covered3 + not_covered: + - Not covered1 + - Not covered2 + processors: + - name: processor1 + options: + - type: parsers + value: parser1option + analysis: + - name: OpenSearch + steps: + - description: random parser description + type: opensearch-query + value: data_type:("fs:stat") + - name: Python Notebook + steps: + - description: random step description + type: pandas + value: query('data_type in ("fs:stat")') + - name: processor2 + options: + - type: format + value: jsonl + analysis: + - name: analysis1 + steps: + - description: &filter-desc-processor2 > + something else + type: opensearch-query + value: data_type:"chrome:history:page_visited") diff --git a/tests/dfiq_test_data/Q1020.yaml b/tests/dfiq_test_data/Q1020.yaml index b92ded8b9..5878cc4f4 100644 --- a/tests/dfiq_test_data/Q1020.yaml +++ b/tests/dfiq_test_data/Q1020.yaml @@ -3,7 +3,9 @@ display_name: What is a question? type: question description: id: Q1020 -dfiq_version: 1.0.0 +uuid: bd46ce6e-c933-46e5-960c-36945aaef401 +internal: false +dfiq_version: 1.1.0 tags: - Web Browser parent_ids: diff --git a/tests/dfiq_test_data/Q1020_no_parents.yaml b/tests/dfiq_test_data/Q1020_no_parents.yaml new file mode 100644 index 000000000..df8c2d5ca --- /dev/null +++ b/tests/dfiq_test_data/Q1020_no_parents.yaml @@ -0,0 +1,11 @@ +--- +display_name: What is a question? +type: question +description: +id: Q1020 +uuid: bd46ce6e-c933-46e5-960c-36945aaef401 +internal: false +dfiq_version: 1.1.0 +tags: + - Web Browser +parent_ids: [] diff --git a/tests/dfiq_test_data/Q1020_uuid_parent.yaml b/tests/dfiq_test_data/Q1020_uuid_parent.yaml new file mode 100644 index 000000000..23a576f74 --- /dev/null +++ b/tests/dfiq_test_data/Q1020_uuid_parent.yaml @@ -0,0 +1,12 @@ +--- +display_name: What is a question? +type: question +description: +id: Q1020 +uuid: bd46ce6e-c933-46e5-960c-36945aaef401 +internal: false +dfiq_version: 1.1.0 +tags: + - Web Browser +parent_ids: + - b2bab31f-1670-4297-8cb1-685747a13468 diff --git a/tests/dfiq_test_data/Q1020_uuid_scenario_parent.yaml b/tests/dfiq_test_data/Q1020_uuid_scenario_parent.yaml new file mode 100644 index 000000000..baf0371bf --- /dev/null +++ b/tests/dfiq_test_data/Q1020_uuid_scenario_parent.yaml @@ -0,0 +1,12 @@ +--- +display_name: What is a question? +type: question +description: +id: Q1020 +uuid: bd46ce6e-c933-46e5-960c-36945aaef401 +internal: false +dfiq_version: 1.1.0 +tags: + - Web Browser +parent_ids: + - S1003 diff --git a/tests/dfiq_test_data/S1003.yaml b/tests/dfiq_test_data/S1003.yaml index 6cac4c2b3..155690ecd 100644 --- a/tests/dfiq_test_data/S1003.yaml +++ b/tests/dfiq_test_data/S1003.yaml @@ -4,7 +4,9 @@ type: scenario description: > Long description 1 id: S1003 -dfiq_version: 1.0.0 +uuid: 2ee16263-56f8-49a5-9b33-d1a2dd8b829c +internal: false +dfiq_version: 1.1.0 tags: - Tag1 - Tag2 diff --git a/tests/dfiq_test_data/dfiq_test_data.zip b/tests/dfiq_test_data/dfiq_test_data.zip index e70e59183..bbe302982 100644 Binary files a/tests/dfiq_test_data/dfiq_test_data.zip and b/tests/dfiq_test_data/dfiq_test_data.zip differ diff --git a/tests/schemas/dfiq.py b/tests/schemas/dfiq.py index dce5a0fba..646d44fa7 100644 --- a/tests/schemas/dfiq.py +++ b/tests/schemas/dfiq.py @@ -28,7 +28,22 @@ def test_dfiq_scenario(self) -> None: self.assertIsNotNone(result.id) self.assertIsNotNone(result.created) self.assertEqual(result.name, "scenario1") - self.assertEqual(result.dfiq_version, "1.0.0") + self.assertEqual(result.dfiq_version, "1.1.0") + self.assertEqual(str(result.uuid), "2ee16263-56f8-49a5-9b33-d1a2dd8b829c") + self.assertEqual(result.description, "Long description 1\n") + self.assertEqual(result.type, DFIQType.scenario) + self.assertEqual(result.dfiq_tags, ["Tag1", "Tag2", "Tag3"]) + + def test_dfiq_scenario_no_id(self) -> None: + with open("tests/dfiq_test_data/DFIQ_Scenario_no_id.yaml", "r") as f: + yaml_string = f.read() + + result = DFIQScenario.from_yaml(yaml_string).save() + self.assertIsNotNone(result.id) + self.assertIsNotNone(result.created) + self.assertEqual(result.name, "scenario1") + self.assertEqual(result.dfiq_version, "1.1.0") + self.assertEqual(str(result.uuid), "2ee16263-56f8-49a5-9b33-d1a2dd8b829c") self.assertEqual(result.description, "Long description 1\n") self.assertEqual(result.type, DFIQType.scenario) self.assertEqual(result.dfiq_tags, ["Tag1", "Tag2", "Tag3"]) @@ -44,7 +59,8 @@ def test_dfiq_facet(self) -> None: self.assertEqual(result.name, "facet1") self.assertEqual(result.description, "Long description of facet1\n") self.assertEqual(result.dfiq_id, "F1005") - self.assertEqual(result.dfiq_version, "1.0.0") + self.assertEqual(result.dfiq_version, "1.1.0") + self.assertEqual(str(result.uuid), "b2bab31f-1670-4297-8cb1-685747a13468") self.assertEqual(result.dfiq_tags, ["Web Browser"]) self.assertEqual(result.parent_ids, ["S1003"]) self.assertEqual(result.type, DFIQType.facet) @@ -59,8 +75,9 @@ def test_dfiq_question(self) -> None: self.assertIsNotNone(result.created) self.assertEqual(result.name, "What is a question?") self.assertEqual(result.description, None) + self.assertEqual(str(result.uuid), "bd46ce6e-c933-46e5-960c-36945aaef401") self.assertEqual(result.dfiq_id, "Q1020") - self.assertEqual(result.dfiq_version, "1.0.0") + self.assertEqual(result.dfiq_version, "1.1.0") self.assertEqual(result.dfiq_tags, ["Web Browser"]) self.assertEqual(result.parent_ids, ["F1005"]) self.assertEqual(result.type, DFIQType.question) @@ -72,9 +89,10 @@ def test_dfiq_approach(self) -> None: result = DFIQApproach.from_yaml(yaml_string).save() self.assertIsNotNone(result.id) + self.assertEqual(result.uuid, "292500f7-9d54-40ca-8254-34821e9b5c4e") + self.assertEqual(result.parent_id, "bd46ce6e-c933-46e5-960c-36945aaef401") self.assertIsNotNone(result.created) self.assertEqual(result.name, "Approach1") - self.assertEqual(result.description.summary, "Description for approach") self.assertEqual(result.description.details, "Details for approach\n") self.assertEqual(result.description.references, ["ref1", "ref2"])