[ASM] - Expander - Standardize enrichment names in asmsystemids grid …

…field (4819 & 5508) (#29098) (#29357)

* asmcloud

* collection, notification and privIP

* remediation, servicedetection and systemids

* tags, serviceowner, summary

* GridFieldSetup

* automations

* fixed some misses

* update playbook names for testing

* update parent README

* remove temp naming for testing

* fix parent and RN

* docker and pre-commit

* update RN

* fix typo

* Apply suggestions from code review



* added Shirleys change to playbook inputs

* Update asmsystemids and created playbook tests.

Updated asmsystemids to a new standard for consistency.
Create new tests for all of our enrichment playbook using asmsystemids,
to check if keys of asmsystemids are valid.

* Update asmsystemids in tenable playbook

- Add test for tenable enrichment playbook for asmsystemids
- Update Rapid7 OS field

* Update release notes

* Formatting for tests files

* Revert AWS enrichment version

* Remove prints from tests unless needed in future

* Update playbooks with bugs fix commit

Applied changes from 3d397fa.

* Add S3 task and update PNGs

* Update AWS enrichment and test

* EXPANDR-5782

* Fix Qualys playbook description

* Update Set ASM system IDs type description

* Update documentation and names of tests

* Move playbook tests to new location

- Updated comment in helper for conftest.py conflict

* Update test files and helper

- New class in helper for loading yaml files as a dictionary
- Updated test files to use a base path and new class object

* Fix formatting for pre-commit

---------

Co-authored-by: John <[email protected]>
Co-authored-by: jwilkes <[email protected]>
Co-authored-by: johnnywilkes <[email protected]>
Co-authored-by: ShirleyDenkberg <[email protected]>

Packs/CortexAttackSurfaceManagement/Integrations/CortexAttackSurfaceManagement/test_data/TestsForPlaybooks/CortexASMAWSEnrichment_test.py

@@ -0,0 +1,70 @@
+import pytest
+from pathlib import Path
+from utils import helper
+from utils.helper import PlaybookDataLoader
+
+
+"""Setup methods"""
+
+
+@pytest.fixture()
+def playbook_data_loader():
+    base_path = Path(__file__).resolve().parents[4]
+    # Assuming the playbook is in the specified path relative to the current file
+    playbook_path = base_path / "Playbooks" / "Cortex_ASM_-_AWS_Enrichment.yml"
+    print(playbook_path)
+    return PlaybookDataLoader(playbook_path)
+
+
+"""Test cases"""
+
+
+def test_expected_playbook_name_and_id(playbook_data_loader: PlaybookDataLoader):
+    """Test the name and ID of the AWS playbook.
+    This tests should help with validating the correct file is being tested.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+    """
+    full_data = playbook_data_loader.full_playbook_data
+    assert full_data.get("name") == "Cortex ASM - AWS Enrichment"
+    assert full_data.get("id") == "Cortex ASM - AWS Enrichment"
+
+
+@pytest.mark.parametrize(
+    "asm_system_id_type", ["ASSET-ID", "ASSET-SUBNET-ID", "ASSET-SG", "ASSET-NIC", "ASSET-VIRTUAL-NET"]
+)
+def test_expected_asmsystemids_all_in_known_set(playbook_data_loader: PlaybookDataLoader, asm_system_id_type: str):
+    """This tests should help with testing the existence of necessary values used by downstream applications.
+
+    Args:
+       playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        asm_system_id_type (str): a value that should be set to the key of "Type" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    key_found = helper.was_grid_field_value_found(
+        tasks_data, "val1", asm_system_id_type
+    )
+    assert key_found
+
+
+@pytest.mark.parametrize(
+    "grid_field_data",
+    [({"gridfield": "asmsystemids", "val1": "ASSET-TYPE", "val2": "AWS EC2"}),
+     ({"gridfield": "asmsystemids", "val1": "ASSET-TYPE", "val2": "AWS S3"})]
+)
+def test_each_asmsystemid_maps_to_known_asset_type(playbook_data_loader: PlaybookDataLoader, grid_field_data: dict):
+    """This tests should help with testing the existence of necessary values for ASSET-TYPE used by downstream applications.
+
+    Args:
+       playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        grid_field_data (dict): a set of values that should be set to the key of "Type" AND "ID" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+                                  It includes checking that the task is also setting "asmsystemids"
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    data_found = helper.check_multiple_grid_field_values(
+        tasks_data, grid_field_data
+    )
+    assert data_found

Packs/CortexAttackSurfaceManagement/Integrations/CortexAttackSurfaceManagement/test_data/TestsForPlaybooks/CortexASMAzureEnrichment_test.py

@@ -0,0 +1,69 @@
+import pytest
+from pathlib import Path
+from utils import helper
+from utils.helper import PlaybookDataLoader
+
+
+"""Setup methods"""
+
+
+@pytest.fixture()
+def playbook_data_loader():
+    base_path = Path(__file__).resolve().parents[4]
+    # Assuming the playbook is in the specified path relative to the current file
+    playbook_path = base_path / "Playbooks" / "Cortex_ASM_-_Azure_Enrichment.yml"
+    print(playbook_path)
+    return PlaybookDataLoader(playbook_path)
+
+
+"""Test cases"""
+
+
+def test_expected_playbook_name_and_id(playbook_data_loader: PlaybookDataLoader):
+    """Test the name and ID of the playbook.
+    This tests should help with validating the correct file is being tested.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+    """
+    full_data = playbook_data_loader.full_playbook_data
+    assert full_data.get("name") == "Cortex ASM - Azure Enrichment"
+    assert full_data.get("id") == "Cortex ASM - Azure Enrichment"
+
+
+@pytest.mark.parametrize(
+    "asm_system_id_type", ["ASSET-ID", "ASSET-NAME", "ASSET-RG", "ASSET-SG", "ASSET-NIC"]
+)
+def test_expected_asmsystemids_all_in_known_set(playbook_data_loader: PlaybookDataLoader, asm_system_id_type: str):
+    """This tests should help with testing the existence of necessary values used by downstream applications.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        asm_system_id_type (str): a value that should be set to the key of "Type" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+    """
+    task_data = playbook_data_loader.playbook_tasks_data
+    key_found = helper.was_grid_field_value_found(
+        task_data, "val1", asm_system_id_type
+    )
+    assert key_found
+
+
+@pytest.mark.parametrize(
+    "grid_field_data",
+    [({"gridfield": "asmsystemids", "val1": "ASSET-TYPE", "val2": "Azure Compute"})],
+)
+def test_each_asmsystemid_maps_to_known_asset_type(playbook_data_loader: PlaybookDataLoader, grid_field_data: dict):
+    """This tests should help with testing the existence of necessary values for ASSET-TYPE used by downstream applications.
+
+    Args:
+       playbook_data_loader: (PlaybookDataLoader): a subset of yml playbook file that only includes the tasks data.
+        grid_field_data (dict): a set of values that should be set to the key of "Type" AND "ID" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+                                  It includes checking that the task is also setting "asmsystemids"
+    """
+    task_data = playbook_data_loader.playbook_tasks_data
+    data_found = helper.check_multiple_grid_field_values(
+        task_data, grid_field_data
+    )
+    assert data_found

Packs/CortexAttackSurfaceManagement/Integrations/CortexAttackSurfaceManagement/test_data/TestsForPlaybooks/CortexASMGCPEnrichment_test.py

@@ -0,0 +1,86 @@
+import pytest
+from pathlib import Path
+from utils import helper
+from utils.helper import PlaybookDataLoader
+
+
+"""Setup methods"""
+
+
+@pytest.fixture()
+def playbook_data_loader():
+    base_path = Path(__file__).resolve().parents[4]
+    # Assuming the playbook is in the specified path relative to the current file
+    playbook_path = base_path / "Playbooks" / "Cortex_ASM_-_GCP_Enrichment.yml"
+    print(playbook_path)
+    return PlaybookDataLoader(playbook_path)
+
+
+"""Test cases"""
+
+
+def test_expected_playbook_name_and_id(playbook_data_loader: PlaybookDataLoader):
+    """Test the name and ID of the playbook.
+    This tests should help with validating the correct file is being tested.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+    """
+    full_data = playbook_data_loader.full_playbook_data
+    assert full_data.get("name") == "Cortex ASM - GCP Enrichment"
+    assert full_data.get("id") == "Cortex ASM - GCP Enrichment"
+
+
+@pytest.mark.parametrize(
+    "asm_system_id_type",
+    [
+        "ASSET-ID",
+        "ASSET-NAME",
+        "ASSET-SG",
+        "ASSET-VIRTUAL-NET",
+        "ASSET-SUBNET-NAME",
+        "ASSET-NIC",
+        "ASSET-ZONE",
+    ],
+)
+def test_expected_asmsystemids_all_in_known_set(playbook_data_loader: PlaybookDataLoader, asm_system_id_type: str):
+    """This tests should help with testing the existence of necessary values used by downstream applications.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        asm_system_id_type (str): a value that should be set to the key of "Type" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    key_found = helper.was_grid_field_value_found(
+        tasks_data, "val1", asm_system_id_type
+    )
+    assert key_found
+
+
+@pytest.mark.parametrize(
+    "grid_field_data",
+    [
+        (
+            {
+                "gridfield": "asmsystemids",
+                "val1": "ASSET-TYPE",
+                "val2": "Google Compute Engine",
+            }
+        )
+    ],
+)
+def test_each_asmsystemid_maps_to_known_asset_type(playbook_data_loader: PlaybookDataLoader, grid_field_data: dict):
+    """This tests should help with testing the existence of necessary values for ASSET-TYPE used by downstream applications.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        grid_field_data (dict): a set of values that should be set to the key of "Type" AND "ID" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+                                  It includes checking that the task is also setting "asmsystemids"
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    data_found = helper.check_multiple_grid_field_values(
+        tasks_data, grid_field_data
+    )
+    assert data_found

Packs/CortexAttackSurfaceManagement/Integrations/CortexAttackSurfaceManagement/test_data/TestsForPlaybooks/CortexASMPrismaCloudEnrichment_test.py

@@ -0,0 +1,47 @@
+import pytest
+from pathlib import Path
+from utils import helper
+from utils.helper import PlaybookDataLoader
+
+
+"""Setup methods"""
+
+
+@pytest.fixture()
+def playbook_data_loader():
+    base_path = Path(__file__).resolve().parents[4]
+    # Assuming the playbook is in the specified path relative to the current file
+    playbook_path = base_path / "Playbooks" / "Cortex_ASM_-_Prisma_Cloud_Enrichment.yml"
+    print(playbook_path)
+    return PlaybookDataLoader(playbook_path)
+
+
+"""Test cases"""
+
+
+def test_expected_playbook_name_and_id(playbook_data_loader: PlaybookDataLoader):
+    """Test the name and ID of the playbook.
+    This tests should help with validating the correct file is being tested.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+    """
+    full_data = playbook_data_loader.full_playbook_data
+    assert full_data.get("name") == "Cortex ASM - Prisma Cloud Enrichment"
+    assert full_data.get("id") == "Cortex ASM - Prisma Cloud Enrichment"
+
+
+@pytest.mark.parametrize("asm_system_id_type", ["PRISMACLOUD-INSTANCE-ID"])
+def test_expected_asmsystemids_all_in_known_set(playbook_data_loader: PlaybookDataLoader, asm_system_id_type: str):
+    """This tests should help with testing the existence of necessary values used by downstream applications.
+
+    Args:
+       playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        asm_system_id_type (str): a value that should be set to the key of "Type" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    key_found = helper.was_grid_field_value_found(
+        tasks_data, "val1", asm_system_id_type
+    )
+    assert key_found

Packs/CortexAttackSurfaceManagement/Integrations/CortexAttackSurfaceManagement/test_data/TestsForPlaybooks/CortexASMQualysEnrichment_test.py

@@ -0,0 +1,47 @@
+import pytest
+from pathlib import Path
+from utils import helper
+from utils.helper import PlaybookDataLoader
+
+
+"""Setup methods"""
+
+
+@pytest.fixture()
+def playbook_data_loader():
+    base_path = Path(__file__).resolve().parents[4]
+    # Assuming the playbook is in the specified path relative to the current file
+    playbook_path = base_path / "Playbooks" / "Cortex_ASM_-_Qualys_Enrichment.yml"
+    print(playbook_path)
+    return PlaybookDataLoader(playbook_path)
+
+
+"""Test cases"""
+
+
+def test_expected_playbook_name_and_id(playbook_data_loader: PlaybookDataLoader):
+    """Test the name and ID of the playbook.
+    This tests should help with validating the correct file is being tested.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+    """
+    full_data = playbook_data_loader.full_playbook_data
+    assert full_data.get("name") == "Cortex ASM - Qualys Enrichment"
+    assert full_data.get("id") == "Cortex ASM - Qualys Enrichment"
+
+
+@pytest.mark.parametrize("asm_system_id_type", ["QUALYS-ASSET-ID"])
+def test_expected_asmsystemids_all_in_known_set(playbook_data_loader: PlaybookDataLoader, asm_system_id_type: str):
+    """This tests should help with testing the existence of necessary values used by downstream applications.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        asm_system_id_type (str): a value that should be set to the key of "Type" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    key_found = helper.was_grid_field_value_found(
+        tasks_data, "val1", asm_system_id_type
+    )
+    assert key_found

Packs/CortexAttackSurfaceManagement/Integrations/CortexAttackSurfaceManagement/test_data/TestsForPlaybooks/CortexASMRapid7Enrichment_test.py

@@ -0,0 +1,50 @@
+import pytest
+from pathlib import Path
+from utils import helper
+from utils.helper import PlaybookDataLoader
+
+
+"""Setup methods"""
+
+
+@pytest.fixture()
+def playbook_data_loader():
+    base_path = Path(__file__).resolve().parents[4]
+    # Assuming the playbook is in the specified path relative to the current file
+    playbook_path = base_path / "Playbooks" / "Cortex_ASM_-_Rapid7_Enrichment.yml"
+    print(playbook_path)
+    return PlaybookDataLoader(playbook_path)
+
+
+"""Test cases"""
+
+
+def test_expected_playbook_name_and_id(playbook_data_loader: PlaybookDataLoader):
+    """Test the name and ID of the playbook.
+    This tests should help with validating the correct file is being tested.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+    """
+    full_data = playbook_data_loader.full_playbook_data
+    assert full_data.get("name") == "Cortex ASM - Rapid7 Enrichment"
+    assert full_data.get("id") == "Cortex ASM - Rapid7 Enrichment"
+
+
+@pytest.mark.parametrize(
+    "asm_system_id_type",
+    ["RAPID7-ASSET-OS", "RAPID7-ASSET-SITE", "RAPID7-ASSET-ID", "RAPID7-ASSET-NAME"],
+)
+def test_expected_asmsystemids_all_in_known_set(playbook_data_loader: PlaybookDataLoader, asm_system_id_type: str):
+    """This tests should help with testing the existence of necessary values used by downstream applications.
+
+    Args:
+        playbook_data_loader (PlaybookDataLoader): a data class that can return a dictionary of the yml playbook file.
+        asm_system_id_type (str): a value that should be set to the key of "Type" under
+                                  the "asmsystemids" grid field (incidentfield-ASM_-_System_IDs.json).
+    """
+    tasks_data = playbook_data_loader.playbook_tasks_data
+    key_found = helper.was_grid_field_value_found(
+        tasks_data, "val1", asm_system_id_type
+    )
+    assert key_found