Merge pull request #52 from Ostorlab/fix/whitelist_ecosystems_for_elf…

…_and_macho

Add support for whitelisting ecosystems.

agent/api_manager/osv_service_api.py

@@ -7,6 +7,9 @@
 
 import requests
 import tenacity
+from tenacity import stop
+from tenacity import wait
+
 
 logger = logging.getLogger(__name__)
 
@@ -27,8 +30,8 @@ class VulnData:
 
 
 @tenacity.retry(
-    stop=tenacity.stop_after_attempt(NUMBER_RETRIES),
-    wait=tenacity.wait_fixed(WAIT_BETWEEN_RETRIES),
+    stop=stop.stop_after_attempt(NUMBER_RETRIES),
+    wait=wait.wait_fixed(WAIT_BETWEEN_RETRIES),
     retry=tenacity.retry_if_exception_type(),
     retry_error_callback=lambda retry_state: retry_state.outcome.result()
     if retry_state.outcome is not None

agent/cve_service_api.py

@@ -5,6 +5,9 @@
 
 import requests
 import tenacity
+from tenacity import stop
+from tenacity import wait
+
 
 CVE_MITRE_BASE_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId="
 REQUEST_TIMEOUT = 60
@@ -24,9 +27,9 @@ class CVE:
 
 
 @tenacity.retry(
-    stop=tenacity.stop_after_attempt(10),
+    stop=stop.stop_after_attempt(10),
     # wait for 30 seconds before retrying
-    wait=tenacity.wait_fixed(30),
+    wait=wait.wait_fixed(30),
     retry=tenacity.retry_if_exception_type(
         (requests.ConnectionError, requests.HTTPError, json.JSONDecodeError)
     ),

agent/osv_agent.py

@@ -36,15 +36,19 @@
     "requirements.txt",
     "yarn.lock",
 ]
+
 OSV_ECOSYSTEM_MAPPING = {
-    "JAVASCRIPT_LIBRARY": "npm",
-    "JAVA_LIBRARY": "Maven",
-    "FLUTTER_FRAMEWORK": "Pub",
-    "CORDOVA_LIBRARY": "npm",
-    "DOTNET_FRAMEWORK": "NuGet",
-    "IOS_FRAMEWORK": "SwiftURL",
+    "JAVASCRIPT_LIBRARY": ["npm"],
+    "JAVA_LIBRARY": ["Maven"],
+    "FLUTTER_FRAMEWORK": ["Pub"],
+    "CORDOVA_FRAMEWORK": ["npm"],
+    "DOTNET_FRAMEWORK": ["NuGet"],
+    "IOS_FRAMEWORK": ["SwiftURL"],
+    "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"],
+    "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"],
 }
 
+
 logging.basicConfig(
     format="%(message)s",
     datefmt="[%X]",
@@ -171,17 +175,25 @@ def _process_fingerprint_file(self, message: m.Message) -> None:
         path = message.data.get("path")
 
         if package_version is None:
-            logger.error("Error: Version must not be None.")
             return None
         if package_name is None:
-            logger.error("Error: Package name must not be None.")
+            logger.warning("Error: Package name must not be None.")
             return None
 
+        ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type), [])
+        whitelisted_ecosystems = None
+        ecosystem = None
+        if len(ecosystems) == 1:
+            ecosystem = ecosystems[0]
+        elif len(ecosystems) > 1:
+            whitelisted_ecosystems = ecosystems
+
         api_result = osv_service_api.query_osv_api(
             package_name=package_name,
             version=package_version,
-            ecosystem=OSV_ECOSYSTEM_MAPPING.get(str(package_type)),
+            ecosystem=ecosystem,
         )
+
         if api_result is None or api_result == {}:
             return None
 
@@ -190,8 +202,8 @@ def _process_fingerprint_file(self, message: m.Message) -> None:
             package_name=package_name,
             package_version=package_version,
             api_key=self.api_key,
+            whitelisted_ecosystems=whitelisted_ecosystems,
         )
-
         if parsed_osv_output is None:
             return None
 

agent/osv_output_handler.py

@@ -32,6 +32,7 @@
     "POTENTIALLY": 5,
 }
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -167,7 +168,7 @@ def parse_vulnerabilities_osv_binary(
         return parsed_vulns
 
     except json.JSONDecodeError as e:
-        logger.error(f"Error decoding JSON: {e}")
+        logger.error("Error decoding JSON: %s", e)
         return []
 
 
@@ -176,13 +177,16 @@ def parse_vulnerabilities_osv_api(
     package_name: str,
     package_version: str,
     api_key: str | None = None,
+    whitelisted_ecosystems: list[str] | None = None,
 ) -> list[VulnData]:
     """Parse the OSV API response to extract vulnerabilities.
+
     Args:
-        output: The API response json.
+        output: The list of vulnerabilities raw from the API response .
         package_name: The package name.
         package_version: The package version.
         api_key: The NVD API key.
+
     Returns:
         Parsed output.
     """
@@ -195,16 +199,24 @@ def parse_vulnerabilities_osv_api(
     highest_risk_vuln_info: dict[str, str] = {}
     if len(vulnerabilities) == 0:
         return []
-    for vulnerability in vulnerabilities:
+
+    whitlisted_vulnerabilities = _whitelist_vulnz_from_ecosystems(
+        vulnerabilities, whitelisted_ecosystems
+    )
+    for vulnerability in whitlisted_vulnerabilities:
         fixed_version = _get_fixed_version(vulnerability.get("affected"))
         if fixed_version != "":
             fixed_versions.append(fixed_version)
         filtered_cves = [
             alias for alias in vulnerability.get("aliases", []) if "CVE" in alias
         ]
-        for cve in filtered_cves:
-            description += f"- [{cve}]({CVE_MITRE_URL}{cve}) "
-            description += f": {vulnerability.get('details')}\n"
+
+        if len(filtered_cves) > 0:
+            for cve in filtered_cves:
+                description += f"- [{cve}]({CVE_MITRE_URL}{cve}) : "
+        else:
+            description += f"- {vulnerability.get('id')} : "
+        description += f"{vulnerability.get('details')}\n"
 
         severity = vulnerability.get("database_specific", {}).get("severity")
         risk = _vuln_risk_rating(risk=severity, cves=filtered_cves, api_key=api_key)
@@ -216,7 +228,7 @@ def parse_vulnerabilities_osv_api(
         if old_risk is not None and new_risk is not None and old_risk > new_risk:
             highest_risk_vuln_info["risk"] = risk
             highest_risk_vuln_info["cvss_v3_vector"] = _get_cvss_v3_vector(
-                vulnerability.get("severity")
+                vulnerability.get("severity", [])
             )
             highest_risk_vuln_info["summary"] = vulnerability.get("summary", "")
 
@@ -245,6 +257,23 @@ def parse_vulnerabilities_osv_api(
     ]
 
 
+def _whitelist_vulnz_from_ecosystems(
+    vulnerabilities: list[dict[str, Any]],
+    whitelisted_ecosystems: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    if whitelisted_ecosystems is None or len(whitelisted_ecosystems) == 0:
+        return vulnerabilities
+    whitelisted = []
+    for vuln in vulnerabilities:
+        affected_data = vuln.get("affected", [])
+        ecosystem = None
+        if len(affected_data) > 0:
+            ecosystem = affected_data[0].get("package", {}).get("ecosystem")
+        if ecosystem in whitelisted_ecosystems:
+            whitelisted.append(vuln)
+    return whitelisted
+
+
 def _aggregate_cves(cve_ids: list[str], api_key: str | None = None) -> str:
     """Generate the description for the vulnerability from all the related CVEs."""
     cve_list_details = ""

tests/conftest.py

@@ -227,3 +227,14 @@ def osv_api_output_risk_invalid() -> dict[str, Any]:
     data = pathlib.Path(file_path).read_text(encoding="utf-8")
     json_data: dict[str, Any] = json.loads(data)
     return json_data
+
+
+@pytest.fixture
+def elf_library_fingerprint_msg() -> message.Message:
+    selector = "v3.fingerprint.file.library"
+    msg_data = {
+        "library_name": "opencv",
+        "library_version": "4.9.0",
+        "library_type": "ELF_LIBRARY",
+    }
+    return message.Message.from_data(selector, data=msg_data)

tests/osv_agent_test.py

@@ -329,9 +329,12 @@ def testAgentOSV_always_emitVulnWithValidTechnicalDetail(
     )
     assert agent_mock[0].data["risk_rating"] == "CRITICAL"
     assert (
-        agent_mock[0].data["technical_detail"]
-        == """- [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands.
-"""
+        """- [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands.\n"""
+        in agent_mock[0].data["technical_detail"]
+    )
+    assert (
+        """- GHSA-f698-m2v9-5fh3 : Versions of `opencv`prior to 6.1.0 are vulnerable to Command Injection. The utils/ script find-opencv.js does not validate user input allowing attackers to execute arbitrary commands.\n\n\n## Recommendation\n\nUpgrade to version 6.1.0.\n\n"""
+        in agent_mock[0].data["technical_detail"]
     )
     assert (
         agent_mock[0].data["recommendation"]
@@ -423,6 +426,7 @@ def testAgentOSV_whenPathInMessage_technicalDetailShouldIncludeIt(
     assert agent_mock[0].data["risk_rating"] == "CRITICAL"
     assert agent_mock[0].data["technical_detail"] == (
         """Dependency `opencv` Found in `lib/arm64-v8a/libBlinkID.so` has a security issue: 
+- GHSA-f698-m2v9-5fh3 : Versions of `opencv`prior to 6.1.0 are vulnerable to Command Injection. The utils/ script find-opencv.js does not validate user input allowing attackers to execute arbitrary commands.\n\n\n## Recommendation\n\nUpgrade to version 6.1.0.\n
 - [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands.
 """
     )
@@ -436,3 +440,39 @@ def testAgentOSV_whenPathInMessage_technicalDetailShouldIncludeIt(
         agent_mock[0].data["recommendation"]
         == "We recommend updating `opencv` to a version greater than or equal to `6.1.0`."
     )
+
+
+def testAgentOSV_whenElfLibraryFingerprintMessage_shouldExcludeNpmEcosystemVulnz(
+    test_agent: osv_agent.OSVAgent,
+    agent_mock: list[message.Message],
+    agent_persist_mock: dict[str | bytes, str | bytes],
+    elf_library_fingerprint_msg: message.Message,
+) -> None:
+    """For fingerprints of elf or macho files, we do not know the corresponding osv ecosystem.
+    We use a list of accepted ecosystems.
+    This unit test ensures no vulnz of excluded ecosystems are reported.
+    """
+    test_agent.process(elf_library_fingerprint_msg)
+
+    assert len(agent_mock) == 1
+
+    assert (
+        agent_mock[0].data["title"]
+        == "Use of Outdated Vulnerable Component: [email protected]"
+    )
+    assert (
+        agent_mock[0].data["dna"]
+        == "Use of Outdated Vulnerable Component: [email protected]"
+    )
+    assert agent_mock[0].data["risk_rating"] == "POTENTIALLY"
+    assert agent_mock[0].data["technical_detail"] == (
+        """```- OSV-2022-394 : OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=47190\n\n```\nCrash type: Incorrect-function-pointer-type\nCrash state:\ncv::split\ncv::split\nTestSplitAndMerge\n```\n\n- OSV-2023-444 : OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=59450\n\n```\nCrash type: Heap-buffer-overflow READ 4\nCrash state:\nopj_jp2_apply_pclr\nopj_jp2_decode\ncv::detail::Jpeg2KOpjDecoderBase::readData\n```\n\n```"""
+    )
+    assert agent_mock[0].data["description"] == (
+        """Dependency `opencv` with version `4.9.0` has a security issue."""
+    )
+
+    assert (
+        agent_mock[0].data["recommendation"]
+        == "We recommend updating `opencv` to the latest available version."
+    )