diff --git a/agent/api_manager/osv_service_api.py b/agent/api_manager/osv_service_api.py index a8880f3..f8115c8 100644 --- a/agent/api_manager/osv_service_api.py +++ b/agent/api_manager/osv_service_api.py @@ -7,6 +7,9 @@ import requests import tenacity +from tenacity import stop +from tenacity import wait + logger = logging.getLogger(__name__) @@ -27,8 +30,8 @@ class VulnData: @tenacity.retry( - stop=tenacity.stop_after_attempt(NUMBER_RETRIES), - wait=tenacity.wait_fixed(WAIT_BETWEEN_RETRIES), + stop=stop.stop_after_attempt(NUMBER_RETRIES), + wait=wait.wait_fixed(WAIT_BETWEEN_RETRIES), retry=tenacity.retry_if_exception_type(), retry_error_callback=lambda retry_state: retry_state.outcome.result() if retry_state.outcome is not None diff --git a/agent/cve_service_api.py b/agent/cve_service_api.py index 142253b..b590154 100644 --- a/agent/cve_service_api.py +++ b/agent/cve_service_api.py @@ -5,6 +5,9 @@ import requests import tenacity +from tenacity import stop +from tenacity import wait + CVE_MITRE_BASE_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=" REQUEST_TIMEOUT = 60 @@ -24,9 +27,9 @@ class CVE: @tenacity.retry( - stop=tenacity.stop_after_attempt(10), + stop=stop.stop_after_attempt(10), # wait for 30 seconds before retrying - wait=tenacity.wait_fixed(30), + wait=wait.wait_fixed(30), retry=tenacity.retry_if_exception_type( (requests.ConnectionError, requests.HTTPError, json.JSONDecodeError) ), diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 438598c..1b70f06 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -36,15 +36,19 @@ "requirements.txt", "yarn.lock", ] + OSV_ECOSYSTEM_MAPPING = { - "JAVASCRIPT_LIBRARY": "npm", - "JAVA_LIBRARY": "Maven", - "FLUTTER_FRAMEWORK": "Pub", - "CORDOVA_LIBRARY": "npm", - "DOTNET_FRAMEWORK": "NuGet", - "IOS_FRAMEWORK": "SwiftURL", + "JAVASCRIPT_LIBRARY": ["npm"], + "JAVA_LIBRARY": ["Maven"], + "FLUTTER_FRAMEWORK": ["Pub"], + "CORDOVA_FRAMEWORK": ["npm"], + "DOTNET_FRAMEWORK": ["NuGet"], + "IOS_FRAMEWORK": ["SwiftURL"], + "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"], + "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], } + logging.basicConfig( format="%(message)s", datefmt="[%X]", @@ -171,17 +175,25 @@ def _process_fingerprint_file(self, message: m.Message) -> None: path = message.data.get("path") if package_version is None: - logger.error("Error: Version must not be None.") return None if package_name is None: - logger.error("Error: Package name must not be None.") + logger.warning("Error: Package name must not be None.") return None + ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type), []) + whitelisted_ecosystems = None + ecosystem = None + if len(ecosystems) == 1: + ecosystem = ecosystems[0] + elif len(ecosystems) > 1: + whitelisted_ecosystems = ecosystems + api_result = osv_service_api.query_osv_api( package_name=package_name, version=package_version, - ecosystem=OSV_ECOSYSTEM_MAPPING.get(str(package_type)), + ecosystem=ecosystem, ) + if api_result is None or api_result == {}: return None @@ -190,8 +202,8 @@ def _process_fingerprint_file(self, message: m.Message) -> None: package_name=package_name, package_version=package_version, api_key=self.api_key, + whitelisted_ecosystems=whitelisted_ecosystems, ) - if parsed_osv_output is None: return None diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index 5117bdc..2fed2b8 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -32,6 +32,7 @@ "POTENTIALLY": 5, } + logger = logging.getLogger(__name__) @@ -167,7 +168,7 @@ def parse_vulnerabilities_osv_binary( return parsed_vulns except json.JSONDecodeError as e: - logger.error(f"Error decoding JSON: {e}") + logger.error("Error decoding JSON: %s", e) return [] @@ -176,13 +177,16 @@ def parse_vulnerabilities_osv_api( package_name: str, package_version: str, api_key: str | None = None, + whitelisted_ecosystems: list[str] | None = None, ) -> list[VulnData]: """Parse the OSV API response to extract vulnerabilities. + Args: - output: The API response json. + output: The list of vulnerabilities raw from the API response . package_name: The package name. package_version: The package version. api_key: The NVD API key. + Returns: Parsed output. """ @@ -195,16 +199,24 @@ def parse_vulnerabilities_osv_api( highest_risk_vuln_info: dict[str, str] = {} if len(vulnerabilities) == 0: return [] - for vulnerability in vulnerabilities: + + whitlisted_vulnerabilities = _whitelist_vulnz_from_ecosystems( + vulnerabilities, whitelisted_ecosystems + ) + for vulnerability in whitlisted_vulnerabilities: fixed_version = _get_fixed_version(vulnerability.get("affected")) if fixed_version != "": fixed_versions.append(fixed_version) filtered_cves = [ alias for alias in vulnerability.get("aliases", []) if "CVE" in alias ] - for cve in filtered_cves: - description += f"- [{cve}]({CVE_MITRE_URL}{cve}) " - description += f": {vulnerability.get('details')}\n" + + if len(filtered_cves) > 0: + for cve in filtered_cves: + description += f"- [{cve}]({CVE_MITRE_URL}{cve}) : " + else: + description += f"- {vulnerability.get('id')} : " + description += f"{vulnerability.get('details')}\n" severity = vulnerability.get("database_specific", {}).get("severity") risk = _vuln_risk_rating(risk=severity, cves=filtered_cves, api_key=api_key) @@ -216,7 +228,7 @@ def parse_vulnerabilities_osv_api( if old_risk is not None and new_risk is not None and old_risk > new_risk: highest_risk_vuln_info["risk"] = risk highest_risk_vuln_info["cvss_v3_vector"] = _get_cvss_v3_vector( - vulnerability.get("severity") + vulnerability.get("severity", []) ) highest_risk_vuln_info["summary"] = vulnerability.get("summary", "") @@ -245,6 +257,23 @@ def parse_vulnerabilities_osv_api( ] +def _whitelist_vulnz_from_ecosystems( + vulnerabilities: list[dict[str, Any]], + whitelisted_ecosystems: list[str] | None = None, +) -> list[dict[str, Any]]: + if whitelisted_ecosystems is None or len(whitelisted_ecosystems) == 0: + return vulnerabilities + whitelisted = [] + for vuln in vulnerabilities: + affected_data = vuln.get("affected", []) + ecosystem = None + if len(affected_data) > 0: + ecosystem = affected_data[0].get("package", {}).get("ecosystem") + if ecosystem in whitelisted_ecosystems: + whitelisted.append(vuln) + return whitelisted + + def _aggregate_cves(cve_ids: list[str], api_key: str | None = None) -> str: """Generate the description for the vulnerability from all the related CVEs.""" cve_list_details = "" diff --git a/tests/conftest.py b/tests/conftest.py index ce455d4..5ed9312 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -227,3 +227,14 @@ def osv_api_output_risk_invalid() -> dict[str, Any]: data = pathlib.Path(file_path).read_text(encoding="utf-8") json_data: dict[str, Any] = json.loads(data) return json_data + + +@pytest.fixture +def elf_library_fingerprint_msg() -> message.Message: + selector = "v3.fingerprint.file.library" + msg_data = { + "library_name": "opencv", + "library_version": "4.9.0", + "library_type": "ELF_LIBRARY", + } + return message.Message.from_data(selector, data=msg_data) diff --git a/tests/osv_agent_test.py b/tests/osv_agent_test.py index 6195f1f..f4ae8d6 100644 --- a/tests/osv_agent_test.py +++ b/tests/osv_agent_test.py @@ -329,9 +329,12 @@ def testAgentOSV_always_emitVulnWithValidTechnicalDetail( ) assert agent_mock[0].data["risk_rating"] == "CRITICAL" assert ( - agent_mock[0].data["technical_detail"] - == """- [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands. -""" + """- [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands.\n""" + in agent_mock[0].data["technical_detail"] + ) + assert ( + """- GHSA-f698-m2v9-5fh3 : Versions of `opencv`prior to 6.1.0 are vulnerable to Command Injection. The utils/ script find-opencv.js does not validate user input allowing attackers to execute arbitrary commands.\n\n\n## Recommendation\n\nUpgrade to version 6.1.0.\n\n""" + in agent_mock[0].data["technical_detail"] ) assert ( agent_mock[0].data["recommendation"] @@ -423,6 +426,7 @@ def testAgentOSV_whenPathInMessage_technicalDetailShouldIncludeIt( assert agent_mock[0].data["risk_rating"] == "CRITICAL" assert agent_mock[0].data["technical_detail"] == ( """Dependency `opencv` Found in `lib/arm64-v8a/libBlinkID.so` has a security issue: +- GHSA-f698-m2v9-5fh3 : Versions of `opencv`prior to 6.1.0 are vulnerable to Command Injection. The utils/ script find-opencv.js does not validate user input allowing attackers to execute arbitrary commands.\n\n\n## Recommendation\n\nUpgrade to version 6.1.0.\n - [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands. """ ) @@ -436,3 +440,39 @@ def testAgentOSV_whenPathInMessage_technicalDetailShouldIncludeIt( agent_mock[0].data["recommendation"] == "We recommend updating `opencv` to a version greater than or equal to `6.1.0`." ) + + +def testAgentOSV_whenElfLibraryFingerprintMessage_shouldExcludeNpmEcosystemVulnz( + test_agent: osv_agent.OSVAgent, + agent_mock: list[message.Message], + agent_persist_mock: dict[str | bytes, str | bytes], + elf_library_fingerprint_msg: message.Message, +) -> None: + """For fingerprints of elf or macho files, we do not know the corresponding osv ecosystem. + We use a list of accepted ecosystems. + This unit test ensures no vulnz of excluded ecosystems are reported. + """ + test_agent.process(elf_library_fingerprint_msg) + + assert len(agent_mock) == 1 + + assert ( + agent_mock[0].data["title"] + == "Use of Outdated Vulnerable Component: opencv@4.9.0" + ) + assert ( + agent_mock[0].data["dna"] + == "Use of Outdated Vulnerable Component: opencv@4.9.0" + ) + assert agent_mock[0].data["risk_rating"] == "POTENTIALLY" + assert agent_mock[0].data["technical_detail"] == ( + """```- OSV-2022-394 : OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=47190\n\n```\nCrash type: Incorrect-function-pointer-type\nCrash state:\ncv::split\ncv::split\nTestSplitAndMerge\n```\n\n- OSV-2023-444 : OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=59450\n\n```\nCrash type: Heap-buffer-overflow READ 4\nCrash state:\nopj_jp2_apply_pclr\nopj_jp2_decode\ncv::detail::Jpeg2KOpjDecoderBase::readData\n```\n\n```""" + ) + assert agent_mock[0].data["description"] == ( + """Dependency `opencv` with version `4.9.0` has a security issue.""" + ) + + assert ( + agent_mock[0].data["recommendation"] + == "We recommend updating `opencv` to the latest available version." + )