From 62a048ac8a0efccf32793c1179b08b0fc7a867e2 Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 08:59:05 +0000 Subject: [PATCH 1/9] Add support for whitelisting ecosystems. --- agent/api_manager/osv_service_api.py | 7 ++++-- agent/cve_service_api.py | 7 ++++-- agent/osv_agent.py | 10 +++++++- agent/osv_output_handler.py | 36 ++++++++++++++++++++++++---- tests/conftest.py | 11 +++++++++ tests/osv_agent_test.py | 36 ++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 10 deletions(-) diff --git a/agent/api_manager/osv_service_api.py b/agent/api_manager/osv_service_api.py index a8880f3..f8115c8 100644 --- a/agent/api_manager/osv_service_api.py +++ b/agent/api_manager/osv_service_api.py @@ -7,6 +7,9 @@ import requests import tenacity +from tenacity import stop +from tenacity import wait + logger = logging.getLogger(__name__) @@ -27,8 +30,8 @@ class VulnData: @tenacity.retry( - stop=tenacity.stop_after_attempt(NUMBER_RETRIES), - wait=tenacity.wait_fixed(WAIT_BETWEEN_RETRIES), + stop=stop.stop_after_attempt(NUMBER_RETRIES), + wait=wait.wait_fixed(WAIT_BETWEEN_RETRIES), retry=tenacity.retry_if_exception_type(), retry_error_callback=lambda retry_state: retry_state.outcome.result() if retry_state.outcome is not None diff --git a/agent/cve_service_api.py b/agent/cve_service_api.py index 142253b..b590154 100644 --- a/agent/cve_service_api.py +++ b/agent/cve_service_api.py @@ -5,6 +5,9 @@ import requests import tenacity +from tenacity import stop +from tenacity import wait + CVE_MITRE_BASE_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=" REQUEST_TIMEOUT = 60 @@ -24,9 +27,9 @@ class CVE: @tenacity.retry( - stop=tenacity.stop_after_attempt(10), + stop=stop.stop_after_attempt(10), # wait for 30 seconds before retrying - wait=tenacity.wait_fixed(30), + wait=wait.wait_fixed(30), retry=tenacity.retry_if_exception_type( (requests.ConnectionError, requests.HTTPError, json.JSONDecodeError) ), diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 438598c..1e96d06 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -36,15 +36,21 @@ "requirements.txt", "yarn.lock", ] + OSV_ECOSYSTEM_MAPPING = { "JAVASCRIPT_LIBRARY": "npm", "JAVA_LIBRARY": "Maven", "FLUTTER_FRAMEWORK": "Pub", - "CORDOVA_LIBRARY": "npm", + "CORDOVA_FRAMEWORK": "npm", "DOTNET_FRAMEWORK": "NuGet", "IOS_FRAMEWORK": "SwiftURL", } +OSV_WHITELISTED_ECOSYSTEM = { + "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], + "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], +} + logging.basicConfig( format="%(message)s", datefmt="[%X]", @@ -182,6 +188,7 @@ def _process_fingerprint_file(self, message: m.Message) -> None: version=package_version, ecosystem=OSV_ECOSYSTEM_MAPPING.get(str(package_type)), ) + if api_result is None or api_result == {}: return None @@ -190,6 +197,7 @@ def _process_fingerprint_file(self, message: m.Message) -> None: package_name=package_name, package_version=package_version, api_key=self.api_key, + whitelisted_ecosystems=OSV_WHITELISTED_ECOSYSTEM.get(str(package_type)), ) if parsed_osv_output is None: diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index 5117bdc..74a8dd3 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -167,7 +167,7 @@ def parse_vulnerabilities_osv_binary( return parsed_vulns except json.JSONDecodeError as e: - logger.error(f"Error decoding JSON: {e}") + logger.error("Error decoding JSON: %s", e) return [] @@ -176,6 +176,7 @@ def parse_vulnerabilities_osv_api( package_name: str, package_version: str, api_key: str | None = None, + whitelisted_ecosystems: list[str] | None = None, ) -> list[VulnData]: """Parse the OSV API response to extract vulnerabilities. Args: @@ -195,16 +196,24 @@ def parse_vulnerabilities_osv_api( highest_risk_vuln_info: dict[str, str] = {} if len(vulnerabilities) == 0: return [] - for vulnerability in vulnerabilities: + + whitlisted_vulnerabilities = _whitelist_vulnz_from_ecosystems( + vulnerabilities, whitelisted_ecosystems + ) + for vulnerability in whitlisted_vulnerabilities: fixed_version = _get_fixed_version(vulnerability.get("affected")) if fixed_version != "": fixed_versions.append(fixed_version) filtered_cves = [ alias for alias in vulnerability.get("aliases", []) if "CVE" in alias ] - for cve in filtered_cves: - description += f"- [{cve}]({CVE_MITRE_URL}{cve}) " - description += f": {vulnerability.get('details')}\n" + + if len(filtered_cves) > 0: + for cve in filtered_cves: + description += f"- [{cve}]({CVE_MITRE_URL}{cve}) : " + else: + description += f"- {vulnerability.get('id')} : " + description += f"{vulnerability.get('details')}\n\n" severity = vulnerability.get("database_specific", {}).get("severity") risk = _vuln_risk_rating(risk=severity, cves=filtered_cves, api_key=api_key) @@ -245,6 +254,23 @@ def parse_vulnerabilities_osv_api( ] +def _whitelist_vulnz_from_ecosystems( + vulnerabilities: list[dict[str, Any]], + whitelisted_ecosystems: list[str] | None = None, +) -> list[dict[str, Any]]: + if whitelisted_ecosystems is None or len(whitelisted_ecosystems) == 0: + return vulnerabilities + whitelisted = [] + for vuln in vulnerabilities: + affected_data = vuln.get("affected", []) + ecosystem = None + if len(affected_data) > 0: + ecosystem = affected_data[0].get("package", {}).get("ecosystem") + if ecosystem in whitelisted_ecosystems: + whitelisted.append(vuln) + return whitelisted + + def _aggregate_cves(cve_ids: list[str], api_key: str | None = None) -> str: """Generate the description for the vulnerability from all the related CVEs.""" cve_list_details = "" diff --git a/tests/conftest.py b/tests/conftest.py index ce455d4..5ed9312 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -227,3 +227,14 @@ def osv_api_output_risk_invalid() -> dict[str, Any]: data = pathlib.Path(file_path).read_text(encoding="utf-8") json_data: dict[str, Any] = json.loads(data) return json_data + + +@pytest.fixture +def elf_library_fingerprint_msg() -> message.Message: + selector = "v3.fingerprint.file.library" + msg_data = { + "library_name": "opencv", + "library_version": "4.9.0", + "library_type": "ELF_LIBRARY", + } + return message.Message.from_data(selector, data=msg_data) diff --git a/tests/osv_agent_test.py b/tests/osv_agent_test.py index 6195f1f..29a1303 100644 --- a/tests/osv_agent_test.py +++ b/tests/osv_agent_test.py @@ -436,3 +436,39 @@ def testAgentOSV_whenPathInMessage_technicalDetailShouldIncludeIt( agent_mock[0].data["recommendation"] == "We recommend updating `opencv` to a version greater than or equal to `6.1.0`." ) + + +def testAgentOSV_whenElfLibraryFingerprintMessage_shouldExcludeNpmEcosystemVulnz( + test_agent: osv_agent.OSVAgent, + agent_mock: list[message.Message], + agent_persist_mock: dict[str | bytes, str | bytes], + elf_library_fingerprint_msg: message.Message, +) -> None: + """For fingerprints of elf or macho files, we do not know the corresponding osv ecosystem. + We use a list of accepted ecosystems. + This unit test ensures no vulnz of excluded ecosystems are reported. + """ + test_agent.process(elf_library_fingerprint_msg) + + assert len(agent_mock) == 1 + + assert ( + agent_mock[0].data["title"] + == "Use of Outdated Vulnerable Component: opencv@4.9.0" + ) + assert ( + agent_mock[0].data["dna"] + == "Use of Outdated Vulnerable Component: opencv@4.9.0" + ) + assert agent_mock[0].data["risk_rating"] == "POTENTIALLY" + assert agent_mock[0].data["technical_detail"] == ( + """```OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=47190\n\n```\nCrash type: Incorrect-function-pointer-type\nCrash state:\ncv::split\ncv::split\nTestSplitAndMerge\n```\n\nOSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=59450\n\n```\nCrash type: Heap-buffer-overflow READ 4\nCrash state:\nopj_jp2_apply_pclr\nopj_jp2_decode\ncv::detail::Jpeg2KOpjDecoderBase::readData\n```\n\n```""" + ) + assert agent_mock[0].data["description"] == ( + """Dependency `opencv` with version `4.9.0` has a security issue.""" + ) + + assert ( + agent_mock[0].data["recommendation"] + == "We recommend updating `opencv` to the latest available version." + ) From 6a003c4a63f79d1a6f3cc50987b0ee45b2136f1f Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 11:32:31 +0000 Subject: [PATCH 2/9] refactor + fox type hint issue. --- agent/osv_agent.py | 9 ++------- agent/osv_output_handler.py | 14 ++++++++++---- tests/osv_agent_test.py | 12 ++++++++---- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 1e96d06..ba7c860 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -46,10 +46,6 @@ "IOS_FRAMEWORK": "SwiftURL", } -OSV_WHITELISTED_ECOSYSTEM = { - "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], - "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], -} logging.basicConfig( format="%(message)s", @@ -177,10 +173,9 @@ def _process_fingerprint_file(self, message: m.Message) -> None: path = message.data.get("path") if package_version is None: - logger.error("Error: Version must not be None.") return None if package_name is None: - logger.error("Error: Package name must not be None.") + logger.warning("Error: Package name must not be None.") return None api_result = osv_service_api.query_osv_api( @@ -197,7 +192,7 @@ def _process_fingerprint_file(self, message: m.Message) -> None: package_name=package_name, package_version=package_version, api_key=self.api_key, - whitelisted_ecosystems=OSV_WHITELISTED_ECOSYSTEM.get(str(package_type)), + package_type=package_type, ) if parsed_osv_output is None: diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index 74a8dd3..bd8c49b 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -32,6 +32,12 @@ "POTENTIALLY": 5, } +OSV_WHITELISTED_ECOSYSTEM = { + "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"], + "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], +} + + logger = logging.getLogger(__name__) @@ -176,7 +182,7 @@ def parse_vulnerabilities_osv_api( package_name: str, package_version: str, api_key: str | None = None, - whitelisted_ecosystems: list[str] | None = None, + package_type: str | None = None, ) -> list[VulnData]: """Parse the OSV API response to extract vulnerabilities. Args: @@ -198,7 +204,7 @@ def parse_vulnerabilities_osv_api( return [] whitlisted_vulnerabilities = _whitelist_vulnz_from_ecosystems( - vulnerabilities, whitelisted_ecosystems + vulnerabilities, OSV_WHITELISTED_ECOSYSTEM.get(package_type) ) for vulnerability in whitlisted_vulnerabilities: fixed_version = _get_fixed_version(vulnerability.get("affected")) @@ -213,7 +219,7 @@ def parse_vulnerabilities_osv_api( description += f"- [{cve}]({CVE_MITRE_URL}{cve}) : " else: description += f"- {vulnerability.get('id')} : " - description += f"{vulnerability.get('details')}\n\n" + description += f"{vulnerability.get('details')}\n" severity = vulnerability.get("database_specific", {}).get("severity") risk = _vuln_risk_rating(risk=severity, cves=filtered_cves, api_key=api_key) @@ -225,7 +231,7 @@ def parse_vulnerabilities_osv_api( if old_risk is not None and new_risk is not None and old_risk > new_risk: highest_risk_vuln_info["risk"] = risk highest_risk_vuln_info["cvss_v3_vector"] = _get_cvss_v3_vector( - vulnerability.get("severity") + vulnerability.get("severity", []) ) highest_risk_vuln_info["summary"] = vulnerability.get("summary", "") diff --git a/tests/osv_agent_test.py b/tests/osv_agent_test.py index 29a1303..f4ae8d6 100644 --- a/tests/osv_agent_test.py +++ b/tests/osv_agent_test.py @@ -329,9 +329,12 @@ def testAgentOSV_always_emitVulnWithValidTechnicalDetail( ) assert agent_mock[0].data["risk_rating"] == "CRITICAL" assert ( - agent_mock[0].data["technical_detail"] - == """- [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands. -""" + """- [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands.\n""" + in agent_mock[0].data["technical_detail"] + ) + assert ( + """- GHSA-f698-m2v9-5fh3 : Versions of `opencv`prior to 6.1.0 are vulnerable to Command Injection. The utils/ script find-opencv.js does not validate user input allowing attackers to execute arbitrary commands.\n\n\n## Recommendation\n\nUpgrade to version 6.1.0.\n\n""" + in agent_mock[0].data["technical_detail"] ) assert ( agent_mock[0].data["recommendation"] @@ -423,6 +426,7 @@ def testAgentOSV_whenPathInMessage_technicalDetailShouldIncludeIt( assert agent_mock[0].data["risk_rating"] == "CRITICAL" assert agent_mock[0].data["technical_detail"] == ( """Dependency `opencv` Found in `lib/arm64-v8a/libBlinkID.so` has a security issue: +- GHSA-f698-m2v9-5fh3 : Versions of `opencv`prior to 6.1.0 are vulnerable to Command Injection. The utils/ script find-opencv.js does not validate user input allowing attackers to execute arbitrary commands.\n\n\n## Recommendation\n\nUpgrade to version 6.1.0.\n - [CVE-2019-10061](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-10061) : utils/find-opencv.js in node-opencv (aka OpenCV bindings for Node.js) prior to 6.1.0 is vulnerable to Command Injection. It does not validate user input allowing attackers to execute arbitrary commands. """ ) @@ -462,7 +466,7 @@ def testAgentOSV_whenElfLibraryFingerprintMessage_shouldExcludeNpmEcosystemVulnz ) assert agent_mock[0].data["risk_rating"] == "POTENTIALLY" assert agent_mock[0].data["technical_detail"] == ( - """```OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=47190\n\n```\nCrash type: Incorrect-function-pointer-type\nCrash state:\ncv::split\ncv::split\nTestSplitAndMerge\n```\n\nOSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=59450\n\n```\nCrash type: Heap-buffer-overflow READ 4\nCrash state:\nopj_jp2_apply_pclr\nopj_jp2_decode\ncv::detail::Jpeg2KOpjDecoderBase::readData\n```\n\n```""" + """```- OSV-2022-394 : OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=47190\n\n```\nCrash type: Incorrect-function-pointer-type\nCrash state:\ncv::split\ncv::split\nTestSplitAndMerge\n```\n\n- OSV-2023-444 : OSS-Fuzz report: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=59450\n\n```\nCrash type: Heap-buffer-overflow READ 4\nCrash state:\nopj_jp2_apply_pclr\nopj_jp2_decode\ncv::detail::Jpeg2KOpjDecoderBase::readData\n```\n\n```""" ) assert agent_mock[0].data["description"] == ( """Dependency `opencv` with version `4.9.0` has a security issue.""" From 65fdc8e47b15fda632cdc974c7ef502b697843fe Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 11:35:27 +0000 Subject: [PATCH 3/9] minor type hint fix. --- agent/osv_output_handler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index bd8c49b..c28cdc7 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -203,10 +203,11 @@ def parse_vulnerabilities_osv_api( if len(vulnerabilities) == 0: return [] - whitlisted_vulnerabilities = _whitelist_vulnz_from_ecosystems( - vulnerabilities, OSV_WHITELISTED_ECOSYSTEM.get(package_type) - ) - for vulnerability in whitlisted_vulnerabilities: + if package_type is not None: + vulnerabilities = _whitelist_vulnz_from_ecosystems( + vulnerabilities, OSV_WHITELISTED_ECOSYSTEM.get(package_type) + ) + for vulnerability in vulnerabilities: fixed_version = _get_fixed_version(vulnerability.get("affected")) if fixed_version != "": fixed_versions.append(fixed_version) From 28359616edfd1691ebc1abb017fb7b0ff1283084 Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 13:32:46 +0000 Subject: [PATCH 4/9] refactor to only have one mapping logic of lib type. --- agent/osv_agent.py | 39 +++++++++++++++++++++++-------------- agent/osv_output_handler.py | 12 ++++-------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index ba7c860..9c59081 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -38,12 +38,14 @@ ] OSV_ECOSYSTEM_MAPPING = { - "JAVASCRIPT_LIBRARY": "npm", - "JAVA_LIBRARY": "Maven", - "FLUTTER_FRAMEWORK": "Pub", - "CORDOVA_FRAMEWORK": "npm", - "DOTNET_FRAMEWORK": "NuGet", - "IOS_FRAMEWORK": "SwiftURL", + "JAVASCRIPT_LIBRARY": ["npm"], + "JAVA_LIBRARY": ["Maven"], + "FLUTTER_FRAMEWORK": ["Pub"], + "CORDOVA_FRAMEWORK": ["npm"], + "DOTNET_FRAMEWORK": ["NuGet"], + "IOS_FRAMEWORK": ["SwiftURL"], + "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"], + "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], } @@ -178,21 +180,28 @@ def _process_fingerprint_file(self, message: m.Message) -> None: logger.warning("Error: Package name must not be None.") return None - api_result = osv_service_api.query_osv_api( - package_name=package_name, - version=package_version, - ecosystem=OSV_ECOSYSTEM_MAPPING.get(str(package_type)), - ) + all_vulnerabilities = [] + ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type)) + if ecosystems is not None and len(ecosystems) > 0: + api_result = osv_service_api.query_osv_api( + package_name=package_name, + version=package_version, + ecosystems=OSV_ECOSYSTEM_MAPPING.get(str(package_type)), + ) - if api_result is None or api_result == {}: - return None + if api_result is None or api_result == {}: + return None + + vulnerabilities = api_result.get("vulns", []) or api_result.get( + "vulnerabilities", [] + ) + all_vulnerabilities.extend(vulnerabilities) parsed_osv_output = osv_output_handler.parse_vulnerabilities_osv_api( - output=api_result, + vulnerabilities=all_vulnerabilities, package_name=package_name, package_version=package_version, api_key=self.api_key, - package_type=package_type, ) if parsed_osv_output is None: diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index c28cdc7..90a3238 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -178,24 +178,24 @@ def parse_vulnerabilities_osv_binary( def parse_vulnerabilities_osv_api( - output: dict[str, Any], + vulnerabilities: list[dict[str, Any]], package_name: str, package_version: str, api_key: str | None = None, - package_type: str | None = None, ) -> list[VulnData]: """Parse the OSV API response to extract vulnerabilities. + Args: - output: The API response json. + output: The list of vulnerabilities raw from the API response . package_name: The package name. package_version: The package version. api_key: The NVD API key. + Returns: Parsed output. """ cves_list: List[str] = [] risks_list: List[str] = [] - vulnerabilities = output.get("vulns", []) or output.get("vulnerabilities", []) fixed_versions: list[str] = [] references: List[dict[str, Any]] = [] description = "" @@ -203,10 +203,6 @@ def parse_vulnerabilities_osv_api( if len(vulnerabilities) == 0: return [] - if package_type is not None: - vulnerabilities = _whitelist_vulnz_from_ecosystems( - vulnerabilities, OSV_WHITELISTED_ECOSYSTEM.get(package_type) - ) for vulnerability in vulnerabilities: fixed_version = _get_fixed_version(vulnerability.get("affected")) if fixed_version != "": From 489185a8d7c38a99b505f8b51e285730e5eda0c2 Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 14:35:43 +0000 Subject: [PATCH 5/9] tweaks. --- agent/osv_agent.py | 28 +++++++++++++++++++++------- agent/osv_output_handler.py | 2 +- tests/osv_output_handler_test.py | 2 +- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 9c59081..9ccd1fa 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -183,19 +183,33 @@ def _process_fingerprint_file(self, message: m.Message) -> None: all_vulnerabilities = [] ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type)) if ecosystems is not None and len(ecosystems) > 0: + for ecosystem in ecosystems: + api_result = osv_service_api.query_osv_api( + package_name=package_name, + version=package_version, + ecosystem=ecosystem, + ) + + if api_result is None or api_result == {}: + continue + + vulnerabilities = api_result.get("vulns", []) or api_result.get( + "vulnerabilities", [] + ) + all_vulnerabilities.extend(vulnerabilities) + else: api_result = osv_service_api.query_osv_api( package_name=package_name, version=package_version, - ecosystems=OSV_ECOSYSTEM_MAPPING.get(str(package_type)), + ecosystem=None, ) - - if api_result is None or api_result == {}: - return None - - vulnerabilities = api_result.get("vulns", []) or api_result.get( + api_result = api_result or {} + all_vulnerabilities = api_result.get("vulns", []) or api_result.get( "vulnerabilities", [] ) - all_vulnerabilities.extend(vulnerabilities) + + if len(all_vulnerabilities) == 0: + return None parsed_osv_output = osv_output_handler.parse_vulnerabilities_osv_api( vulnerabilities=all_vulnerabilities, diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index 90a3238..184a877 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -186,7 +186,7 @@ def parse_vulnerabilities_osv_api( """Parse the OSV API response to extract vulnerabilities. Args: - output: The list of vulnerabilities raw from the API response . + vulnerabilities: The list of vulnerabilities raw from the API response . package_name: The package name. package_version: The package version. api_key: The NVD API key. diff --git a/tests/osv_output_handler_test.py b/tests/osv_output_handler_test.py index 964f629..1f71923 100644 --- a/tests/osv_output_handler_test.py +++ b/tests/osv_output_handler_test.py @@ -40,7 +40,7 @@ def testPasrseOSVOutput_withValidResponse_returnListOfVulnzData( ) -> None: """Parse the output of osv api call.""" cves_data = osv_output_handler.parse_vulnerabilities_osv_api( - osv_api_output, package_name="lodash", package_version="4.10.0" + osv_api_output.get("vulns", []), package_name="lodash", package_version="4.10.0" ) assert len(cves_data) == 1 From b5c0f39c6b9974cfb96bf17fb3f6a4d48bd1398a Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 16:07:00 +0000 Subject: [PATCH 6/9] refactor. --- agent/osv_agent.py | 52 +++++++++++++------------------------ agent/osv_output_handler.py | 16 ++++++------ 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 9ccd1fa..44477bb 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -38,12 +38,12 @@ ] OSV_ECOSYSTEM_MAPPING = { - "JAVASCRIPT_LIBRARY": ["npm"], - "JAVA_LIBRARY": ["Maven"], - "FLUTTER_FRAMEWORK": ["Pub"], - "CORDOVA_FRAMEWORK": ["npm"], - "DOTNET_FRAMEWORK": ["NuGet"], - "IOS_FRAMEWORK": ["SwiftURL"], + "JAVASCRIPT_LIBRARY": "npm", + "JAVA_LIBRARY": "Maven", + "FLUTTER_FRAMEWORK": "Pub", + "CORDOVA_FRAMEWORK": "npm", + "DOTNET_FRAMEWORK": "NuGet", + "IOS_FRAMEWORK": "SwiftURL", "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"], "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], } @@ -180,44 +180,28 @@ def _process_fingerprint_file(self, message: m.Message) -> None: logger.warning("Error: Package name must not be None.") return None - all_vulnerabilities = [] ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type)) - if ecosystems is not None and len(ecosystems) > 0: - for ecosystem in ecosystems: - api_result = osv_service_api.query_osv_api( - package_name=package_name, - version=package_version, - ecosystem=ecosystem, - ) - - if api_result is None or api_result == {}: - continue + whitelisted_ecosystems = None + if isinstance(ecosystems, list): + ecosystem = None + whitelisted_ecosystems = typing.cast(list[str], ecosystems) - vulnerabilities = api_result.get("vulns", []) or api_result.get( - "vulnerabilities", [] - ) - all_vulnerabilities.extend(vulnerabilities) - else: - api_result = osv_service_api.query_osv_api( - package_name=package_name, - version=package_version, - ecosystem=None, - ) - api_result = api_result or {} - all_vulnerabilities = api_result.get("vulns", []) or api_result.get( - "vulnerabilities", [] - ) + api_result = osv_service_api.query_osv_api( + package_name=package_name, + version=package_version, + ecosystem=ecosystem, + ) - if len(all_vulnerabilities) == 0: + if api_result is None or api_result == {}: return None parsed_osv_output = osv_output_handler.parse_vulnerabilities_osv_api( - vulnerabilities=all_vulnerabilities, + output=api_result, package_name=package_name, package_version=package_version, api_key=self.api_key, + whitelisted_ecosystems=whitelisted_ecosystems, ) - if parsed_osv_output is None: return None diff --git a/agent/osv_output_handler.py b/agent/osv_output_handler.py index 184a877..2fed2b8 100644 --- a/agent/osv_output_handler.py +++ b/agent/osv_output_handler.py @@ -32,11 +32,6 @@ "POTENTIALLY": 5, } -OSV_WHITELISTED_ECOSYSTEM = { - "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"], - "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], -} - logger = logging.getLogger(__name__) @@ -178,15 +173,16 @@ def parse_vulnerabilities_osv_binary( def parse_vulnerabilities_osv_api( - vulnerabilities: list[dict[str, Any]], + output: dict[str, Any], package_name: str, package_version: str, api_key: str | None = None, + whitelisted_ecosystems: list[str] | None = None, ) -> list[VulnData]: """Parse the OSV API response to extract vulnerabilities. Args: - vulnerabilities: The list of vulnerabilities raw from the API response . + output: The list of vulnerabilities raw from the API response . package_name: The package name. package_version: The package version. api_key: The NVD API key. @@ -196,6 +192,7 @@ def parse_vulnerabilities_osv_api( """ cves_list: List[str] = [] risks_list: List[str] = [] + vulnerabilities = output.get("vulns", []) or output.get("vulnerabilities", []) fixed_versions: list[str] = [] references: List[dict[str, Any]] = [] description = "" @@ -203,7 +200,10 @@ def parse_vulnerabilities_osv_api( if len(vulnerabilities) == 0: return [] - for vulnerability in vulnerabilities: + whitlisted_vulnerabilities = _whitelist_vulnz_from_ecosystems( + vulnerabilities, whitelisted_ecosystems + ) + for vulnerability in whitlisted_vulnerabilities: fixed_version = _get_fixed_version(vulnerability.get("affected")) if fixed_version != "": fixed_versions.append(fixed_version) From 4d13abf1bc58414ac2d222e959c7b7ead945bd8b Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Wed, 13 Mar 2024 16:12:03 +0000 Subject: [PATCH 7/9] minor refactor. --- agent/osv_agent.py | 6 ++++-- tests/osv_output_handler_test.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 44477bb..ddca5ed 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -182,8 +182,10 @@ def _process_fingerprint_file(self, message: m.Message) -> None: ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type)) whitelisted_ecosystems = None - if isinstance(ecosystems, list): - ecosystem = None + ecosystem = None + if isinstance(ecosystems, str): + ecosystem = ecosystems + elif isinstance(ecosystems, list): whitelisted_ecosystems = typing.cast(list[str], ecosystems) api_result = osv_service_api.query_osv_api( diff --git a/tests/osv_output_handler_test.py b/tests/osv_output_handler_test.py index 1f71923..964f629 100644 --- a/tests/osv_output_handler_test.py +++ b/tests/osv_output_handler_test.py @@ -40,7 +40,7 @@ def testPasrseOSVOutput_withValidResponse_returnListOfVulnzData( ) -> None: """Parse the output of osv api call.""" cves_data = osv_output_handler.parse_vulnerabilities_osv_api( - osv_api_output.get("vulns", []), package_name="lodash", package_version="4.10.0" + osv_api_output, package_name="lodash", package_version="4.10.0" ) assert len(cves_data) == 1 From 0519030c62731c9da96e6276b5f3c8fb27236eb7 Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Thu, 14 Mar 2024 09:00:13 +0000 Subject: [PATCH 8/9] Address comment. --- agent/osv_agent.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index ddca5ed..25f7193 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -38,12 +38,12 @@ ] OSV_ECOSYSTEM_MAPPING = { - "JAVASCRIPT_LIBRARY": "npm", - "JAVA_LIBRARY": "Maven", - "FLUTTER_FRAMEWORK": "Pub", - "CORDOVA_FRAMEWORK": "npm", - "DOTNET_FRAMEWORK": "NuGet", - "IOS_FRAMEWORK": "SwiftURL", + "JAVASCRIPT_LIBRARY": ["npm"], + "JAVA_LIBRARY": ["Maven"], + "FLUTTER_FRAMEWORK": ["Pub"], + "CORDOVA_FRAMEWORK": ["npm"], + "DOTNET_FRAMEWORK": ["NuGet"], + "IOS_FRAMEWORK": ["SwiftURL"], "ELF_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux"], "MACHO_LIBRARY": ["OSS-Fuzz", "Alpine", "Debian", "Linux", "SwiftURL"], } @@ -180,12 +180,12 @@ def _process_fingerprint_file(self, message: m.Message) -> None: logger.warning("Error: Package name must not be None.") return None - ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type)) + ecosystems = OSV_ECOSYSTEM_MAPPING.get(str(package_type), []) whitelisted_ecosystems = None ecosystem = None - if isinstance(ecosystems, str): - ecosystem = ecosystems - elif isinstance(ecosystems, list): + if len(ecosystems) == 1: + ecosystem = ecosystems[0] + elif len(ecosystems) > 1: whitelisted_ecosystems = typing.cast(list[str], ecosystems) api_result = osv_service_api.query_osv_api( From cd2d69fa0fb29a0591cc57159f9e81d6f83a4139 Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Thu, 14 Mar 2024 09:25:06 +0000 Subject: [PATCH 9/9] Typehint fix. --- agent/osv_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/osv_agent.py b/agent/osv_agent.py index 25f7193..1b70f06 100644 --- a/agent/osv_agent.py +++ b/agent/osv_agent.py @@ -186,7 +186,7 @@ def _process_fingerprint_file(self, message: m.Message) -> None: if len(ecosystems) == 1: ecosystem = ecosystems[0] elif len(ecosystems) > 1: - whitelisted_ecosystems = typing.cast(list[str], ecosystems) + whitelisted_ecosystems = ecosystems api_result = osv_service_api.query_osv_api( package_name=package_name,