diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 36bf7c865..77376ee94 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -25,6 +25,7 @@ import requests import toml +import urllib3 import yaml # TODO add logging here @@ -79,3 +80,21 @@ def create_etag(data_src, url, etag_key): is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match + + +def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5): + """ + Returns a requests sessions which retries on 5xx errors with + a backoff_factor + """ + retries = urllib3.util.Retry( + total=max_retries, + backoff_factor=backoff_factor, + raise_on_status=True, + status_forcelist=range(500, 600, 1), + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retries) + session = requests.Session() + session.mount("https://", adapter) + session.mount("http://", adapter) + return session diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index d96036ae4..9415cd94a 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -20,14 +20,15 @@ # VulnerableCode is a free software code from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from packageurl import PackageURL import requests +from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.helpers import requests_with_5xx_retry from vulnerabilities.severity_systems import scoring_systems @@ -43,6 +44,9 @@ def updated_advisories(self): return self.batch_advisories(processed_advisories) +requests_session = requests_with_5xx_retry(max_retries=5, backoff_factor=1) + + def fetch(): """ Return a list of CVE data mappings fetched from the RedHat API. @@ -58,7 +62,7 @@ def fetch(): current_url = url_template.format(page_no) try: print(f"Fetching: {current_url}") - response = requests.get(current_url) + response = requests_session.get(current_url) if response.status_code != requests.codes.ok: # TODO: log me print(f"Failed to fetch results from {current_url}") @@ -90,7 +94,9 @@ def to_advisory(advisory_data): bugzilla = advisory_data.get("bugzilla") if bugzilla: url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) - bugzilla_data = requests.get(f"https://bugzilla.redhat.com/rest/bug/{bugzilla}").json() + bugzilla_data = requests_session.get( + f"https://bugzilla.redhat.com/rest/bug/{bugzilla}" + ).json() if ( bugzilla_data.get("bugs") and len(bugzilla_data["bugs"]) @@ -115,18 +121,24 @@ def to_advisory(advisory_data): # See https://access.redhat.com/articles/2130961 for more details. if "RHSA" in rh_adv.upper(): - rhsa_data = requests.get( + rhsa_data = requests_session.get( f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" ).json() # nopep8 - value = rhsa_data["cvrfdoc"]["aggregate_severity"] - rhsa_aggregate_severity = VulnerabilitySeverity( - system=scoring_systems["rhas"], - value=value, - ) + + rhsa_aggregate_severities = [] + if rhsa_data.get("cvrfdoc"): + # not all RHSA errata have a corresponding CVRF document + value = rhsa_data["cvrfdoc"]["aggregate_severity"] + rhsa_aggregate_severities.append( + VulnerabilitySeverity( + system=scoring_systems["rhas"], + value=value, + ) + ) references.append( Reference( - severities=[rhsa_aggregate_severity], + severities=rhsa_aggregate_severities, url="https://access.redhat.com/errata/{}".format(rh_adv), reference_id=rh_adv, ) diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index ba7631399..967b01684 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -139,7 +139,7 @@ def test_to_advisory(self): } for adv in data: with unittest.mock.patch( - "vulnerabilities.importers.redhat.requests.get", return_value=mock_resp + "vulnerabilities.importers.redhat.requests_session.get", return_value=mock_resp ): adv = redhat.to_advisory(adv) found_advisories.append(adv)