Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix redhat import failure #418

Merged
merged 4 commits into from
Apr 2, 2021
Merged

Fix redhat import failure #418

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2684f05
requests_with_5xx_retry: Retry on 5xx errors
Hritik14 Mar 29, 2021
aff0065
Not all RHSA errata have a CVRF document
Hritik14 Mar 31, 2021
b0329fa
Remove debugging symbols and black -l 100
Hritik14 Apr 1, 2021
c90a871
Mock requests_session in redhat importer
Hritik14 Apr 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions vulnerabilities/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import requests
import toml
import urllib3
import yaml

# TODO add logging here
Expand Down Expand Up @@ -79,3 +80,21 @@ def create_etag(data_src, url, etag_key):


is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match


def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You said there were 104 errors, how are they addressed ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

104 came after retrying with short delays for multiple times. I increased the backoff_factor to 1 in the redhat importer.

"""
Returns a requests sessions which retries on 5xx errors with
a backoff_factor
"""
retries = urllib3.util.Retry(
total=max_retries,
backoff_factor=backoff_factor,
raise_on_status=True,
status_forcelist=range(500, 600, 1),
)
adapter = requests.adapters.HTTPAdapter(max_retries=retries)
session = requests.Session()
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
32 changes: 22 additions & 10 deletions vulnerabilities/importers/redhat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,15 @@
# VulnerableCode is a free software code from nexB Inc. and others.
# Visit https://github.com/nexB/vulnerablecode/ for support and download.

from packageurl import PackageURL
import requests
from packageurl import PackageURL

from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import DataSource
from vulnerabilities.data_source import DataSourceConfiguration
from vulnerabilities.data_source import Reference
from vulnerabilities.data_source import VulnerabilitySeverity
from vulnerabilities.helpers import requests_with_5xx_retry
from vulnerabilities.severity_systems import scoring_systems


Expand All @@ -43,6 +44,9 @@ def updated_advisories(self):
return self.batch_advisories(processed_advisories)


requests_session = requests_with_5xx_retry(max_retries=5, backoff_factor=1)


def fetch():
"""
Return a list of CVE data mappings fetched from the RedHat API.
Expand All @@ -58,7 +62,7 @@ def fetch():
current_url = url_template.format(page_no)
try:
print(f"Fetching: {current_url}")
response = requests.get(current_url)
response = requests_session.get(current_url)
if response.status_code != requests.codes.ok:
# TODO: log me
print(f"Failed to fetch results from {current_url}")
Expand Down Expand Up @@ -90,7 +94,9 @@ def to_advisory(advisory_data):
bugzilla = advisory_data.get("bugzilla")
if bugzilla:
url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla)
bugzilla_data = requests.get(f"https://bugzilla.redhat.com/rest/bug/{bugzilla}").json()
bugzilla_data = requests_session.get(
f"https://bugzilla.redhat.com/rest/bug/{bugzilla}"
).json()
if (
bugzilla_data.get("bugs")
and len(bugzilla_data["bugs"])
Expand All @@ -115,18 +121,24 @@ def to_advisory(advisory_data):
# See https://access.redhat.com/articles/2130961 for more details.

if "RHSA" in rh_adv.upper():
rhsa_data = requests.get(
rhsa_data = requests_session.get(
f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json"
).json() # nopep8
value = rhsa_data["cvrfdoc"]["aggregate_severity"]
rhsa_aggregate_severity = VulnerabilitySeverity(
system=scoring_systems["rhas"],
value=value,
)

rhsa_aggregate_severities = []
if rhsa_data.get("cvrfdoc"):
# not all RHSA errata have a corresponding CVRF document
value = rhsa_data["cvrfdoc"]["aggregate_severity"]
rhsa_aggregate_severities.append(
VulnerabilitySeverity(
system=scoring_systems["rhas"],
value=value,
)
)

references.append(
Reference(
severities=[rhsa_aggregate_severity],
severities=rhsa_aggregate_severities,
url="https://access.redhat.com/errata/{}".format(rh_adv),
reference_id=rh_adv,
)
Expand Down
2 changes: 1 addition & 1 deletion vulnerabilities/tests/test_redhat_importer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def test_to_advisory(self):
}
for adv in data:
with unittest.mock.patch(
"vulnerabilities.importers.redhat.requests.get", return_value=mock_resp
"vulnerabilities.importers.redhat.requests_session.get", return_value=mock_resp
):
adv = redhat.to_advisory(adv)
found_advisories.append(adv)
Expand Down