Add in utils

Rename helpers.py to utils.py

Signed-off-by: Tushar Goel <[email protected]>

vulnerabilities/importer.py

@@ -43,12 +43,12 @@
 from univers.version_range import VersionRange
 from univers.versions import Version
 
-from vulnerabilities.helpers import classproperty
-from vulnerabilities.helpers import evolve_purl
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.oval_parser import OvalParser
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.severity_systems import ScoringSystem
+from vulnerabilities.utils import classproperty
+from vulnerabilities.utils import evolve_purl
+from vulnerabilities.utils import nearest_patched_package
 
 logger = logging.getLogger(__name__)
 

vulnerabilities/importers/alpine_linux.py

@@ -33,7 +33,6 @@
 from packageurl import PackageURL
 from univers.versions import AlpineLinuxVersion
 
-from vulnerabilities.helpers import is_cve
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
@@ -44,6 +43,7 @@
 from vulnerabilities.references import WireSharkReference
 from vulnerabilities.references import XsaReference
 from vulnerabilities.references import ZbxReference
+from vulnerabilities.utils import is_cve
 
 LOGGER = logging.getLogger(__name__)
 BASE_URL = "https://secdb.alpinelinux.org/"

vulnerabilities/importers/apache_httpd.py

@@ -29,13 +29,13 @@
 from univers.version_range import VersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.severity_systems import APACHE_HTTPD
+from vulnerabilities.utils import nearest_patched_package
 
 
 class ApacheHTTPDImporter(Importer):

vulnerabilities/importers/apache_kafka.py

@@ -28,11 +28,11 @@
 from univers.version_range import VersionRange
 from univers.versions import MavenVersion
 
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.utils import nearest_patched_package
 
 GH_PAGE_URL = "https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html"
 ASF_PAGE_URL = "https://kafka.apache.org/cve-list"

vulnerabilities/importers/apache_tomcat.py

@@ -30,12 +30,12 @@
 from univers.versions import MavenVersion
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import create_etag
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import MavenVersionAPI
+from vulnerabilities.utils import create_etag
+from vulnerabilities.utils import nearest_patched_package
 
 
 class ApacheTomcatImporter(Importer):

vulnerabilities/importers/archlinux.py

@@ -31,11 +31,11 @@
 from packageurl import PackageURL
 
 from vulnerabilities import severity_systems
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.utils import nearest_patched_package
 
 
 class ArchlinuxImporter(Importer):

vulnerabilities/importers/debian.py

@@ -33,11 +33,6 @@
 from univers.version_range import DebianVersionRange
 from univers.versions import DebianVersion
 
-from vulnerabilities.helpers import AffectedPackage as LegacyAffectedPackage
-from vulnerabilities.helpers import dedupe
-from vulnerabilities.helpers import get_affected_packages_by_patched_package
-from vulnerabilities.helpers import get_item
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
@@ -47,6 +42,11 @@
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
+from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
+from vulnerabilities.utils import dedupe
+from vulnerabilities.utils import get_affected_packages_by_patched_package
+from vulnerabilities.utils import get_item
+from vulnerabilities.utils import nearest_patched_package
 
 logger = logging.getLogger(__name__)
 

vulnerabilities/importers/debian_oval.py

@@ -26,9 +26,9 @@
 
 import requests
 
-from vulnerabilities.helpers import create_etag
 from vulnerabilities.importer import OvalImporter
 from vulnerabilities.package_managers import DebianVersionAPI
+from vulnerabilities.utils import create_etag
 
 
 class DebianOvalImporter(OvalImporter):

vulnerabilities/importers/elixir_security.py

@@ -26,12 +26,12 @@
 from univers.version_range import VersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import load_yaml
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import HexVersionAPI
+from vulnerabilities.utils import load_yaml
+from vulnerabilities.utils import nearest_patched_package
 
 
 class ElixirSecurityImporter(GitImporter):

vulnerabilities/importers/gentoo.py

@@ -26,10 +26,10 @@
 
 from packageurl import PackageURL
 
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
+from vulnerabilities.utils import nearest_patched_package
 
 
 class GentooImporter(GitImporter):

vulnerabilities/importers/github.py

@@ -34,12 +34,8 @@
 from univers.version_range import VersionRange
 from univers.version_range import build_range_from_github_advisory_constraint
 
-from vulnerabilities import helpers
 from vulnerabilities import severity_systems
-from vulnerabilities.helpers import AffectedPackage as LegacyAffectedPackage
-from vulnerabilities.helpers import get_affected_packages_by_patched_package
-from vulnerabilities.helpers import get_item
-from vulnerabilities.helpers import nearest_patched_package
+from vulnerabilities import utils
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
@@ -56,6 +52,10 @@
 from vulnerabilities.package_managers import PypiVersionAPI
 from vulnerabilities.package_managers import RubyVersionAPI
 from vulnerabilities.package_managers import VersionAPI
+from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage
+from vulnerabilities.utils import get_affected_packages_by_patched_package
+from vulnerabilities.utils import get_item
+from vulnerabilities.utils import nearest_patched_package
 
 logger = logging.getLogger(__name__)
 
@@ -191,7 +191,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
             end_cursor_exp = ""
             while True:
                 graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)}
-                response = helpers.fetch_github_graphql_query(graphql_query)
+                response = utils.fetch_github_graphql_query(graphql_query)
 
                 page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo")
                 end_cursor = get_item(page_info, "endCursor")

vulnerabilities/importers/istio.py

@@ -30,12 +30,12 @@
 from univers.version_range import VersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import nearest_patched_package
-from vulnerabilities.helpers import split_markdown_front_matter
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.utils import nearest_patched_package
+from vulnerabilities.utils import split_markdown_front_matter
 
 is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match
 

vulnerabilities/importers/kaybee.py

@@ -22,11 +22,11 @@
 
 from packageurl import PackageURL
 
-from vulnerabilities.helpers import load_yaml
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
+from vulnerabilities.utils import load_yaml
+from vulnerabilities.utils import nearest_patched_package
 
 
 class KaybeeImporter(GitImporter):

vulnerabilities/importers/mozilla.py

@@ -8,12 +8,12 @@
 from packageurl import PackageURL
 
 from vulnerabilities import severity_systems
-from vulnerabilities.helpers import is_cve
-from vulnerabilities.helpers import split_markdown_front_matter
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.utils import is_cve
+from vulnerabilities.utils import split_markdown_front_matter
 
 REPOSITORY = "mozilla/foundation-security-advisories"
 MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$")

vulnerabilities/importers/nginx.py

@@ -32,7 +32,6 @@
 from univers.version_range import NginxVersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import evolve_purl
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
@@ -45,6 +44,7 @@
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.package_managers import PackageVersion
 from vulnerabilities.severity_systems import GENERIC
+from vulnerabilities.utils import evolve_purl
 
 logger = logging.getLogger(__name__)
 

vulnerabilities/importers/npm.py

@@ -33,12 +33,12 @@
 from univers.version_range import VersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import load_json
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import NpmVersionAPI
+from vulnerabilities.utils import load_json
+from vulnerabilities.utils import nearest_patched_package
 
 NPM_URL = "https://registry.npmjs.org{}"
 

vulnerabilities/importers/nvd.py

@@ -30,14 +30,14 @@
 from django.db.models.query import QuerySet
 
 from vulnerabilities import severity_systems
-from vulnerabilities.helpers import get_item
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
 from vulnerabilities.models import Advisory
+from vulnerabilities.utils import get_item
 
 
 class NVDImporter(Importer):

vulnerabilities/importers/postgresql.py

@@ -27,11 +27,11 @@
 from packageurl import PackageURL
 
 from vulnerabilities import severity_systems
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.utils import nearest_patched_package
 
 
 class PostgreSQLImporter(Importer):

vulnerabilities/importers/project_kb_msr2019.py

@@ -23,11 +23,11 @@
 import csv
 import urllib.request
 
-from vulnerabilities.helpers import create_etag
-from vulnerabilities.helpers import is_cve
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
+from vulnerabilities.utils import create_etag
+from vulnerabilities.utils import is_cve
 
 # Reading CSV file from  a url using `requests` is bit too complicated.
 # Use `urllib.request` for that purpose.

vulnerabilities/importers/pysec.py

@@ -36,13 +36,14 @@
 from univers.versions import SemverVersion
 from univers.versions import Version
 
-from vulnerabilities.helpers import dedupe
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
+from vulnerabilities.utils import build_description
+from vulnerabilities.utils import dedupe
 
 logger = logging.getLogger(__name__)
 
@@ -74,12 +75,7 @@ def parse_advisory_data(raw_data: dict) -> Optional[AdvisoryData]:
     raw_id = raw_data.get("id") or ""
     summary = raw_data.get("summary") or ""
     details = raw_data.get("details") or ""
-    summary_and_details = []
-    if summary:
-        summary_and_details.append(summary)
-    if details:
-        summary_and_details.append(details)
-    summary = ".".join(summary_and_details)
+    summary = build_description(summary=summary, description=details)
     aliases = raw_data.get("aliases") or []
     if raw_id:
         aliases.append(raw_id)
@@ -202,8 +198,8 @@ def get_affected_version_range(affected_pkg, raw_id):
             logger.error(
                 f"InvalidVersionRange affected_pkg_version_range Error - {raw_id !r} {e!r}"
             )
-    else:
-        logger.error(f"affected_pkg_version_range not found - {raw_id !r} ")
+    # else:
+    #     logger.error(f"affected_pkg_version_range not found - {raw_id !r} ")
 
 
 def get_fixed_version(fixed_range, raw_id) -> List[Version]:

vulnerabilities/importers/redhat.py

@@ -30,14 +30,14 @@
 from univers.version_range import RpmVersionRange
 
 from vulnerabilities import severity_systems
-from vulnerabilities.helpers import get_item
-from vulnerabilities.helpers import requests_with_5xx_retry
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.rpm_utils import rpm_to_purl
+from vulnerabilities.utils import get_item
+from vulnerabilities.utils import requests_with_5xx_retry
 
 logger = logging.getLogger(__name__)
 

vulnerabilities/importers/retiredotnet.py

@@ -27,10 +27,10 @@
 
 from packageurl import PackageURL
 
-from vulnerabilities.helpers import AffectedPackage
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
+from vulnerabilities.utils import AffectedPackage
 
 
 class RetireDotnetImporter(GitImporter):

vulnerabilities/importers/ruby.py

@@ -30,12 +30,12 @@
 from univers.version_range import VersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import load_yaml
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import RubyVersionAPI
+from vulnerabilities.utils import load_yaml
+from vulnerabilities.utils import nearest_patched_package
 
 
 class RubyImporter(GitImporter):

vulnerabilities/importers/rust.py

@@ -34,11 +34,11 @@
 from univers.version_range import VersionRange
 from univers.versions import SemverVersion
 
-from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import GitImporter
 from vulnerabilities.importer import Reference
 from vulnerabilities.package_managers import CratesVersionAPI
+from vulnerabilities.utils import nearest_patched_package
 
 
 class RustImporter(GitImporter):