Merge pull request aboutcode-org#525 from Hritik14/importer-refactor

Separate import and improve operations

Signed-off-by: Philippe Ombredanne <[email protected]>

.gitignore

@@ -125,3 +125,12 @@ Pipfile
 
 # VSCode
 .vscode
+
+# Various junk and temp files
+.DS_Store
+*~
+.*.sw[po]
+.build
+.ve
+*.bak
+/.cache/

pytest.ini

@@ -1,4 +1,34 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = vulnerablecode.settings
 markers =
-    webtest
+    webtest
+addopts =
+	--doctest-modules
+# Ignore the following doctests until these files are migrated to
+# import-improve structure
+	--ignore=vulnerabilities/importers/alpine_linux.py
+	--ignore=vulnerabilities/importers/apache_httpd.py
+	--ignore=vulnerabilities/importers/apache_kafka.py
+	--ignore=vulnerabilities/importers/apache_tomcat.py
+	--ignore=vulnerabilities/importers/archlinux.py
+	--ignore=vulnerabilities/importers/debian.py
+	--ignore=vulnerabilities/importers/elixir_security.py
+	--ignore=vulnerabilities/importers/gentoo.py
+	--ignore=vulnerabilities/importers/github.py
+	--ignore=vulnerabilities/importers/istio.py
+	--ignore=vulnerabilities/importers/kaybee.py
+	--ignore=vulnerabilities/importers/npm.py
+	--ignore=vulnerabilities/importers/nvd.py
+	--ignore=vulnerabilities/importers/openssl.py
+	--ignore=vulnerabilities/importers/postgresql.py
+	--ignore=vulnerabilities/importers/project_kb_msr2019.py
+	--ignore=vulnerabilities/importers/redhat.py
+	--ignore=vulnerabilities/importers/retiredotnet.py
+	--ignore=vulnerabilities/importers/ruby.py
+	--ignore=vulnerabilities/importers/rust.py
+	--ignore=vulnerabilities/importers/safety_db.py
+	--ignore=vulnerabilities/importers/suse_backports.py
+	--ignore=vulnerabilities/importers/suse_scores.py
+	--ignore=vulnerabilities/importers/ubuntu_usn.py
+	--ignore=vulnerabilities/management/commands/create_cpe_to_purl_map.py
+	--ignore=vulnerabilities/lib_oval.py

requirements.txt

@@ -8,12 +8,12 @@ django-widget-tweaks>=1.4.8
 packageurl-python>=0.9.4
 binaryornot>=0.4.4
 GitPython>=3.1.17
-univers>=21.4.16.6
+univers>=30.0.0
 saneyaml>=0.5.2
 beautifulsoup4>=4.9.3
 python-dateutil>=2.8.1
 toml>=0.10.2
-lxml>=4.6.3
+lxml>=4.6.4
 gunicorn>=20.1.0
 django-environ==0.4.5
 defusedxml==0.7.1

vulnerabilities/data_inference.py

@@ -0,0 +1,98 @@
+import dataclasses
+import logging
+from typing import List
+from typing import Optional
+from uuid import uuid4
+
+from packageurl import PackageURL
+from django.db.models.query import QuerySet
+
+from vulnerabilities.data_source import Reference
+from vulnerabilities.data_source import AdvisoryData
+
+logger = logging.getLogger(__name__)
+
+MAX_CONFIDENCE = 100
+
+
+@dataclasses.dataclass(order=True)
+class Inference:
+    """
+    This data class expresses the contract between data improvers and the improve runner.
+
+    Only inferences with highest confidence for one vulnerability <-> package
+    relationship is to be inserted into the database
+    """
+
+    vulnerability_id: str = None
+    aliases: List[str] = dataclasses.field(default_factory=list)
+    confidence: int = MAX_CONFIDENCE
+    summary: Optional[str] = None
+    affected_purls: List[PackageURL] = dataclasses.field(default_factory=list)
+    fixed_purl: PackageURL = dataclasses.field(default_factory=list)
+    references: List[Reference] = dataclasses.field(default_factory=list)
+
+    def __post_init__(self):
+        if self.confidence > MAX_CONFIDENCE or self.confidence < 0:
+            raise ValueError
+
+        assert (
+            self.vulnerability_id
+            or self.aliases
+            or self.summary
+            or self.affected_purls
+            or self.fixed_purl
+            or self.references
+        )
+
+        versionless_purls = []
+        for purl in self.affected_purls + [self.fixed_purl]:
+            if not purl.version:
+                versionless_purls.append(purl)
+
+        assert (
+            not versionless_purls
+        ), f"Version-less purls are not supported in an Inference: {versionless_purls}"
+
+    @classmethod
+    def from_advisory_data(cls, advisory_data, confidence, affected_purls, fixed_purl):
+        """
+        Return an Inference object while keeping the same values as of advisory_data
+        for vulnerability_id, summary and references
+        """
+        return cls(
+            aliases=advisory_data.aliases,
+            confidence=confidence,
+            summary=advisory_data.summary,
+            affected_purls=affected_purls,
+            fixed_purl=fixed_purl,
+            references=advisory_data.references,
+        )
+
+
+class Improver:
+    """
+    Improvers are responsible to improve the already imported data by a datasource.
+    Inferences regarding the data could be generated based on multiple factors.
+    """
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        """
+        Return QuerySet for the advisories this improver is interested in
+        """
+        raise NotImplementedError
+
+    def get_inferences(self, advisory_data: AdvisoryData) -> List[Inference]:
+        """
+        Generate and return Inferences for the given advisory data
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def qualified_name(cls):
+        """
+        Fully qualified name prefixed with the module name of the improver
+        used in logging.
+        """
+        return f"{cls.__module__}.{cls.__qualname__}"