Add new content ID function

vulnerabilities/importer.py

@@ -9,6 +9,7 @@
 
 import dataclasses
 import datetime
+import functools
 import logging
 import os
 import shutil
@@ -46,7 +47,8 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclasses.dataclass(order=True)
+@dataclasses.dataclass(eq=True)
+@functools.total_ordering
 class VulnerabilitySeverity:
     # FIXME: this should be named scoring_system, like in the model
     system: ScoringSystem
@@ -55,15 +57,23 @@ class VulnerabilitySeverity:
     published_at: Optional[datetime.datetime] = None
 
     def to_dict(self):
-        published_at_dict = (
-            {"published_at": self.published_at.isoformat()} if self.published_at else {}
-        )
-        return {
+        data = {
             "system": self.system.identifier,
             "value": self.value,
             "scoring_elements": self.scoring_elements,
-            **published_at_dict,
         }
+        if self.published_at:
+            data["published_at"] = self.published_at.isoformat()
+        return data
+
+    def __lt__(self, other):
+        if not isinstance(other, VulnerabilitySeverity):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (self.system.identifier, self.value, self.scoring_elements, self.published_at)
 
     @classmethod
     def from_dict(cls, severity: dict):
@@ -79,7 +89,8 @@ def from_dict(cls, severity: dict):
         )
 
 
-@dataclasses.dataclass(order=True)
+@dataclasses.dataclass(eq=True)
+@functools.total_ordering
 class Reference:
     reference_id: str = ""
     reference_type: str = ""
@@ -90,21 +101,22 @@ def __post_init__(self):
         if not self.url:
             raise TypeError("Reference must have a url")
 
-    def normalized(self):
-        severities = sorted(self.severities)
-        return Reference(
-            reference_id=self.reference_id,
-            url=self.url,
-            severities=severities,
-            reference_type=self.reference_type,
-        )
+    def __lt__(self, other):
+        if not isinstance(other, Reference):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (self.reference_id, self.reference_type, self.url, tuple(self.severities))
 
     def to_dict(self):
+        """Return a normalized dictionary representation"""
         return {
             "reference_id": self.reference_id,
             "reference_type": self.reference_type,
             "url": self.url,
-            "severities": [severity.to_dict() for severity in self.severities],
+            "severities": [severity.to_dict() for severity in sorted(self.severities)],
         }
 
     @classmethod
@@ -140,7 +152,8 @@ class NoAffectedPackages(Exception):
     """
 
 
-@dataclasses.dataclass(order=True, frozen=True)
+@functools.total_ordering
+@dataclasses.dataclass(eq=True)
 class AffectedPackage:
     """
     Relate a Package URL with a range of affected versions and a fixed version.
@@ -170,6 +183,19 @@ def get_fixed_purl(self):
             raise ValueError(f"Affected Package {self.package!r} does not have a fixed version")
         return update_purl_version(purl=self.package, version=str(self.fixed_version))
 
+    def __lt__(self, other):
+        if not isinstance(other, AffectedPackage):
+            return NotImplemented
+        return self._cmp_key() < other._cmp_key()
+
+    # TODO: Add cache
+    def _cmp_key(self):
+        return (
+            str(self.package),
+            str(self.affected_version_range or ""),
+            str(self.fixed_version or ""),
+        )
+
     @classmethod
     def merge(
         cls, affected_packages: Iterable

vulnerabilities/migrations/0089_alter_advisory_unique_content_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.16 on 2025-02-12 13:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0088_fix_alpine_purl_type"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="advisory",
+            name="unique_content_id",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="A 64 character unique identifier for the content of the advisory since we use sha256 as hex",
+                max_length=64,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -53,6 +53,7 @@
 from vulnerabilities import utils
 from vulnerabilities.severity_systems import EPSS
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
+from vulnerabilities.utils import compute_content_id
 from vulnerabilities.utils import normalize_purl
 from vulnerabilities.utils import purl_to_dict
 from vulnerablecode import __version__ as VULNERABLECODE_VERSION
@@ -1315,8 +1316,10 @@ class Advisory(models.Model):
     """
 
     unique_content_id = models.CharField(
-        max_length=32,
+        max_length=64,
+        db_index=True,
         blank=True,
+        help_text="A 64 character unique identifier for the content of the advisory since we use sha256 as hex",
     )
     aliases = models.JSONField(blank=True, default=list, help_text="A list of alias strings")
     summary = models.TextField(
@@ -1357,16 +1360,8 @@ class Meta:
         ordering = ["aliases", "date_published", "unique_content_id"]
 
     def save(self, *args, **kwargs):
-        checksum = hashlib.md5()
-        for field in (
-            self.summary,
-            self.affected_packages,
-            self.references,
-            self.weaknesses,
-        ):
-            value = json.dumps(field, separators=(",", ":")).encode("utf-8")
-            checksum.update(value)
-        self.unique_content_id = checksum.hexdigest()
+        advisory_data = self.to_advisory_data()
+        self.unique_content_id = compute_content_id(advisory_data, include_metadata=False)
         super().save(*args, **kwargs)
 
     def to_advisory_data(self) -> "AdvisoryData":

vulnerabilities/pipelines/remove_duplicate_advisories.py

@@ -0,0 +1,83 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+from itertools import groupby
+
+from aboutcode.pipeline import LoopProgress
+from django.db.models import Count
+from django.db.models import Q
+
+from vulnerabilities.models import Advisory
+from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.utils import compute_content_id
+
+
+class RemoveDuplicateAdvisoriesPipeline(VulnerableCodePipeline):
+    """Pipeline to remove duplicate advisories based on their content."""
+
+    pipeline_id = "remove_duplicate_advisories"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.recompute_content_ids,
+            cls.remove_duplicates,
+        )
+
+    def remove_duplicates(self):
+        """
+        Find advisories with the same content and keep only the latest one.
+        """
+
+        duplicated_advisories = groupby(
+            Advisory.objects.order_by("unique_content_id").all().paginated(),
+            key=lambda x: x.unique_content_id,
+        )
+        progress = LoopProgress(total_iterations=Advisory.objects.count(), logger=self.log)
+        for _content_id, advisories in progress.iter(duplicated_advisories):
+            advisories = list(advisories)
+            self.log(
+                f"Removing duplicates for content ID {_content_id} {len(advisories)}",
+                level=logging.INFO,
+            )
+            oldest = min(advisories, key=lambda x: x.date_imported)
+            try:
+                advisory_ids = []
+                for adv in advisories:
+                    if adv.id != oldest.id:
+                        advisory_ids.append(adv.id)
+                Advisory.objects.filter(id__in=advisory_ids).delete()
+            except Exception as e:
+                self.log(f"Error deleting advisories: {e}", level=logging.ERROR)
+
+            self.log(
+                f"Kept advisory {oldest.id} and removed "
+                f"{len(list(advisories)) - 1} duplicates for content ID {_content_id}",
+                level=logging.INFO,
+            )
+
+    def recompute_content_ids(self):
+        """
+        Recompute content IDs for all advisories.
+        """
+
+        advisories = []
+
+        progress = LoopProgress(
+            total_iterations=Advisory.objects.count(),
+            progress_step=1,
+            logger=self.log,
+        )
+
+        for advisory in progress.iter(Advisory.objects.all().paginated()):
+            advisory.unique_content_id = compute_content_id(advisory)
+            advisories.append(advisory)
+
+        Advisory.objects.bulk_update(advisories, ["unique_content_id"], batch_size=1000)

vulnerabilities/severity_systems.py

@@ -42,6 +42,9 @@ def compute(self, scoring_elements: str) -> str:
     def get(self, scoring_elements: str):
         return NotImplementedError
 
+    def __str__(self):
+        return f"{self.identifier}"
+
 
 @dataclasses.dataclass(order=True)
 class Cvssv2ScoringSystem(ScoringSystem):

vulnerabilities/tests/test_add_cvsssv31.py

@@ -29,6 +29,8 @@ def setUp(self):
                         }
                     ],
                     "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-1234",
+                    "reference_id": "CVE-2024-1234",
+                    "reference_type": "cve",
                 }
             ],
             date_collected="2024-09-27T19:38:00Z",