feat: azure linux 3 support

.yardstick.yaml

@@ -1 +1 @@
-store-root: data/yardstick
+store-root: data/yardstick

config/grype-db-manager/include.d/validate.yaml

@@ -23,6 +23,7 @@ gates:
       # (default 0, meaning the test scan must have the same or fewer FNs than the OSS scan to pass the gate)
       max_new_false_negatives: 10
       max_year: 2021
+      candidate_tool_label: 'custom-db'
 
     # these are the set of images we will capture grype scans for using an existing published DB and a newly build DB.
     # The assumption is that they should perform similarly or the new DB should perform better. We do allow for the new
@@ -39,3 +40,14 @@ gates:
       - docker.io/debian:7@sha256:81e88820a7759038ffa61cff59dfcc12d3772c3a2e75b7cfe963c952da2ad264
       - registry.access.redhat.com/ubi8@sha256:68fecea0d255ee253acbf0c860eaebb7017ef5ef007c25bee9eeffd29ce85b29
       - docker.io/ubuntu:20.04@sha256:9d42d0e3e57bc067d10a75ee33bdd1a5298e95e5fc3c5d1fce98b455cb879249
+
+
+  - gate:
+      max_f1_regression: 0.15
+      max_unlabeled_percent: 25
+      max_new_false_negatives: 10
+      max_year: 2022 # important - Azure Linux 3 doesn't have enough CVEs going back to 2021
+      candidate_tool_label: 'custom-db'
+
+    images:
+      - docker.io/anchore/test_images:azurelinux3-63671fe@sha256:2d761ba36575ddd4e07d446f4f2a05448298c20e5bdcd3dedfbbc00f9865240d

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/adrg/xdg v0.5.0
 	github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
-	github.com/anchore/grype v0.80.2
+	github.com/anchore/grype v0.80.3-0.20240924175453-be83782134b3
 	github.com/anchore/syft v1.13.0
 	github.com/dustin/go-humanize v1.0.1
 	github.com/glebarez/sqlite v1.11.0
@@ -22,7 +22,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/iancoleman/strcase v0.3.0
 	github.com/jinzhu/copier v0.4.0
-	github.com/klauspost/compress v1.17.10
+	github.com/klauspost/compress v1.17.9
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/profile v1.7.0

go.sum

@@ -252,8 +252,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE+o2gozGEBoUMpX27lsku+xrMwlmBZJtbg=
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/grype v0.80.2 h1:/iQFxH5T92bueBnCn/wkr4MAYX7Jd3H+qQ0eib/gZ/Q=
-github.com/anchore/grype v0.80.2/go.mod h1:NwocNorrJLiaTyceEVkd5LsoawPKyrJ/V1fb4i/5/IQ=
+github.com/anchore/grype v0.80.3-0.20240924175453-be83782134b3 h1:LWo4f3/u2JuGWsU/Qp1nNCRgeoizdxwacA15emIVK9s=
+github.com/anchore/grype v0.80.3-0.20240924175453-be83782134b3/go.mod h1:NwocNorrJLiaTyceEVkd5LsoawPKyrJ/V1fb4i/5/IQ=
 github.com/anchore/packageurl-go v0.1.1-0.20240507183024-848e011fc24f h1:B/E9ixKNCasntpoch61NDaQyGPDXLEJlL+B9B/PbdbA=
 github.com/anchore/packageurl-go v0.1.1-0.20240507183024-848e011fc24f/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4=
 github.com/anchore/stereoscope v0.0.3 h1:JRPHySy8S6P+Ff3IDiQ29ap1i8/laUQxDk9K1eFh/2U=
@@ -675,8 +675,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
-github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
-github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=

manager/src/grype_db_manager/cli/config.py

@@ -10,7 +10,7 @@
 import yaml
 from dataclass_wizard import asdict, fromdict
 from yamlinclude import YamlIncludeConstructor
-from yardstick.validate import GateConfig
+from yardstick.cli.config import Validation
 
 from grype_db_manager import db, s3utils
 
@@ -50,7 +50,7 @@ class Grype:
 class ValidateDB:
     images: list[str] = field(default_factory=list)
     grype: Grype = field(default_factory=Grype)
-    gate: GateConfig = field(default_factory=GateConfig)
+    gate: Validation = field(default_factory=Validation)
 
     def __post_init__(self):
         # flatten elements in images (in case yaml anchors are used)

manager/src/grype_db_manager/cli/db.py

@@ -98,10 +98,11 @@ def show_db(cfg: config.Application, db_uuid: str) -> None:
     is_flag=True,
     help="do not ensure the minimum expected namespaces are present",
 )
-@click.option("--allow-empty-matches",
-              "allow_empty_matches",
-              is_flag=True,
-    help="set 'fail_on_empty_matches' to false when invoking yardstick validate"
+@click.option(
+    "--allow-empty-matches",
+    "allow_empty_matches",
+    is_flag=True,
+    help="set 'fail_on_empty_matches' to false when invoking yardstick validate",
 )
 @click.argument("db-uuid")
 @click.pass_obj
@@ -147,6 +148,8 @@ def validate_db(
         if allow_empty_matches:
             rs.gate.fail_on_empty_match_set = False
 
+        logging.info(f"writing config for result set result_set_{idx}")
+
         result_sets[f"result_set_{idx}"] = ycfg.ResultSet(
             description=f"generated result set for gate {idx}",
             validations=[rs.gate],

manager/tests/unit/cli/test_config.py

@@ -116,6 +116,7 @@ def test_load(test_dir_path):
         maxNewFalseNegatives: 10
         maxUnlabeledPercent: 50
         maxYear: 2021
+        name: default
         referenceToolLabel: reference
         requiredNamespaces: []
       grype:

pkg/process/v5/transformers/os/test-fixtures/azure-linux-3.json

@@ -0,0 +1,26 @@
+[
+  {
+    "Vulnerability": {
+      "Name": "CVE-2023-29403",
+      "NamespaceName": "mariner:3.0",
+      "Description": "CVE-2023-29403 affecting package golang for versions less than 1.20.7-1. A patched version of the package is available.",
+      "Severity": "High",
+      "Link": "https://nvd.nist.gov/vuln/detail/CVE-2023-29403",
+      "CVSS": [],
+      "FixedIn": [
+        {
+          "Name": "golang",
+          "NamespaceName": "mariner:3.0",
+          "VersionFormat": "rpm",
+          "Version": "0:1.20.7-1.azl3",
+          "Module": "",
+          "VendorAdvisory": {
+            "NoAdvisory": false,
+            "AdvisorySummary": []
+          }
+        }
+      ],
+      "Metadata": {}
+    }
+  }
+]

pkg/process/v5/transformers/os/test-fixtures/mariner-20.json

@@ -0,0 +1,26 @@
+[
+  {
+    "Vulnerability": {
+      "Name": "CVE-2021-37621",
+      "NamespaceName": "mariner:2.0",
+      "Description": "CVE-2021-37621 affecting package exiv2 for versions less than 0.27.5-1. An upgraded version of the package is available that resolves this issue.",
+      "Severity": "Medium",
+      "Link": "https://nvd.nist.gov/vuln/detail/CVE-2021-37621",
+      "CVSS": [],
+      "FixedIn": [
+        {
+          "Name": "exiv2",
+          "NamespaceName": "mariner:2.0",
+          "VersionFormat": "rpm",
+          "Version": "0:0.27.5-1.cm2",
+          "Module": "",
+          "VendorAdvisory": {
+            "NoAdvisory": false,
+            "AdvisorySummary": []
+          }
+        }
+      ],
+      "Metadata": {}
+    }
+  }
+]

pkg/process/v5/transformers/os/transform.go

@@ -30,16 +30,21 @@ func buildGrypeNamespace(group string) (namespace.Namespace, error) {
 	}
 
 	providerName := d.String()
+	distroName := d.String()
 
 	switch d {
 	case distro.OracleLinux:
 		providerName = "oracle"
 	case distro.AmazonLinux:
 		providerName = "amazon"
+	case distro.Mariner, distro.Azure:
+		providerName = "mariner"
+		if strings.HasPrefix(feedGroupComponents[1], "3") {
+			distroName = distro.Azure.String() // Mariner Linux 3 is known as "Azure Linux 3"
+		}
 	}
 
-	ns, err := namespace.FromString(fmt.Sprintf("%s:distro:%s:%s", providerName, d.String(), feedGroupComponents[1]))
-
+	ns, err := namespace.FromString(fmt.Sprintf("%s:distro:%s:%s", providerName, distroName, feedGroupComponents[1]))
 	if err != nil {
 		return nil, err
 	}

pkg/process/v5/transformers/os/transform_test.go

@@ -626,6 +626,84 @@ func TestParseVulnerabilitiesEntry(t *testing.T) {
 				Description:  "A flaw was found in PostgreSQL, where some PostgreSQL extensions did not use the search_path safely in their installation script. This flaw allows an attacker with sufficient privileges to trick an administrator into executing a specially crafted script during the extension's installation or update. The highest threat from this vulnerability is to confidentiality, integrity, as well as system availability.",
 			},
 		},
+		{
+			name:       "mariner linux 2.0",
+			numEntries: 1,
+			fixture:    "test-fixtures/mariner-20.json",
+			vulns: []grypeDB.Vulnerability{
+				{
+					ID:          "CVE-2021-37621",
+					PackageName: "exiv2",
+					Namespace:   "mariner:distro:mariner:2.0",
+					PackageQualifiers: []qualifier.Qualifier{
+						rpmmodularity.Qualifier{
+							Kind: "rpm-modularity",
+						},
+					},
+					RelatedVulnerabilities: []grypeDB.VulnerabilityReference{
+						{
+							ID:        "CVE-2021-37621",
+							Namespace: "nvd:cpe",
+						},
+					},
+					VersionConstraint: "< 0:0.27.5-1.cm2",
+					VersionFormat:     "rpm",
+					Fix: grypeDB.Fix{
+						Versions: []string{"0:0.27.5-1.cm2"},
+						State:    grypeDB.FixedState,
+					},
+					Advisories: nil,
+				},
+			},
+			metadata: grypeDB.VulnerabilityMetadata{
+				ID:           "CVE-2021-37621",
+				Namespace:    "mariner:distro:mariner:2.0",
+				DataSource:   "https://nvd.nist.gov/vuln/detail/CVE-2021-37621",
+				RecordSource: "vulnerabilities:mariner:2.0",
+				Severity:     "Medium",
+				URLs:         []string{"https://nvd.nist.gov/vuln/detail/CVE-2021-37621"},
+				Description:  "CVE-2021-37621 affecting package exiv2 for versions less than 0.27.5-1. An upgraded version of the package is available that resolves this issue.",
+				Cvss:         nil,
+			},
+		},
+		{
+			name:       "azure linux 3",
+			numEntries: 1,
+			fixture:    "test-fixtures/azure-linux-3.json",
+			vulns: []grypeDB.Vulnerability{
+				{
+					ID:          "CVE-2023-29403",
+					PackageName: "golang",
+					Namespace:   "mariner:distro:azurelinux:3.0",
+					PackageQualifiers: []qualifier.Qualifier{
+						rpmmodularity.Qualifier{
+							Kind: "rpm-modularity",
+						},
+					},
+					RelatedVulnerabilities: []grypeDB.VulnerabilityReference{
+						{
+							ID:        "CVE-2023-29403",
+							Namespace: "nvd:cpe",
+						},
+					},
+					VersionConstraint: "< 0:1.20.7-1.azl3",
+					VersionFormat:     "rpm",
+					Fix: grypeDB.Fix{
+						Versions: []string{"0:1.20.7-1.azl3"},
+						State:    grypeDB.FixedState,
+					},
+				},
+			},
+			metadata: grypeDB.VulnerabilityMetadata{
+				ID:           "CVE-2023-29403",
+				Namespace:    "mariner:distro:azurelinux:3.0",
+				DataSource:   "https://nvd.nist.gov/vuln/detail/CVE-2023-29403",
+				RecordSource: "vulnerabilities:mariner:3.0",
+				Severity:     "High",
+				URLs:         []string{"https://nvd.nist.gov/vuln/detail/CVE-2023-29403"},
+				Description:  "CVE-2023-29403 affecting package golang for versions less than 1.20.7-1. A patched version of the package is available.",
+			},
+		},
 		{
 			name:       "mariner entry with version range",
 			numEntries: 1,

pyproject.toml

@@ -38,6 +38,7 @@ mergedeep = "^1.3.4"
 pyyaml = ">=5.0.1, <7"
 yardstick = {git = "https://github.com/anchore/yardstick", rev = "v0.10.0"}
 # yardstick = {path = "../yardstick", develop = true}
+# vunnel = {path = "../vunnel", develop = true}
 colr = "^0.9.1"
 pyyaml-include = "^1.3.1"
 python-magic = "^0.4.27"

test/db/acceptance.sh

@@ -27,7 +27,7 @@ fi
 title "Validating DB"
 
 ALLOW_EMPTY=""
-if [[ "$SCHEMA_VERSION" == "1" || "$SCHEMA_VERSION" == "2" || "$SCHEMA_VERSION" == "3" ]]; then
+if [[ "$SCHEMA_VERSION" == "1" || "$SCHEMA_VERSION" == "2" || "$SCHEMA_VERSION" == "3" || "$SCHEMA_VERSION" == "4" ]]; then
   ALLOW_EMPTY="--allow-empty-matches"
 fi