ROX-16421: support OCI image index media type

e2etests/testcase_test.go

@@ -3955,4 +3955,21 @@ All OpenShift Container Platform 4.10 users are advised to upgrade to these upda
 			},
 		},
 	},
+	{
+		// OCI v1 Image Index manifest type.
+		image:                   "gcr.io/distroless/static-debian11@sha256:a01d47d4036cae5a67a9619e3d06fa14a6811a2247b4da72b4233ece4efebd57",
+		registry:                "https://gcr.io",
+		source:                  "NVD",
+		onlyCheckSpecifiedVulns: true,
+		namespace:               "debian:11",
+		expectedFeatures: []apiV1.Feature{
+			{
+				Name:          "tzdata",
+				NamespaceName: "debian:11",
+				VersionFormat: "dpkg",
+				Version:       "2021a-1+deb11u9",
+				AddedBy:       "sha256:fff4e558ad3a0d44f9ec0bf32771df4a102c6d93fce253dd3eb6657865b78a49",
+			},
+		},
+	},
 }

go.mod

@@ -30,7 +30,6 @@ require (
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.1.0-rc2
 	github.com/pborman/uuid v1.2.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
@@ -56,6 +55,7 @@ require (
 	github.com/containers/storage v1.45.3 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 )
 
@@ -152,7 +152,7 @@ replace (
 	github.com/facebookincubator/nvdtools => github.com/stackrox/nvdtools v0.0.0-20230214202552-24e684e3ad9d
 	github.com/fullsailor/pkcs7 => github.com/stackrox/pkcs7 v0.0.0-20220914154527-cfdb0aa47179
 	github.com/gogo/protobuf => github.com/connorgorman/protobuf v1.2.2-0.20210115205927-b892c1b298f7
-	github.com/heroku/docker-registry-client => github.com/stackrox/docker-registry-client v0.0.0-20220204234128-07f109db0819
+	github.com/heroku/docker-registry-client => github.com/stackrox/docker-registry-client v0.0.0-20230411213734-d75b95d65d28
 
 	github.com/operator-framework/helm-operator-plugins => github.com/stackrox/helm-operator v0.0.10-0.20220919093109-89f9785764c6
 	github.com/stackrox/rox => github.com/stackrox/stackrox v0.0.0-20230301153935-a7fafd5bc0bd

go.sum

@@ -221,7 +221,6 @@ github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mz
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
-github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
@@ -587,7 +586,6 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lyft/protoc-gen-star v0.5.1/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
-github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -679,7 +677,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
-github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
 github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
@@ -793,7 +790,6 @@ github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -830,8 +826,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
-github.com/stackrox/docker-registry-client v0.0.0-20220204234128-07f109db0819 h1:1jcyovr8lC0vBax5TFNzpXOChl/sJr/IoLVUFkYEMpM=
-github.com/stackrox/docker-registry-client v0.0.0-20220204234128-07f109db0819/go.mod h1:tEU0CD7y2mq9HAWFZY48THyKPFy6oMv19UT5bnTvrRo=
+github.com/stackrox/docker-registry-client v0.0.0-20230411213734-d75b95d65d28 h1:o1EYbTl0p/upuWUfFPtthLZDNstw7+uVZSnB9GLKbU4=
+github.com/stackrox/docker-registry-client v0.0.0-20230411213734-d75b95d65d28/go.mod h1:L3hj9CMuaAE4Yh2nhL6wANtbJ8/Y7hbi+8dHA6yDRf4=
 github.com/stackrox/dotnet-scraper v0.0.0-20201023051640-72ef543323dd h1:vEjp7Q66zd4W72//Uk3uyVN50Mh/nFLbN9pb7CVK7mE=
 github.com/stackrox/dotnet-scraper v0.0.0-20201023051640-72ef543323dd/go.mod h1:HILeV3i/EyJz844GcrC3+oU7oZONhjfujaIYBMJ/bZE=
 github.com/stackrox/istio-cves v0.0.0-20221007013142-0bde9b541ec8 h1:rUIvoAHokPcd92aJT2gJwVeyE8tMuaqS5l5s3cEgXFY=

pkg/clairify/types/types.go

@@ -9,7 +9,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/docker/distribution/manifest/ocischema"
+	"github.com/docker/distribution/manifest/manifestlist"
+	ociV1 "github.com/docker/distribution/manifest/ocischema"
 	manifestV1 "github.com/docker/distribution/manifest/schema1"
 	manifestV2 "github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/reference"
@@ -49,10 +50,11 @@ type ClairClient interface {
 // Registry is the Docker Registry Client interface.
 type Registry interface {
 	Manifest(repository, reference string) (*manifestV1.SignedManifest, error)
-	ManifestOCI(repository, reference string) (*ocischema.DeserializedManifest, error)
 	SignedManifest(repository, reference string) (*manifestV1.SignedManifest, error)
+	ManifestList(repository, reference string) (*manifestlist.DeserializedManifestList, error)
 	ManifestV2(repository, reference string) (*manifestV2.DeserializedManifest, error)
-	ManifestList(repository, reference string) (*registry.ManifestList, error)
+	ImageIndex(repository, reference string) (*manifestlist.DeserializedManifestList, error)
+	ManifestOCI(repository, reference string) (*ociV1.DeserializedManifest, error)
 
 	ManifestDigest(repository, reference string) (digest.Digest, string, error)
 	DownloadLayer(repository string, digest digest.Digest) (io.ReadCloser, error)

pkg/scan/scan.go

@@ -5,13 +5,13 @@ import (
 	"fmt"
 	"io"
 
-	v5Manifest "github.com/containers/image/v5/manifest"
+	imageManifest "github.com/containers/image/v5/manifest"
 	"github.com/docker/distribution"
-	"github.com/docker/distribution/manifest/schema1"
-	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/manifest/manifestlist"
+	manifestV1 "github.com/docker/distribution/manifest/schema1"
+	manifestV2 "github.com/docker/distribution/manifest/schema2"
 	"github.com/heroku/docker-registry-client/registry"
 	"github.com/opencontainers/go-digest"
-	ociSpec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	clair "github.com/stackrox/scanner"
@@ -24,6 +24,15 @@ const (
 	emptyLayer = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
 )
 
+var manifestTypes = []string{
+	manifestlist.MediaTypeManifestList,
+	manifestV2.MediaTypeManifest,
+	registry.MediaTypeImageIndex,
+	registry.MediaTypeImageManifest,
+	manifestV1.MediaTypeSignedManifest,
+	manifestV1.MediaTypeManifest,
+}
+
 // analyzeLayers processes all of the layers and returns the lineage for the last layer so that we can uniquely identify it
 func analyzeLayers(storage database.Datastore, registry types.Registry, image *types.Image, layers []string, uncertifiedRHEL bool) (string, error) {
 	var prevLayer string
@@ -104,7 +113,7 @@ func isEmptyLayer(layer string) bool {
 	return layer == emptyLayer
 }
 
-func parseV1Layers(manifest *schema1.SignedManifest) []string {
+func parseV1Layers(manifest *manifestV1.SignedManifest) []string {
 	var layers []string
 	// FSLayers has the most recent layer first, append them so that parent layers are first in the slice
 	for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
@@ -137,7 +146,7 @@ func renderDigest(manifest payloadGetter) (digest.Digest, error) {
 	if err != nil {
 		return "", err
 	}
-	dig, err := v5Manifest.Digest(canonical)
+	dig, err := imageManifest.Digest(canonical)
 	if err != nil {
 		return dig, err
 	}
@@ -146,7 +155,7 @@ func renderDigest(manifest payloadGetter) (digest.Digest, error) {
 
 func handleManifest(reg types.Registry, manifestType, remote, ref string) (digest.Digest, []string, error) {
 	switch manifestType {
-	case schema1.MediaTypeManifest:
+	case manifestV1.MediaTypeManifest:
 		manifest, err := reg.Manifest(remote, ref)
 		if err != nil {
 			return "", nil, err
@@ -157,7 +166,7 @@ func handleManifest(reg types.Registry, manifestType, remote, ref string) (diges
 		}
 		layers := parseV1Layers(manifest)
 		return dig, layers, nil
-	case schema1.MediaTypeSignedManifest:
+	case manifestV1.MediaTypeSignedManifest:
 		manifest, err := reg.SignedManifest(remote, ref)
 		if err != nil {
 			return "", nil, err
@@ -168,7 +177,7 @@ func handleManifest(reg types.Registry, manifestType, remote, ref string) (diges
 		}
 		layers := parseV1Layers(manifest)
 		return dig, layers, nil
-	case schema2.MediaTypeManifest:
+	case manifestV2.MediaTypeManifest:
 		manifest, err := reg.ManifestV2(remote, ref)
 		if err != nil {
 			return "", nil, err
@@ -179,7 +188,7 @@ func handleManifest(reg types.Registry, manifestType, remote, ref string) (diges
 		}
 		layers := parseLayers(manifest.Layers)
 		return dig, layers, nil
-	case ociSpec.MediaTypeImageManifest:
+	case registry.MediaTypeImageManifest:
 		manifest, err := reg.ManifestOCI(remote, ref)
 		if err != nil {
 			return "", nil, err
@@ -189,25 +198,30 @@ func handleManifest(reg types.Registry, manifestType, remote, ref string) (diges
 			return "", nil, err
 		}
 		return dig, parseLayers(manifest.Layers), nil
-	case registry.MediaTypeManifestList:
+	case manifestlist.MediaTypeManifestList:
 		manifestList, err := reg.ManifestList(remote, ref)
 		if err != nil {
 			return "", nil, err
 		}
 		for _, manifest := range manifestList.Manifests {
+			// TODO(ROX-13284): Support multi-arch images.
+			if manifest.Platform.OS == "linux" && manifest.Platform.Architecture == "amd64" {
+				return handleManifest(reg, manifest.MediaType, remote, manifest.Digest.String())
+			}
+		}
+		return "", nil, errors.New("No corresponding manifest found from Docker manifest list object")
+	case registry.MediaTypeImageIndex:
+		imageIndex, err := reg.ImageIndex(remote, ref)
+		if err != nil {
+			return "", nil, err
+		}
+		for _, manifest := range imageIndex.Manifests {
+			// TODO(ROX-13284): Support multi-arch images.
 			if manifest.Platform.OS == "linux" && manifest.Platform.Architecture == "amd64" {
-				manifest, err := reg.ManifestV2(remote, manifest.Digest)
-				if err != nil {
-					return "", nil, err
-				}
-				dig, err := renderDigest(manifest)
-				if err != nil {
-					return "", nil, err
-				}
-				return dig, parseLayers(manifest.Layers), nil
+				return handleManifest(reg, manifest.MediaType, remote, manifest.Digest.String())
 			}
 		}
-		return "", nil, errors.New("No corresponding manifest found from manifest list object")
+		return "", nil, errors.New("No corresponding manifest found from OCI image index object")
 	default:
 		return "", nil, fmt.Errorf("Could not parse manifest type %q", manifestType)
 	}
@@ -220,23 +234,23 @@ func FetchLayers(reg types.Registry, image *types.Image) (string, []string, erro
 		ref = image.SHA
 	}
 
-	digest, manifestType, err := reg.ManifestDigest(image.Remote, ref)
+	d, manifestType, err := reg.ManifestDigest(image.Remote, ref)
 	if err != nil {
-		// Some registries have no implemented the docker registry API correctly so the fall back here is to just try all the manifest types
-		manifestTypes := []string{registry.MediaTypeManifestList, schema2.MediaTypeManifest, ociSpec.MediaTypeImageManifest, schema1.MediaTypeSignedManifest, schema1.MediaTypeManifest}
+		// Some registries have not implemented the docker registry API correctly,
+		// so just try all the manifest types.
 		for _, m := range manifestTypes {
-			digest, layers, manifestErr := handleManifest(reg, m, image.Remote, ref)
+			d, layers, manifestErr := handleManifest(reg, m, image.Remote, ref)
 			if manifestErr != nil {
 				continue
 			}
-			return digest.String(), layers, nil
+			return d.String(), layers, nil
 		}
 		return "", nil, err
 	}
 	// No digest is needed from handleManifest because the manifest digest gives us the definitive digest
-	_, layers, err := handleManifest(reg, manifestType, image.Remote, digest.String())
+	_, layers, err := handleManifest(reg, manifestType, image.Remote, d.String())
 	if err != nil {
 		return "", nil, err
 	}
-	return digest.String(), layers, nil
+	return d.String(), layers, nil
 }