Oras go integration (ratify-project#50)

* Integrate oras-go library

* Fixed issue with retrieving manifests

* Add workaround for sbom verifier

This adds the ability to use built-in referrer store plugins from
the sbom verifier plugin

.gitignore

@@ -1,2 +1,3 @@
 bin
-dist/
+dist/
+local_oras_cache/

config/config.json

@@ -4,7 +4,8 @@
         "plugins": [
             {
                 "name": "oras",
-                "useHttp": true
+                "useHttp": true,
+                "localCachePath": "./local_oras_cache"
             }
         ]
     },
@@ -20,15 +21,15 @@
         "plugins": [
             {
                 "name":"notaryv2",
-                "artifactTypes" : "org.cncf.notary.v2",
+                "artifactTypes" : "application/vnd.cncf.notary.v2.signature",
                 "verificationCerts": [
                     "<cert folder>"
                   ]
             },
             {
                 "name":"sbom",
                 "artifactTypes":"org.example.sbom.v0",
-                "nestedReferences": "org.cncf.notary.v2"
+                "nestedReferences": "application/vnd.cncf.notary.v2.signature"
             }
 
         ]

go.mod

@@ -3,17 +3,18 @@ module github.com/deislabs/hora
 go 1.16
 
 require (
-	github.com/docker/cli v20.10.7+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/google/go-containerregistry v0.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/notaryproject/notation-go-lib v1.0.0-alpha-1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/oras-project/artifacts-spec v1.0.0-draft.1
 	github.com/pkg/errors v0.9.1
 	github.com/sigstore/cosign v1.1.0
 	github.com/sigstore/sigstore v0.0.0-20210729211320-56a91f560f44
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1
 	github.com/xlab/treeprint v1.1.0
+	oras.land/oras-go v0.5.1-0.20211103031936-61d91f9abd65
 )

pkg/ocispecs/referencedescriptor.go

@@ -4,7 +4,7 @@ import (
 	oci "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-const MediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json"
+const MediaTypeArtifactManifest = "application/vnd.cncf.oras.artifact.manifest.v1+json"
 
 type ReferenceDescriptor struct {
 	oci.Descriptor

pkg/referrerstore/oras/cosign.go

@@ -3,46 +3,54 @@ package oras
 import (
 	"github.com/deislabs/hora/pkg/common"
 	"github.com/deislabs/hora/pkg/ocispecs"
-	"github.com/deislabs/hora/pkg/referrerstore/oras/registry"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
+	"github.com/opencontainers/go-digest"
 	oci "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sigstore/cosign/pkg/cosign"
+	"net/http"
+	"strings"
 )
 
 const CosignArtifactType = "org.sigstore.cosign.v1"
 
-func getCosignReferences(client *registry.Client, subjectReference common.Reference) (*[]ocispecs.ReferenceDescriptor, error) {
+func getCosignReferences(subjectReference common.Reference) (*[]ocispecs.ReferenceDescriptor, error) {
 	var references []ocispecs.ReferenceDescriptor
+	if strings.Split(subjectReference.Original, "@")[1] == "" {
+		return &references, nil
+	}
 	ref, err := name.ParseReference(subjectReference.Original)
 	if err != nil {
-		return nil, err
+		return &references, err
 	}
 	hash := v1.Hash{
 		Algorithm: subjectReference.Digest.Algorithm().String(),
 		Hex:       subjectReference.Digest.Hex(),
 	}
 	signatureTag := cosign.AttachedImageTag(ref.Context(), hash, cosign.SignatureTagSuffix)
-	tagRef := common.Reference{
-		Path: subjectReference.Path,
-		Tag:  signatureTag.TagStr(),
-	}
-	desc, err := client.GetManifestMetadata(tagRef)
 
-	if err != nil && err != registry.ManifestNotFound {
-		return nil, err
+	desc, err := remote.Get(signatureTag)
+	if terr, ok := err.(*transport.Error); ok && terr.StatusCode == http.StatusNotFound {
+		return &references, nil
 	}
-
-	if err == nil {
-		references = append(references, ocispecs.ReferenceDescriptor{
-			ArtifactType: CosignArtifactType,
-			Descriptor: oci.Descriptor{
-				MediaType: desc.MediaType,
-				Digest:    desc.Digest,
-				Size:      desc.Size,
-			},
-		})
+	if err != nil {
+		return &references, err
 	}
+	descDig, err := digest.Parse(desc.Digest.String())
+	if err != nil {
+		return &references, err
+	}
+
+	references = append(references, ocispecs.ReferenceDescriptor{
+		ArtifactType: CosignArtifactType,
+		Descriptor: oci.Descriptor{
+			MediaType: string(desc.MediaType),
+			Digest:    descDig,
+			Size:      desc.Size,
+		},
+	})
 
 	return &references, nil
 }

pkg/referrerstore/oras/oras.go

@@ -4,42 +4,42 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"regexp"
-	"strings"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	oci "github.com/opencontainers/image-spec/specs-go/v1"
+	"oras.land/oras-go/pkg/content"
+	"oras.land/oras-go/pkg/oras"
 
 	"github.com/deislabs/hora/pkg/common"
 	"github.com/deislabs/hora/pkg/ocispecs"
 	"github.com/deislabs/hora/pkg/referrerstore"
 	"github.com/deislabs/hora/pkg/referrerstore/config"
 	"github.com/deislabs/hora/pkg/referrerstore/factory"
-	"github.com/deislabs/hora/pkg/referrerstore/oras/registry"
 	"github.com/opencontainers/go-digest"
+	artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
 )
 
 const (
-	storeName = "oras"
+	storeName             = "oras"
+	defaultLocalCachePath = "~/.hora/local_oras_cache"
 )
 
 type OrasStoreConf struct {
-	Name          string `json:"name"`
-	UseHttp       bool   `json:"useHttp,omitempty"`
-	CosignEnabled bool   `json:"cosign-enabled,omitempty"`
-	AuthProvider  string `json:"auth-provider,omitempty"`
+	Name           string `json:"name"`
+	UseHttp        bool   `json:"useHttp,omitempty"`
+	CosignEnabled  bool   `json:"cosign-enabled,omitempty"`
+	AuthProvider   string `json:"auth-provider,omitempty"`
+	LocalCachePath string `json:"localCachePath,omitempty"`
 }
 
 type orasStoreFactory struct{}
 
 type orasStore struct {
-	config    *OrasStoreConf
-	rawConfig config.StoreConfig
+	config     *OrasStoreConf
+	rawConfig  config.StoreConfig
+	localCache *content.OCI
 }
 
-// Detect the loopback IP (127.0.0.1)
-var reLoopback = regexp.MustCompile(regexp.QuoteMeta("127.0.0.1"))
-
-// Detect the loopback IPV6 (::1)
-var reipv6Loopback = regexp.MustCompile(regexp.QuoteMeta("::1"))
-
 func init() {
 	factory.Register(storeName, &orasStoreFactory{})
 }
@@ -60,7 +60,16 @@ func (s *orasStoreFactory) Create(version string, storeConfig config.StorePlugin
 		return nil, fmt.Errorf("auth provider %s is not supported", conf.AuthProvider)
 	}
 
-	return &orasStore{config: &conf, rawConfig: config.StoreConfig{Version: version, Store: storeConfig}}, nil
+	// Set up the local cache where content will land when we pull
+	if conf.LocalCachePath == "" {
+		conf.LocalCachePath = defaultLocalCachePath
+	}
+	localRegistry, err := content.NewOCI(conf.LocalCachePath)
+	if err != nil {
+		return nil, fmt.Errorf("could not create local oras cache at path #{conf.LocalCachePath}: #{err}")
+	}
+
+	return &orasStore{config: &conf, rawConfig: config.StoreConfig{Version: version, Store: storeConfig}, localCache: localRegistry}, nil
 }
 
 func (store *orasStore) Name() string {
@@ -72,19 +81,31 @@ func (store *orasStore) GetConfig() *config.StoreConfig {
 }
 
 func (store *orasStore) ListReferrers(ctx context.Context, subjectReference common.Reference, artifactTypes []string, nextToken string) (referrerstore.ListReferrersResult, error) {
-	client, err := store.createRegistryClient(subjectReference.Path)
+	// TODO: handle nextToken
+	registryClient, err := store.createRegistryClient(subjectReference)
 	if err != nil {
 		return referrerstore.ListReferrersResult{}, err
 	}
 
-	referrers, err := client.GetReferrers(subjectReference, artifactTypes, nextToken)
+	var referrerDescriptors []artifactspec.Descriptor
+	if artifactTypes == nil {
+		artifactTypes = []string{""}
+	}
+	for _, artifactType := range artifactTypes {
+		_, res, err := oras.Discover(ctx, registryClient.Resolver, subjectReference.Original, artifactType)
+		if err != nil {
+			return referrerstore.ListReferrersResult{}, err
+		}
+		referrerDescriptors = append(referrerDescriptors, res...)
+	}
 
-	if err != nil && err != registry.ReferrersNotSupported {
-		return referrerstore.ListReferrersResult{}, err
+	var referrers []ocispecs.ReferenceDescriptor
+	for _, referrer := range referrerDescriptors {
+		referrers = append(referrers, ArtifactDescriptorToReferenceDescriptor(referrer))
 	}
 
 	if store.config.CosignEnabled {
-		cosignReferences, err := getCosignReferences(client, subjectReference)
+		cosignReferences, err := getCosignReferences(subjectReference)
 		if err != nil {
 			return referrerstore.ListReferrersResult{}, err
 		}
@@ -95,62 +116,58 @@ func (store *orasStore) ListReferrers(ctx context.Context, subjectReference comm
 }
 
 func (store *orasStore) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
-	client, err := store.createRegistryClient(subjectReference.Path)
+	registryClient, err := store.createRegistryClient(subjectReference)
 	if err != nil {
 		return nil, err
 	}
 
-	blob, _, err := client.GetReferenceBlob(subjectReference, digest)
-
+	ref := fmt.Sprintf("%s@%s", subjectReference.Path, digest)
+	desc, err := oras.Copy(ctx, registryClient, ref, store.localCache, "")
 	if err != nil {
 		return nil, err
 	}
 
-	return blob, nil
-
+	return store.getRawContentFromCache(ctx, desc)
 }
 
 func (store *orasStore) GetReferenceManifest(ctx context.Context, subjectReference common.Reference, referenceDesc ocispecs.ReferenceDescriptor) (ocispecs.ReferenceManifest, error) {
-	client, err := store.createRegistryClient(subjectReference.Path)
+	ref, err := name.ParseReference(fmt.Sprintf("%s@%s", subjectReference.Path, referenceDesc.Digest))
 	if err != nil {
 		return ocispecs.ReferenceManifest{}, err
 	}
-
-	subjectReference.Digest = referenceDesc.Digest
-	return client.GetReferenceManifest(subjectReference)
-}
-
-func (store *orasStore) createRegistryClient(path string) (*registry.Client, error) {
-	registryStr, _ := registry.GetRegistryRepoString(path)
-	authConfig, err := registry.DefaultAuthProvider.Provide(registryStr)
+	dig, err := remote.Get(ref)
 	if err != nil {
-		return nil, err
+		return ocispecs.ReferenceManifest{}, err
+	}
+	var manifest = artifactspec.Manifest{}
+	if err := json.Unmarshal(dig.Manifest, &manifest); err != nil {
+		return ocispecs.ReferenceManifest{}, err
 	}
 
-	return registry.NewClient(
-		registry.NewAuthtransport(
-			nil,
-			authConfig.Username,
-			authConfig.Password,
-		),
-		isInsecureRegistry(registryStr, store.config),
-	), nil
+	return ArtifactManifestToReferenceManifest(manifest), nil
 }
 
-func isInsecureRegistry(registry string, config *OrasStoreConf) bool {
-	if config.UseHttp {
-		return true
-	}
-	if strings.HasPrefix(registry, "localhost:") {
-		return true
+func (store *orasStore) createRegistryClient(targetRef common.Reference) (*content.Registry, error) {
+	// TODO: support authentication
+	registryOpts := content.RegistryOptions{
+		Configs:   nil,
+		Username:  "",
+		Password:  "",
+		Insecure:  isInsecureRegistry(targetRef.Original, store.config),
+		PlainHTTP: store.config.UseHttp,
 	}
+	return content.NewRegistryWithDiscover(targetRef.Original, registryOpts)
+}
 
-	if reLoopback.MatchString(registry) {
-		return true
+func (store *orasStore) getRawContentFromCache(ctx context.Context, descriptor oci.Descriptor) ([]byte, error) {
+	reader, err := store.localCache.Fetch(ctx, descriptor)
+	if err != nil {
+		return nil, err
 	}
-	if reipv6Loopback.MatchString(registry) {
-		return true
+	buf := make([]byte, descriptor.Size)
+	_, err = reader.Read(buf)
+	if err != nil {
+		return nil, err
 	}
-
-	return false
+	return buf, nil
 }