From 6c0265d1c4d22e891c18e9f0c11556191c1ac0af Mon Sep 17 00:00:00 2001 From: Tim Schrodi Date: Wed, 2 Jun 2021 12:17:12 +0200 Subject: [PATCH] Improve the repository context handling in the resolver also added: - a optional cache for the oci resolver - a util function to traverse all component references - unit tests for the resolver and the cache --- bindings-go/apis/v2/accesstypes.go | 2 +- bindings-go/apis/v2/cdutils/utils.go | 2 +- bindings-go/apis/v2/componentdescriptor.go | 13 +- bindings-go/apis/v2/default.go | 4 + bindings-go/apis/v2/helper.go | 16 +- bindings-go/apis/v2/list.go | 2 +- bindings-go/ctf/componentarchive.go | 2 +- bindings-go/ctf/ctf.go | 9 +- bindings-go/ctf/ctfutils/ctfutils.go | 81 ++++++++++ bindings-go/go.mod | 1 + bindings-go/go.sum | 2 + bindings-go/oci/oci_suite_test.go | 38 ++++- bindings-go/oci/resolve.go | 111 ++++++++++--- bindings-go/oci/resolve_test.go | 177 +++++++++++++++++++++ 14 files changed, 419 insertions(+), 41 deletions(-) create mode 100644 bindings-go/ctf/ctfutils/ctfutils.go create mode 100644 bindings-go/oci/resolve_test.go diff --git a/bindings-go/apis/v2/accesstypes.go b/bindings-go/apis/v2/accesstypes.go index b6e440d7..f0b6b810 100644 --- a/bindings-go/apis/v2/accesstypes.go +++ b/bindings-go/apis/v2/accesstypes.go @@ -72,7 +72,7 @@ func (O *OCIRegistryAccess) SetData(bytes []byte) error { // OCIBlobType is the access type of a oci blob in a manifest. const OCIBlobType = "ociBlob" -// OCIRegistryAccess describes the access for a oci registry. +// OCIBlobAccess describes the access for a oci registry. type OCIBlobAccess struct { ObjectType `json:",inline"` diff --git a/bindings-go/apis/v2/cdutils/utils.go b/bindings-go/apis/v2/cdutils/utils.go index 0e51a16f..4ac84897 100644 --- a/bindings-go/apis/v2/cdutils/utils.go +++ b/bindings-go/apis/v2/cdutils/utils.go @@ -114,7 +114,7 @@ func SetRawLabel(labels []v2.Label, name string, val []byte) []v2.Label { }) } -// SetExtraIdentity sets a extra identity field of a identity object. +// SetExtraIdentityField sets a extra identity field of a identity object. func SetExtraIdentityField(o *v2.IdentityObjectMeta, key, val string) { if o.ExtraIdentity == nil { o.ExtraIdentity = v2.Identity{} diff --git a/bindings-go/apis/v2/componentdescriptor.go b/bindings-go/apis/v2/componentdescriptor.go index 4562f432..abbbc80b 100644 --- a/bindings-go/apis/v2/componentdescriptor.go +++ b/bindings-go/apis/v2/componentdescriptor.go @@ -64,7 +64,7 @@ const ( ExternalRelation ResourceRelation = "external" ) -// Spec defines a versioned virtual component with a source and dependencies. +// ComponentDescriptor defines a versioned component with a source and dependencies. // +k8s:deepcopy-gen=true // +k8s:openapi-gen=true type ComponentDescriptor struct { @@ -93,7 +93,6 @@ type ComponentSpec struct { Resources []Resource `json:"resources"` } -// +k8s:deepcopy-gen=true // RepositoryContext describes a repository context. // +k8s:deepcopy-gen=true // +k8s:openapi-gen=true @@ -104,8 +103,8 @@ type RepositoryContext struct { BaseURL string `json:"baseUrl"` } -// +k8s:deepcopy-gen=true // ObjectMeta defines a object that is uniquely identified by its name and version. +// +k8s:deepcopy-gen=true type ObjectMeta struct { // Name is the context unique name of the object. Name string `json:"name"` @@ -228,7 +227,7 @@ func (o *IdentityObjectMeta) SetExtraIdentity(identity Identity) { o.ExtraIdentity = identity } -// GetLabels returns the identity of the object. +// GetIdentity returns the identity of the object. func (o *IdentityObjectMeta) GetIdentity() Identity { identity := map[string]string{} for k, v := range o.ExtraIdentity { @@ -243,8 +242,8 @@ func (o *IdentityObjectMeta) GetIdentityDigest() []byte { return o.GetIdentity().Digest() } -// +k8s:deepcopy-gen=true // ObjectType describes the type of a object +// +k8s:deepcopy-gen=true type ObjectType struct { // Type describes the type of the object. Type string `json:"type"` @@ -333,7 +332,7 @@ func NewEmptyUnstructured(ttype string) *UnstructuredAccessType { return NewUnstructuredType(ttype, nil) } -// NewCustomType creates a new custom typed object. +// NewUnstructuredType creates a new unstructured typed object. func NewUnstructuredType(ttype string, data map[string]interface{}) *UnstructuredAccessType { unstr := &UnstructuredAccessType{} unstr.Object = data @@ -529,7 +528,7 @@ func (o *ComponentReference) SetLabels(labels []Label) { o.Labels = labels } -// GetLabels returns the identity of the object. +// GetIdentity returns the identity of the object. func (o *ComponentReference) GetIdentity() Identity { identity := map[string]string{} for k, v := range o.ExtraIdentity { diff --git a/bindings-go/apis/v2/default.go b/bindings-go/apis/v2/default.go index 89267bf9..4a160aca 100644 --- a/bindings-go/apis/v2/default.go +++ b/bindings-go/apis/v2/default.go @@ -16,6 +16,9 @@ package v2 // DefaultComponent applies defaults to a component func DefaultComponent(component *ComponentDescriptor) error { + if component.RepositoryContexts == nil { + component.RepositoryContexts = make([]RepositoryContext, 0) + } if component.Sources == nil { component.Sources = make([]Source, 0) } @@ -30,6 +33,7 @@ func DefaultComponent(component *ComponentDescriptor) error { return nil } +// DefaultList defaults a list of components. func DefaultList(list *ComponentDescriptorList) error { for i, comp := range list.Components { if len(comp.Metadata.Version) == 0 { diff --git a/bindings-go/apis/v2/helper.go b/bindings-go/apis/v2/helper.go index 406b369f..fc4984aa 100644 --- a/bindings-go/apis/v2/helper.go +++ b/bindings-go/apis/v2/helper.go @@ -73,9 +73,21 @@ func NewNameSelector(name string) selector.Interface { // GetEffectiveRepositoryContext returns the current active repository context. func (c ComponentDescriptor) GetEffectiveRepositoryContext() RepositoryContext { + if len(c.RepositoryContexts) == 0 { + return RepositoryContext{} + } return c.RepositoryContexts[len(c.RepositoryContexts)-1] } +// InjectRepositoryContext appends the given repository context to components descriptor repository history. +// The context is not appended if the effective repository context already matches the current context. +func InjectRepositoryContext(cd *ComponentDescriptor, repoCtx RepositoryContext) { + effective := cd.GetEffectiveRepositoryContext() + if repoCtx != effective { + cd.RepositoryContexts = append(cd.RepositoryContexts, repoCtx) + } +} + // GetComponentReferences returns all component references that matches the given selectors. func (c ComponentDescriptor) GetComponentReferences(selectors ...IdentitySelector) ([]ComponentReference, error) { refs := make([]ComponentReference, 0) @@ -99,7 +111,7 @@ func (c ComponentDescriptor) GetComponentReferencesByName(name string) ([]Compon return c.GetComponentReferences(NewNameSelector(name)) } -// GetResourceByDefaultSelector returns resources that match the given selectors. +// GetResourceByJSONScheme returns resources that match the given selectors. func (c ComponentDescriptor) GetResourceByJSONScheme(src interface{}) ([]Resource, error) { sel, err := selector.NewJSONSchemaSelectorFromGoStruct(src) if err != nil { @@ -223,7 +235,7 @@ func (c ComponentDescriptor) GetResourcesByType(rtype string, selectors ...Ident }) } -// GetResourcesByType returns all local and external resources of a specific resource type. +// GetResourcesByName returns all local and external resources with a name. func (c ComponentDescriptor) GetResourcesByName(name string, selectors ...IdentitySelector) ([]Resource, error) { return c.getResourceBySelectors( append(selectors, NewNameSelector(name)), diff --git a/bindings-go/apis/v2/list.go b/bindings-go/apis/v2/list.go index 24c9b73e..411b8114 100644 --- a/bindings-go/apis/v2/list.go +++ b/bindings-go/apis/v2/list.go @@ -38,7 +38,7 @@ func (c *ComponentDescriptorList) GetComponent(name, version string) (ComponentD return ComponentDescriptor{}, errors.New("NotFound") } -// GetComponent returns all components that match the given name. +// GetComponentByName returns all components that match the given name. func (c *ComponentDescriptorList) GetComponentByName(name string) []ComponentDescriptor { comps := make([]ComponentDescriptor, 0) for _, comp := range c.Components { diff --git a/bindings-go/ctf/componentarchive.go b/bindings-go/ctf/componentarchive.go index 59ca0b98..d19fb138 100644 --- a/bindings-go/ctf/componentarchive.go +++ b/bindings-go/ctf/componentarchive.go @@ -220,7 +220,7 @@ func (ca *ComponentArchive) AddSource(src *v2.Source, info BlobInfo, reader io.R return nil } -// AddResource adds a blob resource to the current archive. +// AddResourceFromResolver adds a blob resource to the current archive. // If the specified resource already exists it will be overwritten. func (ca *ComponentArchive) AddResourceFromResolver(ctx context.Context, res *v2.Resource, resolver BlobResolver) error { if res == nil { diff --git a/bindings-go/ctf/ctf.go b/bindings-go/ctf/ctf.go index d1fcb655..f1c4d829 100644 --- a/bindings-go/ctf/ctf.go +++ b/bindings-go/ctf/ctf.go @@ -44,7 +44,8 @@ var UnsupportedResolveType = errors.New("UnsupportedResolveType") // ComponentResolver describes a general interface to resolve a component descriptor type ComponentResolver interface { - Resolve(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*v2.ComponentDescriptor, BlobResolver, error) + Resolve(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*v2.ComponentDescriptor, error) + ResolveWithBlobResolver(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*v2.ComponentDescriptor, BlobResolver, error) } // BlobResolver defines a resolver that can fetch @@ -151,7 +152,7 @@ func (ctf *CTF) AddComponentArchive(ca *ComponentArchive, format ArchiveFormat) return ctf.AddComponentArchiveWithName(filename, ca, format) } -// AddComponentArchive adds or updates a component archive in the ctf archive. +// AddComponentArchiveWithName adds or updates a component archive in the ctf archive. // The archive is added to the ctf with the given name func (ctf *CTF) AddComponentArchiveWithName(filename string, ca *ComponentArchive, format ArchiveFormat) error { file, err := ctf.tempFs.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) @@ -294,7 +295,7 @@ func (a *AggregatedBlobResolver) getResolver(res v2.Resource) (BlobResolver, err return nil, UnsupportedResolveType } -// AggregateBlobResolvers aggregartes two resolvers to one by using aggregated blob resolver. +// AggregateBlobResolvers aggregates two resolvers to one by using aggregated blob resolver. func AggregateBlobResolvers(a, b BlobResolver) (BlobResolver, error) { aggregated, ok := a.(*AggregatedBlobResolver) if ok { @@ -312,6 +313,6 @@ func AggregateBlobResolvers(a, b BlobResolver) (BlobResolver, error) { return aggregated, nil } - // create a new aggreagted resolver if neither a nor b are aggregations + // create a new aggregated resolver if neither a nor b are aggregations return NewAggregatedBlobResolver(a, b) } diff --git a/bindings-go/ctf/ctfutils/ctfutils.go b/bindings-go/ctf/ctfutils/ctfutils.go new file mode 100644 index 00000000..052318d0 --- /dev/null +++ b/bindings-go/ctf/ctfutils/ctfutils.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package ctfutils + +import ( + "context" + "fmt" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + "github.com/gardener/component-spec/bindings-go/ctf" +) + +// ResolveList resolves all component descriptors of a given root component descriptor. +func ResolveList(ctx context.Context, + resolver ctf.ComponentResolver, + repoCtx cdv2.RepositoryContext, + name, + version string) (*cdv2.ComponentDescriptorList, error) { + + list := &cdv2.ComponentDescriptorList{} + err := ResolveRecursive(ctx, resolver, repoCtx, name, version, func(cd *cdv2.ComponentDescriptor) (stop bool, err error) { + if _, err := list.GetComponent(cd.Name, cd.Version); err != nil { + list.Components = append(list.Components, *cd) + } + return false, nil + }) + if err != nil { + return nil, err + } + return list, nil +} + +// ResolvedCallbackFunc describes a function that is called when a component descriptor is resolved. +// The function can optionally return an bool which when set to true stops the resolve of further component descriptors +type ResolvedCallbackFunc func(descriptor *cdv2.ComponentDescriptor) (stop bool, err error) + +// ResolveRecursive recursively resolves all component descriptors dependencies. +// Everytime a new component descriptor is resolved the given callback function is called. +// The resolve of further components can be stopped when +// - the callback returns true for the stop parameter +// - the callback returns an error +// - all components are successfully resolved. +func ResolveRecursive(ctx context.Context, resolver ctf.ComponentResolver, repoCtx cdv2.RepositoryContext, name, version string, cb ResolvedCallbackFunc) error { + cd, err := resolver.Resolve(ctx, repoCtx, name, version) + if err != nil { + return fmt.Errorf("unable to resolve component descriptor for %q %q %q: %w", repoCtx.BaseURL, name, version, err) + } + stop, err := cb(cd) + if err != nil { + return fmt.Errorf("error while calling callback for %q %q %q: %w", repoCtx.BaseURL, name, version, err) + } + if stop { + return nil + } + return resolveRecursive(ctx, resolver, repoCtx, cd, cb) +} + +func resolveRecursive(ctx context.Context, resolver ctf.ComponentResolver, repoCtx cdv2.RepositoryContext, cd *cdv2.ComponentDescriptor, cb ResolvedCallbackFunc) error { + components := make([]*cdv2.ComponentDescriptor, len(cd.ComponentReferences)) + for _, ref := range cd.ComponentReferences { + cd, err := resolver.Resolve(ctx, repoCtx, ref.ComponentName, ref.Version) + if err != nil { + return fmt.Errorf("unable to resolve component descriptor for %q %q %q: %w", repoCtx.BaseURL, ref.ComponentName, ref.Version, err) + } + stop, err := cb(cd) + if err != nil { + return fmt.Errorf("error while calling callback for %q %q %q: %w", repoCtx.BaseURL, ref.ComponentName, ref.Version, err) + } + if stop { + return nil + } + } + for _, ref := range components { + if err := resolveRecursive(ctx, resolver, repoCtx, ref, cb); err != nil { + return err + } + } + return nil +} diff --git a/bindings-go/go.mod b/bindings-go/go.mod index a28c9272..3ffcec53 100644 --- a/bindings-go/go.mod +++ b/bindings-go/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/ghodss/yaml v1.0.0 + github.com/go-logr/logr v0.4.0 github.com/mandelsoft/vfs v0.0.0-20201002134249-3c471f64a4d1 github.com/onsi/ginkgo v1.14.0 github.com/onsi/gomega v1.10.1 diff --git a/bindings-go/go.sum b/bindings-go/go.sum index c307d757..3b2dbd61 100644 --- a/bindings-go/go.sum +++ b/bindings-go/go.sum @@ -22,6 +22,8 @@ github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2H github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= diff --git a/bindings-go/oci/oci_suite_test.go b/bindings-go/oci/oci_suite_test.go index 23b59a19..e252312b 100644 --- a/bindings-go/oci/oci_suite_test.go +++ b/bindings-go/oci/oci_suite_test.go @@ -5,10 +5,13 @@ package oci_test import ( + "context" + "io" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1" cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" "github.com/gardener/component-spec/bindings-go/oci" @@ -21,7 +24,6 @@ func TestConfig(t *testing.T) { var _ = Describe("helper", func(){ - Context("OCIRef", func() { It("should correctly parse a repository url without a protocol and a component", func() { @@ -52,7 +54,39 @@ var _ = Describe("helper", func(){ Expect(ref).To(Equal("example.com:443/component-descriptors/somecomp:v0.0.0")) }) - }) }) + +// testClient describes a test oci client. +type testClient struct { + getManifest func(ctx context.Context, ref string) (*ocispecv1.Manifest, error) + fetch func(ctx context.Context, ref string, desc ocispecv1.Descriptor, writer io.Writer) error +} + +var _ oci.Client = &testClient{} + +func (t testClient) GetManifest(ctx context.Context, ref string) (*ocispecv1.Manifest, error) { + return t.getManifest(ctx, ref) +} + +func (t testClient) Fetch(ctx context.Context, ref string, desc ocispecv1.Descriptor, writer io.Writer) error { + return t.fetch(ctx, ref, desc, writer) +} + +// testCache describes a test resolve cache. +type testCache struct { + get func (ctx context.Context, repoCtx cdv2.RepositoryContext, name, version string) (*cdv2.ComponentDescriptor, error) + store func(ctx context.Context, descriptor *cdv2.ComponentDescriptor) error +} + +var _ oci.Cache = &testCache{} + +func (t testCache) Get(ctx context.Context, repoCtx cdv2.RepositoryContext, name, version string) (*cdv2.ComponentDescriptor, error) { + return t.get(ctx, repoCtx, name, version) +} + +func (t testCache) Store(ctx context.Context, descriptor *cdv2.ComponentDescriptor) error { + return t.store(ctx, descriptor) +} + diff --git a/bindings-go/oci/resolve.go b/bindings-go/oci/resolve.go index 5d63e831..26ad59c0 100644 --- a/bindings-go/oci/resolve.go +++ b/bindings-go/oci/resolve.go @@ -34,8 +34,10 @@ import ( "github.com/gardener/component-spec/bindings-go/apis/v2/cdutils" "github.com/gardener/component-spec/bindings-go/codec" "github.com/gardener/component-spec/bindings-go/ctf" + "github.com/go-logr/logr" ) +// Client defines a readonly oci artifact client to fetch manifests and blobs. type Client interface { // GetManifest returns the ocispec Manifest for a reference GetManifest(ctx context.Context, ref string) (*ocispecv1.Manifest, error) @@ -59,45 +61,82 @@ func OCIRef(repoCtx v2.RepositoryContext, name, version string) (string, error) return fmt.Sprintf("%s:%s", ref, version), nil } -// Resolver is a generic resolve to resolve a component descriptor from a oci registry +// Cache describes a interface to cache component descriptors. +// The cache expects that a component descriptor identified by repoCtx, name and version is immutable. +// Currently only the raw component descriptor can be cached. +// The blob resolver might be added in the future. +type Cache interface { + // Get reads a component descriptor from the cache. + Get(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*v2.ComponentDescriptor, error) + // Store stores a component descriptor in the cache. + Store(ctx context.Context, descriptor *v2.ComponentDescriptor) error +} + +// Resolver is a generic resolve to resolve a component descriptor from a oci registry. +// This resolver implements the ctf.ComponentResolver interface. type Resolver struct { - repoCtx v2.RepositoryContext + log logr.Logger client Client + cache Cache decodeOpts []codec.DecodeOption } // NewResolver creates a new resolver. -func NewResolver(decodeOpts ...codec.DecodeOption) *Resolver { +func NewResolver(client Client, decodeOpts ...codec.DecodeOption) *Resolver { return &Resolver{ + log: logr.Discard(), + client: client, decodeOpts: decodeOpts, } } -// WithRepositoryContext sets the repository context of the resolver -func (r *Resolver) WithRepositoryContext(ctx v2.RepositoryContext) *Resolver { - r.repoCtx = ctx +// WithCache sets the oci client context of the resolver +func (r *Resolver) WithCache(cache Cache) *Resolver { + r.cache = cache return r } -// WithOCIClient sets the oci client context of the resolver -func (r *Resolver) WithOCIClient(client Client) *Resolver { - r.client = client +// WithLog sets the logger for the resolver. +func (r *Resolver) WithLog(log logr.Logger) *Resolver { + r.log = log return r } // Resolve resolves a component descriptor by name and version within the configured context. -func (r *Resolver) Resolve(ctx context.Context, name, version string) (*v2.ComponentDescriptor, ctf.BlobResolver, error) { - if r.repoCtx.Type != v2.OCIRegistryType { - return nil, nil, fmt.Errorf("unsupported type %s expected %s", r.repoCtx.Type, v2.OCIRegistryType) - } - ref, err := OCIRef(r.repoCtx, name, version) - if err != nil { - return nil, nil, fmt.Errorf("unable to generate oci reference: %w", err) +func (r *Resolver) Resolve(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*v2.ComponentDescriptor, error) { + cd, _, err := r.resolve(ctx, repoCtx, name, version, false) + return cd, err +} + +// ResolveWithBlobResolver resolves a component descriptor by name and version within the configured context. +// And it also returns a blob resolver to access the local artifacts. +func (r *Resolver) ResolveWithBlobResolver(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*v2.ComponentDescriptor, ctf.BlobResolver, error) { + return r.resolve(ctx, repoCtx, name, version, true) +} + +// resolve resolves a component descriptor by name and version within the configured context. +// If withBlobResolver is false the returned blobresolver is always nil +func (r *Resolver) resolve(ctx context.Context, repoCtx v2.RepositoryContext, name, version string, withBlobResolver bool) (*v2.ComponentDescriptor, ctf.BlobResolver, error) { + log := r.log.WithValues("repoCtx", repoCtx.BaseURL, "name", name, "version", version) + if r.cache != nil { + cd, err := r.cache.Get(ctx, repoCtx, name, version) + if err != nil { + log.Error(err, "unable to get component descriptor") + } else { + if withBlobResolver { + manifest, ref , err := r.fetchManifest(ctx, repoCtx, name, version) + if err != nil { + return nil, nil, err + } + return cd, NewBlobResolver(r.client, ref, manifest, cd), nil + } + return cd, nil, nil + } } - manifest, err := r.client.GetManifest(ctx, ref) + manifest, ref , err := r.fetchManifest(ctx, repoCtx, name, version) if err != nil { - return nil, nil, fmt.Errorf("unable to fetch manifest from ref %s: %w", ref, err) + return nil, nil, err } componentConfig, err := r.getComponentConfig(ctx, ref, manifest) @@ -131,12 +170,41 @@ func (r *Resolver) Resolve(ctx context.Context, name, version string) (*v2.Compo if err := codec.Decode(componentDescriptorBytes, cd, r.decodeOpts...); err != nil { return nil, nil, fmt.Errorf("unable to decode component descriptor: %w", err) } - return cd, NewBlobResolver(r.client, ref, manifest, cd), nil + v2.InjectRepositoryContext(cd, repoCtx) + + if r.cache != nil { + if err := r.cache.Store(ctx, cd.DeepCopy()); err != nil { + log.Error(err, "unable to store component descriptor") + } + } + + if withBlobResolver { + return cd, NewBlobResolver(r.client, ref, manifest, cd), nil + } + return cd, nil, nil +} + +// fetchManifest fetches the oci manifest. +// The manifest and the oci ref is returned. +func (r *Resolver) fetchManifest(ctx context.Context, repoCtx v2.RepositoryContext, name, version string) (*ocispecv1.Manifest, string, error) { + if repoCtx.Type != v2.OCIRegistryType { + return nil, "", fmt.Errorf("unsupported type %s expected %s", repoCtx.Type, v2.OCIRegistryType) + } + ref, err := OCIRef(repoCtx, name, version) + if err != nil { + return nil, "", fmt.Errorf("unable to generate oci reference: %w", err) + } + + manifest, err := r.client.GetManifest(ctx, ref) + if err != nil { + return nil, "", fmt.Errorf("unable to fetch manifest from ref %s: %w", ref, err) + } + return manifest, ref, nil } // ToComponentArchive creates a tar archive in the CTF (Cnudie Transport Format) from the given component descriptor. -func (r *Resolver) ToComponentArchive(ctx context.Context, name, version string, writer io.Writer) error { - cd, blobresolver, err := r.Resolve(ctx, name, version) +func (r *Resolver) ToComponentArchive(ctx context.Context, repoCtx v2.RepositoryContext, name, version string, writer io.Writer) error { + cd, blobresolver, err := r.ResolveWithBlobResolver(ctx, repoCtx, name, version) if err != nil { return err } @@ -147,7 +215,6 @@ func (r *Resolver) ToComponentArchive(ctx context.Context, name, version string, return fmt.Errorf("unable to add resource %s to archive: %w", res.GetName(), err) } } - return ca.WriteTar(writer) } diff --git a/bindings-go/oci/resolve_test.go b/bindings-go/oci/resolve_test.go new file mode 100644 index 00000000..6faf8ce8 --- /dev/null +++ b/bindings-go/oci/resolve_test.go @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: 2021 SAP SE or an SAP affiliate company and Gardener contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package oci_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + + "github.com/gardener/component-spec/bindings-go/codec" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/opencontainers/go-digest" + ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1" + + cdv2 "github.com/gardener/component-spec/bindings-go/apis/v2" + "github.com/gardener/component-spec/bindings-go/oci" +) + +var _ = Describe("resolve", func(){ + + Context("Resolve", func() { + + It("should fetch a component descriptor", func() { + ctx := context.Background() + ociClient := &testClient{ + getManifest: func(ctx context.Context, ref string) (*ocispecv1.Manifest, error) { + return &ocispecv1.Manifest{ + Config: ocispecv1.Descriptor{ + MediaType: oci.ComponentDescriptorConfigMimeType, + Digest: digest.FromString("config"), + }, + Layers: []ocispecv1.Descriptor{ + { + MediaType: oci.ComponentDescriptorJSONMimeType, + Digest: digest.FromString("cd"), + }, + }, + }, nil + }, + fetch: func(ctx context.Context, ref string, desc ocispecv1.Descriptor, writer io.Writer) error { + switch desc.Digest.String() { + case digest.FromString("config").String(): + config := oci.ComponentDescriptorConfig{ + ComponentDescriptorLayer: &oci.OciBlobRef{ + MediaType: oci.ComponentDescriptorConfigMimeType, + Digest: digest.FromString("cd").String(), + }, + } + return json.NewEncoder(writer).Encode(config) + case digest.FromString("cd").String(): + data, err := codec.Encode(defaultComponentDescriptor("example.com/my-comp", "0.0.0")) + if err != nil { + return err + } + if _, err := io.Copy(writer, bytes.NewBuffer(data)); err != nil { + return err + } + return nil + default: + return errors.New("unknown desc") + } + }, + } + cd, err := oci.NewResolver(ociClient).Resolve(ctx, cdv2.RepositoryContext{ + Type: cdv2.OCIRegistryType, + BaseURL: "example.com", + }, "example.com/my-comp", "0.0.0") + Expect(err).ToNot(HaveOccurred()) + Expect(cd.GetEffectiveRepositoryContext().BaseURL).To(Equal("example.com"), "the repository context should be injected") + }) + + It("should not fetch from the client of a cache is provided", func() { + ctx := context.Background() + ociCache := &testCache{ + get: func(ctx context.Context, repoCtx cdv2.RepositoryContext, name, version string) (*cdv2.ComponentDescriptor, error) { + return defaultComponentDescriptor("example.com/my-comp", "0.0.0"), nil + }, + store: func(ctx context.Context, descriptor *cdv2.ComponentDescriptor) error { + Expect(false).To(BeTrue(), "should not be called") + return nil + }, + } + ociClient := &testClient{ + getManifest: func(ctx context.Context, ref string) (*ocispecv1.Manifest, error) { + Expect(false).To(BeTrue(), "should not be called") + return nil, nil + }, + fetch: func(ctx context.Context, ref string, desc ocispecv1.Descriptor, writer io.Writer) error { + Expect(false).To(BeTrue(), "should not be called") + return nil + }, + } + cd, err := oci.NewResolver(ociClient).WithCache(ociCache).Resolve(ctx, cdv2.RepositoryContext{ + Type: cdv2.OCIRegistryType, + BaseURL: "example.com", + }, "example.com/my-comp", "0.0.0") + Expect(err).ToNot(HaveOccurred()) + Expect(cd.Name).To(Equal("example.com/my-comp")) + }) + + It("should store a component descriptor in the cache", func() { + ctx := context.Background() + storeCalled := false + ociCache := &testCache{ + get: func(ctx context.Context, repoCtx cdv2.RepositoryContext, name, version string) (*cdv2.ComponentDescriptor, error) { + return nil, errors.New("not found") + }, + store: func(ctx context.Context, descriptor *cdv2.ComponentDescriptor) error { + storeCalled = true + return nil + }, + } + ociClient := &testClient{ + getManifest: func(ctx context.Context, ref string) (*ocispecv1.Manifest, error) { + return &ocispecv1.Manifest{ + Config: ocispecv1.Descriptor{ + MediaType: oci.ComponentDescriptorConfigMimeType, + Digest: digest.FromString("config"), + }, + Layers: []ocispecv1.Descriptor{ + { + MediaType: oci.ComponentDescriptorJSONMimeType, + Digest: digest.FromString("cd"), + }, + }, + }, nil + }, + fetch: func(ctx context.Context, ref string, desc ocispecv1.Descriptor, writer io.Writer) error { + switch desc.Digest.String() { + case digest.FromString("config").String(): + config := oci.ComponentDescriptorConfig{ + ComponentDescriptorLayer: &oci.OciBlobRef{ + MediaType: oci.ComponentDescriptorConfigMimeType, + Digest: digest.FromString("cd").String(), + }, + } + return json.NewEncoder(writer).Encode(config) + case digest.FromString("cd").String(): + data, err := codec.Encode(defaultComponentDescriptor("example.com/my-comp", "0.0.0")) + if err != nil { + return err + } + if _, err := io.Copy(writer, bytes.NewBuffer(data)); err != nil { + return err + } + return nil + default: + return errors.New("unknown desc") + } + }, + } + cd, err := oci.NewResolver(ociClient).WithCache(ociCache).Resolve(ctx, cdv2.RepositoryContext{ + Type: cdv2.OCIRegistryType, + BaseURL: "example.com", + }, "example.com/my-comp", "0.0.0") + Expect(err).ToNot(HaveOccurred()) + Expect(cd.GetEffectiveRepositoryContext().BaseURL).To(Equal("example.com"), "the repository context should be injected") + Expect(storeCalled).To(BeTrue(), "the cache store function should be called") + }) + + }) + +}) + +func defaultComponentDescriptor(name, version string) *cdv2.ComponentDescriptor { + cd := &cdv2.ComponentDescriptor{} + cd.Name = name + cd.Version = version + cd.Provider = cdv2.InternalProvider + _ = cdv2.DefaultComponent(cd) + return cd +}