From 957d42a62a4cac5a83602820ee226494ca1ff60e Mon Sep 17 00:00:00 2001 From: Uwe Krueger Date: Wed, 21 Dec 2022 16:08:32 +0100 Subject: [PATCH] hash component command (#228) * hash component command * hash components in various flavours * improve online help for intermediate commands * make format work under IDE * fix accept header creation for oci blob access header with empty mediatype not accepted by Artifactory. --- TODO.md | 24 +- cmds/ocm/app/app.go | 2 + .../ocmcmds/common/cmds/signing/cmd.go | 35 ++- .../common/options/hashoption/option.go | 82 ++++++ .../common/options/signoption/option.go | 29 +- cmds/ocm/commands/ocmcmds/components/cmd.go | 2 + .../commands/ocmcmds/components/hash/cmd.go | 251 ++++++++++++++++++ .../ocmcmds/components/hash/cmd_test.go | 88 ++++++ .../ocmcmds/components/hash/options.go | 58 ++++ .../ocmcmds/components/hash/suite_test.go | 17 ++ cmds/ocm/commands/verbs/describe/cmd.go | 2 +- cmds/ocm/commands/verbs/hash/cmd.go | 23 ++ cmds/ocm/commands/verbs/verbs.go | 1 + cmds/ocm/pkg/output/options.go | 21 +- cmds/ocm/pkg/output/table.go | 6 +- docs/reference/ocm.md | 3 +- docs/reference/ocm_describe.md | 2 +- docs/reference/ocm_describe_artifacts.md | 2 +- docs/reference/ocm_describe_plugins.md | 2 +- docs/reference/ocm_hash.md | 25 ++ docs/reference/ocm_hash_componentversions.md | 128 +++++++++ hack/format.sh | 2 +- pkg/cobrautils/template.go | 4 +- pkg/common/walk.go | 22 +- pkg/contexts/ocm/compdesc/signing.go | 17 ++ .../v3alpha1/jsonscheme/bindata.go | 2 +- .../versions/v2/jsonscheme/bindata.go | 2 +- pkg/contexts/ocm/signing/handle.go | 161 ++++++----- pkg/contexts/ocm/transfer/transfer.go | 8 +- pkg/docker/fetcher.go | 6 +- 30 files changed, 880 insertions(+), 147 deletions(-) create mode 100644 cmds/ocm/commands/ocmcmds/common/options/hashoption/option.go create mode 100644 cmds/ocm/commands/ocmcmds/components/hash/cmd.go create mode 100644 cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go create mode 100644 cmds/ocm/commands/ocmcmds/components/hash/options.go create mode 100644 cmds/ocm/commands/ocmcmds/components/hash/suite_test.go create mode 100644 cmds/ocm/commands/verbs/hash/cmd.go create mode 100644 docs/reference/ocm_hash.md create mode 100644 docs/reference/ocm_hash_componentversions.md diff --git a/TODO.md b/TODO.md index 7857c4421..2112b1470 100644 --- a/TODO.md +++ b/TODO.md @@ -1,26 +1,4 @@ -- [X] Transfer should support force option for override -- [ ] Clarify how to list digests in (oci) ctf and artifact set under common oci API abstraction -- [ ] Align OCI and OCM Repository Name with type structure and reference syntax -- [ ] Aligned Denotation for OCM and OCI elements -- [X] Generalize Sort Field Detection and check - - first ugly implementation done -- [ ] Artifact download with side artifacts (based on dedicated option) +- [ ] Rework option complete handling with sessions - [ ] Rework Processing Chain/Iterable to work go-like with channels -- [ ] Generic History based sort function to be used to transform object sequence for tree output -- [ ] Recursion + Tree mode for OCI Indices - [ ] Better separation between typehandler/chain and chain/output -- [X] Rework digest handling based on Resource Type and Access Method Spec - - first cut and reorg done -- [X] Complete handling of all possible input variations for the various CLI commands. -- [X] Add signature handling -- [ ] Identify required scenarios for OCM transport handling and improve transport handler accordingly -- [ ] Add type factories for generic spec -- [ ] Provide size and digest info for OCM blob access -- [ ] CD validation for sources enforces git -- [ ] decide on and introduce logging framework -- [ ] add optional reference name to resource input -- [ ] remove registry from default output and show only with wide output option (get artifact) -- [ ] error handling for applying config (especially usage of ErrNoContext) -- [X] migrate resource add input spec to typed spec -- [X] Add Close method to access method to support external resource management diff --git a/cmds/ocm/app/app.go b/cmds/ocm/app/app.go index b735d0666..3d5d99611 100644 --- a/cmds/ocm/app/app.go +++ b/cmds/ocm/app/app.go @@ -39,6 +39,7 @@ import ( "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/describe" "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/download" "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/get" + "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/hash" "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/install" "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/show" "github.com/open-component-model/ocm/cmds/ocm/commands/verbs/sign" @@ -187,6 +188,7 @@ func newCliCommand(opts *CLIOptions, mod ...func(clictx.Context, *cobra.Command) cmd.AddCommand(create.NewCommand(opts.Context)) cmd.AddCommand(add.NewCommand(opts.Context)) cmd.AddCommand(sign.NewCommand(opts.Context)) + cmd.AddCommand(hash.NewCommand(opts.Context)) cmd.AddCommand(verify.NewCommand(opts.Context)) cmd.AddCommand(show.NewCommand(opts.Context)) cmd.AddCommand(transfer.NewCommand(opts.Context)) diff --git a/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go b/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go index f978434d5..f97ab93bc 100644 --- a/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go +++ b/cmds/ocm/commands/ocmcmds/common/cmds/signing/cmd.go @@ -21,6 +21,8 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/clictx" "github.com/open-component-model/ocm/pkg/contexts/ocm" "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" "github.com/open-component-model/ocm/pkg/contexts/ocm/signing" "github.com/open-component-model/ocm/pkg/errors" ) @@ -89,16 +91,20 @@ func (o *SignatureCommand) Run() error { if err != nil { return err } - return utils.HandleOutput(NewAction(o.spec.terms, o, sopts), handler, utils.StringElemSpecs(o.Refs...)...) + return utils.HandleOutput(NewAction(o.spec.terms, common.NewPrinter(o.Context.StdOut()), sopts), handler, utils.StringElemSpecs(o.Refs...)...) } ///////////////////////////////////////////////////////////////////////////// +type Action interface { + output.Output + Digest(o *comphdlr.Object) (*metav1.DigestSpec, *compdesc.ComponentDescriptor, error) +} + type action struct { desc []string - cmd *SignatureCommand printer common.Printer - state common.WalkingState + state signing.WalkingState baseresolver ocm.ComponentVersionResolver sopts *signing.Options errlist *errors.ErrorList @@ -106,24 +112,33 @@ type action struct { var _ output.Output = (*action)(nil) -func NewAction(desc []string, cmd *SignatureCommand, sopts *signing.Options) output.Output { +func NewAction(desc []string, p common.Printer, sopts *signing.Options) Action { return &action{ desc: desc, - cmd: cmd, - printer: common.NewPrinter(cmd.Context.StdOut()), - state: common.NewWalkingState(), + printer: p, + state: signing.NewWalkingState(), baseresolver: sopts.Resolver, sopts: sopts, errlist: errors.ErrListf(desc[1]), } } +func (a *action) Digest(o *comphdlr.Object) (*metav1.DigestSpec, *compdesc.ComponentDescriptor, error) { + sopts := *a.sopts + sopts.Resolver = ocm.NewCompoundResolver(o.Repository, a.sopts.Resolver) + d, err := signing.Apply(a.printer, &a.state, o.ComponentVersion, &sopts, true) + var cd *compdesc.ComponentDescriptor + vi := a.state.Get(common.VersionedElementKey(o.ComponentVersion)) + if vi != nil { + cd = vi.Descriptor + } + return d, cd, err +} + func (a *action) Add(e interface{}) error { o := e.(*comphdlr.Object) cv := o.ComponentVersion - sopts := *a.sopts - sopts.Resolver = ocm.NewCompoundResolver(o.Repository, a.sopts.Resolver) - d, err := signing.Apply(a.printer, &a.state, cv, &sopts, true) + d, _, err := a.Digest(o) a.errlist.Add(err) if err == nil { a.printer.Printf("successfully %s %s:%s (digest %s:%s)\n", a.desc[0], cv.GetName(), cv.GetVersion(), d.HashAlgorithm, d.Value) diff --git a/cmds/ocm/commands/ocmcmds/common/options/hashoption/option.go b/cmds/ocm/commands/ocmcmds/common/options/hashoption/option.go new file mode 100644 index 000000000..83c2a5945 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/common/options/hashoption/option.go @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package hashoption + +import ( + "github.com/spf13/pflag" + + "github.com/open-component-model/ocm/cmds/ocm/pkg/options" + "github.com/open-component-model/ocm/cmds/ocm/pkg/utils" + "github.com/open-component-model/ocm/pkg/contexts/clictx" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/normalizations/jsonv1" + ocmsign "github.com/open-component-model/ocm/pkg/contexts/ocm/signing" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/signing" + "github.com/open-component-model/ocm/pkg/signing/hasher/sha256" +) + +func From(o options.OptionSetProvider) *Option { + var opt *Option + o.AsOptionSet().Get(&opt) + return opt +} + +var _ options.Options = (*Option)(nil) + +func New() *Option { + return &Option{} +} + +type Option struct { + Hasher signing.Hasher + NormAlgorithm string + hashAlgorithm string +} + +func (o *Option) AddFlags(fs *pflag.FlagSet) { + fs.StringVarP(&o.NormAlgorithm, "normalization", "N", jsonv1.Algorithm, "normalization algorithm") + fs.StringVarP(&o.hashAlgorithm, "hash", "H", sha256.Algorithm, "hash algorithm") +} + +func (o *Option) Complete(ctx clictx.Context) error { + if o.NormAlgorithm == "" { + o.NormAlgorithm = jsonv1.Algorithm + } + if o.hashAlgorithm == "" { + o.hashAlgorithm = sha256.Algorithm + } + x := compdesc.Normalizations.Get(o.NormAlgorithm) + if x == nil { + return errors.ErrUnknown(compdesc.KIND_NORM_ALGORITHM, o.NormAlgorithm) + } + o.Hasher = signingattr.Get(ctx).GetHasher(o.hashAlgorithm) + if o.Hasher == nil { + return errors.ErrUnknown(compdesc.KIND_HASH_ALGORITHM, o.hashAlgorithm) + } + return nil +} + +func (o *Option) Usage() string { + s := ` +The following normalization modes are supported with option --normalization: +` + utils.FormatList(jsonv1.Algorithm, compdesc.Normalizations.Names()...) + + s += ` + +The following hash modes are supported with option --hash: +` + utils.FormatList(sha256.Algorithm, signing.DefaultRegistry().HasherNames()...) + + signing.DefaultRegistry().HasherNames() + return s +} + +var _ ocmsign.Option = (*Option)(nil) + +func (o *Option) ApplySigningOption(opts *ocmsign.Options) { + opts.NormalizationAlgo = o.NormAlgorithm + opts.Hasher = o.Hasher +} diff --git a/cmds/ocm/commands/ocmcmds/common/options/signoption/option.go b/cmds/ocm/commands/ocmcmds/common/options/signoption/option.go index 76c0583fd..ba1646add 100644 --- a/cmds/ocm/commands/ocmcmds/common/options/signoption/option.go +++ b/cmds/ocm/commands/ocmcmds/common/options/signoption/option.go @@ -12,6 +12,7 @@ import ( "github.com/mandelsoft/vfs/pkg/vfs" "github.com/spf13/pflag" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/hashoption" "github.com/open-component-model/ocm/cmds/ocm/pkg/options" "github.com/open-component-model/ocm/cmds/ocm/pkg/utils" "github.com/open-component-model/ocm/pkg/contexts/clictx" @@ -43,7 +44,6 @@ type Option struct { local bool SignMode bool signAlgorithm string - hashAlgorithm string publicKeys []string privateKeys []string Issuer string @@ -58,19 +58,18 @@ type Option struct { SignatureNames []string Update bool Signer signing.Signer - Hasher signing.Hasher Keys signing.KeyRegistry - NormAlgorithm string + + Hash hashoption.Option } func (o *Option) AddFlags(fs *pflag.FlagSet) { fs.StringArrayVarP(&o.SignatureNames, "signature", "s", nil, "signature name") fs.StringArrayVarP(&o.publicKeys, "public-key", "k", nil, "public key setting") if o.SignMode { + o.Hash.AddFlags(fs) fs.StringArrayVarP(&o.privateKeys, "private-key", "K", nil, "private key setting") fs.StringVarP(&o.signAlgorithm, "algorithm", "S", rsa.Algorithm, "signature handler") - fs.StringVarP(&o.NormAlgorithm, "normalization", "N", jsonv1.Algorithm, "normalization algorithm") - fs.StringVarP(&o.hashAlgorithm, "hash", "H", sha256.Algorithm, "hash algorithm") fs.StringVarP(&o.Issuer, "issuer", "I", "", "issuer name") fs.BoolVarP(&o.Update, "update", "", o.SignMode, "update digest in component versions") fs.BoolVarP(&o.Recursively, "recursive", "R", false, "recursively sign component versions") @@ -97,27 +96,17 @@ func (o *Option) Complete(ctx clictx.Context) error { o.Keys = signing.NewKeyRegistry() } if o.SignMode { - if o.NormAlgorithm == "" { - o.NormAlgorithm = jsonv1.Algorithm + err := o.Hash.Complete(ctx) + if err != nil { + return err } if o.signAlgorithm == "" { o.signAlgorithm = rsa.Algorithm } - if o.hashAlgorithm == "" { - o.hashAlgorithm = sha256.Algorithm - } - x := compdesc.Normalizations.Get(o.NormAlgorithm) - if x == nil { - return errors.ErrUnknown(compdesc.KIND_NORM_ALGORITHM, o.NormAlgorithm) - } o.Signer = signingattr.Get(ctx).GetSigner(o.signAlgorithm) if o.Signer == nil { return errors.ErrUnknown(compdesc.KIND_SIGN_ALGORITHM, o.signAlgorithm) } - o.Hasher = signingattr.Get(ctx).GetHasher(o.hashAlgorithm) - if o.Hasher == nil { - return errors.ErrUnknown(compdesc.KIND_HASH_ALGORITHM, o.hashAlgorithm) - } } else { o.Recursively = !o.local } @@ -236,8 +225,8 @@ func (o *Option) ApplySigningOption(opts *ocmsign.Options) { opts.Verify = o.Verify opts.Recursively = o.Recursively opts.Keys = o.Keys - opts.NormalizationAlgo = o.NormAlgorithm - opts.Hasher = o.Hasher + opts.NormalizationAlgo = o.Hash.NormAlgorithm + opts.Hasher = o.Hash.Hasher if o.Issuer != "" { opts.Issuer = o.Issuer } diff --git a/cmds/ocm/commands/ocmcmds/components/cmd.go b/cmds/ocm/commands/ocmcmds/components/cmd.go index de147d9cd..5d3cab8a4 100644 --- a/cmds/ocm/commands/ocmcmds/components/cmd.go +++ b/cmds/ocm/commands/ocmcmds/components/cmd.go @@ -10,6 +10,7 @@ import ( "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/add" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/download" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/get" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/hash" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/sign" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/verify" "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/names" @@ -31,6 +32,7 @@ func NewCommand(ctx clictx.Context) *cobra.Command { func AddCommands(ctx clictx.Context, cmd *cobra.Command) { cmd.AddCommand(add.NewCommand(ctx, add.Verb)) cmd.AddCommand(get.NewCommand(ctx, get.Verb)) + cmd.AddCommand(hash.NewCommand(ctx, hash.Verb)) cmd.AddCommand(sign.NewCommand(ctx, sign.Verb)) cmd.AddCommand(verify.NewCommand(ctx, verify.Verb)) cmd.AddCommand(download.NewCommand(ctx, download.Verb)) diff --git a/cmds/ocm/commands/ocmcmds/components/hash/cmd.go b/cmds/ocm/commands/ocmcmds/components/hash/cmd.go new file mode 100644 index 000000000..eef016e41 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/components/hash/cmd.go @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package hash + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/open-component-model/ocm/cmds/ocm/commands/common/options/closureoption" + ocmcommon "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/handlers/comphdlr" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/hashoption" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/lookupoption" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/repooption" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/versionconstraintsoption" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/names" + "github.com/open-component-model/ocm/cmds/ocm/commands/verbs" + "github.com/open-component-model/ocm/cmds/ocm/pkg/output" + "github.com/open-component-model/ocm/cmds/ocm/pkg/processing" + "github.com/open-component-model/ocm/cmds/ocm/pkg/utils" + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/clictx" + "github.com/open-component-model/ocm/pkg/contexts/ocm" + "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc" +) + +var ( + Names = names.Components + Verb = verbs.Hash +) + +type Command struct { + utils.BaseCommand + + Refs []string +} + +// NewCommand creates a new ctf command. +func NewCommand(ctx clictx.Context, names ...string) *cobra.Command { + return utils.SetupCommand( + &Command{BaseCommand: utils.NewBaseCommand(ctx, + versionconstraintsoption.New(), + output.OutputOptions(outputs, &Option{}, closureoption.New( + "component reference", output.Fields("IDENTITY"), addIdentityField), lookupoption.New(), hashoption.New(), repooption.New(), + ))}, + utils.Names(Names, names...)..., + ) +} + +func (o *Command) ForName(name string) *cobra.Command { + return &cobra.Command{ + Use: "[] {}", + Short: "hash component version", + Long: ` +Hash lists normalized forms for all component versions specified, if only a component is specified +all versions are listed. +`, + Example: ` +$ ocm hash componentversion ghcr.io/mandelsoft/kubelink +$ ocm hash componentversion --repo OCIRegistry:ghcr.io mandelsoft/kubelink +`, + } +} + +func (o *Command) Complete(args []string) error { + o.Refs = args + if len(args) == 0 && repooption.From(o).Spec == "" { + return fmt.Errorf("a repository or at least one argument that defines the reference is needed") + } + return nil +} + +func (o *Command) Run() error { + session := ocm.NewSession(nil) + defer session.Close() + + err := o.ProcessOnOptions(ocmcommon.CompleteOptionsWithSession(o, session)) + if err != nil { + return err + } + + err = From(o).Complete(o) + if err != nil { + return err + } + + handler := comphdlr.NewTypeHandler(o.Context.OCM(), session, repooption.From(o).Repository, comphdlr.OptionsFor(o)) + return utils.HandleArgs(output.From(o), handler, o.Refs...) +} + +///////////////////////////////////////////////////////////////////////////// + +func addIdentityField(e interface{}) []string { + p := e.(*comphdlr.Object) + return []string{p.Identity.String()} +} + +func TableOutput(opts *output.Options, h *handler, mapping processing.MappingFunction, wide ...string) *output.TableOutput { + def := &output.TableOutput{ + Headers: output.Fields("COMPONENT", "VERSION", "HASH", wide), + Options: opts, + Chain: comphdlr.Sort.Map(h.digester), + Mapping: mapping, + } + return closureoption.TableOutput(def, comphdlr.ClosureExplode) +} + +//////////////////////////////////////////////////////////////////////////////// + +type Object struct { + Spec ocm.RefSpec + History common.History + Descriptor *compdesc.ComponentDescriptor + Error error +} + +type Manifest struct { + History common.History `json:"context"` + Component string `json:"component"` + Version string `json:"version"` + Normalized string `json:"normalized,omitempty"` + Hash string `json:"hash,omitempty"` + Error string `json:"error,omitempty"` +} + +//////////////////////////////////////////////////////////////////////////////// + +var outputs = output.NewOutputs(getRegular, output.Outputs{ + "wide": getWide, +}).AddChainedManifestOutputs(output.ComposeChain(closureoption.OutputChainFunction(comphdlr.ClosureExplode, comphdlr.Sort), mapManifest)) + +func mapManifest(opts *output.Options) processing.ProcessChain { + h := newHandler(opts) + return processing.Map(h.digester).Map(h.manifester) +} + +func getRegular(opts *output.Options) output.Output { + h := newHandler(opts) + return TableOutput(opts, h, h.mapGetRegularOutput).New() +} + +func getWide(opts *output.Options) output.Output { + h := newHandler(opts) + return TableOutput(opts, h, h.mapGetWideOutput, "NORMALIZED FORM").New() +} + +//////////////////////////////////////////////////////////////////////////////// + +type handler struct { + opts *hashoption.Option + mode *Option +} + +func newHandler(opts *output.Options) *handler { + return &handler{ + opts: hashoption.From(opts), + mode: From(opts), + } +} + +func (h *handler) manifester(e interface{}) interface{} { + p := e.(*Object) + + tag := "-" + if p.Spec.Version != nil { + tag = *p.Spec.Version + } + + hist := p.History + if hist == nil { + hist = common.History{} + } + + m := &Manifest{ + History: hist, + Version: tag, + Component: p.Spec.Component, + } + + if p.Descriptor == nil { + if p.Error == nil { + m.Error = fmt.Sprintf("") + } else { + m.Error = p.Error.Error() + } + return m + } + norm, hash, err := compdesc.NormHash(p.Descriptor, h.opts.NormAlgorithm, h.opts.Hasher.Create()) + if err != nil { + m.Error = err.Error() + } else { + m.Normalized = string(norm) + m.Hash = hash + } + return m +} + +func (h *handler) digester(e interface{}) interface{} { + p := e.(*comphdlr.Object) + o := &Object{ + Spec: p.Spec, + History: p.History, + } + if p.ComponentVersion != nil { + o.Descriptor = p.ComponentVersion.GetDescriptor() + if h.mode.action != nil { + _, o.Descriptor, o.Error = h.mode.action.Digest(p) + } + } + return o +} + +func (h *handler) mapGetRegularOutput(e interface{}) interface{} { + p := e.(*Object) + + tag := "-" + if p.Spec.Version != nil { + tag = *p.Spec.Version + } + if p.Descriptor == nil { + if p.Error != nil { + return []string{p.Spec.Component, tag, "error: " + p.Error.Error()} + } + return []string{p.Spec.Component, tag, ""} + } + hash, err := compdesc.Hash(p.Descriptor, h.opts.NormAlgorithm, h.opts.Hasher.Create()) + if err != nil { + return []string{p.Spec.Component, tag, "error: " + err.Error()} + } + return []string{p.Spec.Component, tag, hash} +} + +func (h *handler) mapGetWideOutput(e interface{}) interface{} { + p := e.(*Object) + + tag := "-" + if p.Spec.Version != nil { + tag = *p.Spec.Version + } + if p.Descriptor == nil { + return []string{p.Spec.Component, tag, ""} + } + norm, hash, err := compdesc.NormHash(p.Descriptor, h.opts.NormAlgorithm, h.opts.Hasher.Create()) + if err != nil { + return []string{p.Spec.Component, tag, "error: " + err.Error(), ""} + } + return []string{p.Spec.Component, tag, hash, string(norm)} +} diff --git a/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go b/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go new file mode 100644 index 000000000..98c037b57 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/components/hash/cmd_test.go @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package hash_test + +import ( + "bytes" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/cmds/ocm/testhelper" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/open-component-model/ocm/pkg/common/accessio" + metav1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/mime" +) + +const ARCH = "/tmp/ca" +const VERSION = "v1" +const COMP = "test.de/x" +const PROVIDER = "mandelsoft" + +var _ = Describe("Test Environment", func() { + var env *TestEnv + + BeforeEach(func() { + env = NewTestEnv() + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("hash component archive", func() { + env.ComponentArchive(ARCH, accessio.FormatDirectory, COMP, VERSION, func() { + env.Provider(PROVIDER) + }) + + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute("hash", "components", ARCH, "-o", "wide")).To(Succeed()) + Expect(buf.String()).To(StringEqualTrimmedWithContext( + ` +COMPONENT VERSION HASH NORMALIZED FORM +test.de/x v1 7aff33d597be2bd358672bd79fb904666d955c3890d75d32cde356eb86355c65 [{"component":[{"componentReferences":[]},{"name":"test.de/x"},{"provider":"mandelsoft"},{"resources":[]},{"sources":[]},{"version":"v1"}]},{"meta":[{"schemaVersion":"v2"}]}] +`)) + }) + + It("hash component archive with resources", func() { + env.ComponentArchive(ARCH, accessio.FormatDirectory, COMP, VERSION, func() { + env.Provider(PROVIDER) + env.Resource("test", VERSION, resourcetypes.PLAIN_TEXT, metav1.LocalRelation, func() { + env.BlobStringData(mime.MIME_TEXT, "testdata") + }) + }) + + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute("hash", "components", ARCH, "-o", "wide")).To(Succeed()) + Expect(buf.String()).To(StringEqualTrimmedWithContext( + ` +COMPONENT : test.de/x +VERSION : v1 +HASH : c3f4916f8913fe99ad0abb18106c5f4b89950bae19b57aeb342bc054eebad056 +NORMALIZED FORM: [{"component":[{"componentReferences":[]},{"name":"test.de/x"},{"provider":"mandelsoft"},{"resources":[[{"digest":[{"hashAlgorithm":"sha256"},{"normalisationAlgorithm":"genericBlobDigest/v1"},{"value":"810ff2fb242a5dee4220f2cb0e6a519891fb67f2f828a6cab4ef8894633b1f50"}]},{"name":"test"},{"relation":"local"},{"type":"plainText"},{"version":"v1"}]]},{"sources":[]},{"version":"v1"}]},{"meta":[{"schemaVersion":"v2"}]}] +---`)) + }) + + It("hash component archive with resources", func() { + env.ComponentArchive(ARCH, accessio.FormatDirectory, COMP, VERSION, func() { + env.Provider(PROVIDER) + env.Resource("test", VERSION, resourcetypes.PLAIN_TEXT, metav1.LocalRelation, func() { + env.BlobStringData(mime.MIME_TEXT, "testdata") + }) + }) + + buf := bytes.NewBuffer(nil) + Expect(env.CatchOutput(buf).Execute("hash", "components", ARCH, "--actual", "-o", "wide")).To(Succeed()) + Expect(buf.String()).To(StringEqualTrimmedWithContext( + ` +COMPONENT : test.de/x +VERSION : v1 +HASH : f8a0fcfc2e0501d35657aa88ca6848abc15fc527a3a2b0055f72db977cf01d6b +NORMALIZED FORM: [{"component":[{"componentReferences":[]},{"name":"test.de/x"},{"provider":"mandelsoft"},{"resources":[[{"name":"test"},{"relation":"local"},{"type":"plainText"},{"version":"v1"}]]},{"sources":[]},{"version":"v1"}]},{"meta":[{"schemaVersion":"v2"}]}] +---`)) + }) +}) diff --git a/cmds/ocm/commands/ocmcmds/components/hash/options.go b/cmds/ocm/commands/ocmcmds/components/hash/options.go new file mode 100644 index 000000000..d6a1fd703 --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/components/hash/options.go @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package hash + +import ( + "github.com/spf13/pflag" + + signingcmd "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/cmds/signing" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/hashoption" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/lookupoption" + "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/repooption" + "github.com/open-component-model/ocm/cmds/ocm/pkg/options" + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/contexts/ocm/attrs/signingattr" + "github.com/open-component-model/ocm/pkg/contexts/ocm/signing" +) + +func From(o options.OptionSetProvider) *Option { + var opt *Option + o.AsOptionSet().Get(&opt) + return opt +} + +var _ options.Options = (*Option)(nil) + +type Option struct { + Actual bool + + action signingcmd.Action +} + +func (o *Option) AddFlags(fs *pflag.FlagSet) { + fs.BoolVarP(&o.Actual, "actual", "", false, "use actual component descriptor") +} + +func (o *Option) Complete(cmd *Command) error { + if o.Actual { + return nil + } + repo := repooption.From(cmd).Repository + lookup := lookupoption.From(cmd) + sopts := signing.NewOptions(hashoption.From(cmd), signing.Resolver(repo, lookup.Resolver)) + err := sopts.Complete(signingattr.Get(cmd.Context.OCMContext())) + if err == nil { + o.action = signingcmd.NewAction([]string{"", ""}, common.NewPrinter(nil), sopts) + } + return err +} + +func (o *Option) Usage() string { + s := ` +If the option --actual is given the component descriptor actually +found is used as it is, otherwise the required digests are calculated on-the-fly. +` + return s +} diff --git a/cmds/ocm/commands/ocmcmds/components/hash/suite_test.go b/cmds/ocm/commands/ocmcmds/components/hash/suite_test.go new file mode 100644 index 000000000..4e3a9b91e --- /dev/null +++ b/cmds/ocm/commands/ocmcmds/components/hash/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package hash_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "OCM hash components") +} diff --git a/cmds/ocm/commands/verbs/describe/cmd.go b/cmds/ocm/commands/verbs/describe/cmd.go index 2dbf0eab3..2e4f7047a 100644 --- a/cmds/ocm/commands/verbs/describe/cmd.go +++ b/cmds/ocm/commands/verbs/describe/cmd.go @@ -17,7 +17,7 @@ import ( // NewCommand creates a new command. func NewCommand(ctx clictx.Context) *cobra.Command { cmd := utils.MassageCommand(&cobra.Command{ - Short: "Describe artifacts", + Short: "Describe various elements by using appropriate sub commands.", }, verbs.Describe) cmd.AddCommand(resources.NewCommand(ctx)) cmd.AddCommand(plugins.NewCommand(ctx)) diff --git a/cmds/ocm/commands/verbs/hash/cmd.go b/cmds/ocm/commands/verbs/hash/cmd.go new file mode 100644 index 000000000..d5634caaf --- /dev/null +++ b/cmds/ocm/commands/verbs/hash/cmd.go @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package hash + +import ( + "github.com/spf13/cobra" + + components "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/components/hash" + "github.com/open-component-model/ocm/cmds/ocm/commands/verbs" + "github.com/open-component-model/ocm/cmds/ocm/pkg/utils" + "github.com/open-component-model/ocm/pkg/contexts/clictx" +) + +// NewCommand creates a new command. +func NewCommand(ctx clictx.Context) *cobra.Command { + cmd := utils.MassageCommand(&cobra.Command{ + Short: "Hash and normalization operations", + }, verbs.Hash) + cmd.AddCommand(components.NewCommand(ctx)) + return cmd +} diff --git a/cmds/ocm/commands/verbs/verbs.go b/cmds/ocm/commands/verbs/verbs.go index 8e4786cb5..711104e8a 100644 --- a/cmds/ocm/commands/verbs/verbs.go +++ b/cmds/ocm/commands/verbs/verbs.go @@ -7,6 +7,7 @@ package verbs const ( Get = "get" Describe = "describe" + Hash = "hash" Add = "add" Create = "create" Transfer = "transfer" diff --git a/cmds/ocm/pkg/output/options.go b/cmds/ocm/pkg/output/options.go index 5bf833881..8a2e69a61 100644 --- a/cmds/ocm/pkg/output/options.go +++ b/cmds/ocm/pkg/output/options.go @@ -15,7 +15,6 @@ import ( "github.com/open-component-model/ocm/cmds/ocm/pkg/options" "github.com/open-component-model/ocm/pkg/contexts/clictx" "github.com/open-component-model/ocm/pkg/errors" - "github.com/open-component-model/ocm/pkg/out" "github.com/open-component-model/ocm/pkg/utils" ) @@ -42,7 +41,7 @@ type Options struct { Output Output Sort []string FixedColums int - Context out.Context // this context could be ocm context. + Context clictx.Context // this context could be ocm context. Logging logging.Context } @@ -100,14 +99,24 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { func (o *Options) Complete(ctx clictx.Context) error { o.Context = ctx - var fields []string + + // process sub options first, to assure that output options are available for output + // mode creation + err := o.OptionSet.ProcessOnOptions(options.CompleteOptionsWithCLIContext(ctx)) + if err != nil { + return err + } if f := o.Outputs[o.OutputMode]; f == nil { return errors.ErrInvalid("output mode", o.OutputMode) } else { o.Output = f(o) } + var avail utils.StringSlice + + var fields []string + if s, ok := o.Output.(SortFields); ok { avail = s.GetSortFields() } @@ -125,11 +134,7 @@ func (o *Options) Complete(ctx clictx.Context) error { } } o.Sort = fields - err := o.OptionSet.ProcessOnOptions(options.CompleteOptionsWithCLIContext(ctx)) - if err != nil { - return err - } - return err + return nil } func (o *Options) CompleteAll(ctx clictx.Context) error { diff --git a/cmds/ocm/pkg/output/table.go b/cmds/ocm/pkg/output/table.go index eefc0389c..17641ee59 100644 --- a/cmds/ocm/pkg/output/table.go +++ b/cmds/ocm/pkg/output/table.go @@ -14,6 +14,7 @@ import ( func FormatTable(ctx Context, gap string, data [][]string) { columns := []int{} max := 0 + maxtitle := 0 formats := []string{} if len(data) > 1 { @@ -24,6 +25,9 @@ func FormatTable(ctx Context, gap string, data [][]string) { } else { formats = append(formats, "-") } + if len(data[0][i]) > maxtitle { + maxtitle = len(data[0][i]) + } } } @@ -50,7 +54,7 @@ func FormatTable(ctx Context, gap string, data [][]string) { } else { for c, col := range row { if c < len(first) { - Outf(ctx, "%s%s: %s\n", gap, first[c], col) + Outf(ctx, "%s%-*s: %s\n", gap, maxtitle, first[c], col) } else { Outf(ctx, "%s%d: %s\n", gap, c, col) } diff --git a/docs/reference/ocm.md b/docs/reference/ocm.md index 5a627b665..1d2464761 100644 --- a/docs/reference/ocm.md +++ b/docs/reference/ocm.md @@ -138,9 +138,10 @@ attributes are supported: * [ocm bootstrap](ocm_bootstrap.md) — bootstrap components * [ocm clean](ocm_clean.md) — Cleanup/re-organize elements * [ocm create](ocm_create.md) — Create transport or component archive -* [ocm describe](ocm_describe.md) — Describe artifacts +* [ocm describe](ocm_describe.md) — Describe various elements by using appropriate sub commands. * [ocm download](ocm_download.md) — Download oci artifacts, resources or complete components * [ocm get](ocm_get.md) — Get information about artifacts and components +* [ocm hash](ocm_hash.md) — Hash and normalization operations * [ocm install](ocm_install.md) — Install elements. * [ocm show](ocm_show.md) — Show tags or versions * [ocm sign](ocm_sign.md) — Sign components diff --git a/docs/reference/ocm_describe.md b/docs/reference/ocm_describe.md index bcece1ccf..11de60e71 100644 --- a/docs/reference/ocm_describe.md +++ b/docs/reference/ocm_describe.md @@ -1,4 +1,4 @@ -## ocm describe — Describe Artifacts +## ocm describe — Describe Various Elements By Using Appropriate Sub Commands. ### Synopsis diff --git a/docs/reference/ocm_describe_artifacts.md b/docs/reference/ocm_describe_artifacts.md index 35e53b95f..a5d466823 100644 --- a/docs/reference/ocm_describe_artifacts.md +++ b/docs/reference/ocm_describe_artifacts.md @@ -74,6 +74,6 @@ $ ocm describe artifact --repo OCIRegistry:ghcr.io mandelsoft/kubelink ##### Parents -* [ocm describe](ocm_describe.md) — Describe artifacts +* [ocm describe](ocm_describe.md) — Describe various elements by using appropriate sub commands. * [ocm](ocm.md) — Open Component Model command line client diff --git a/docs/reference/ocm_describe_plugins.md b/docs/reference/ocm_describe_plugins.md index 9d2a4be48..5f8d9837a 100644 --- a/docs/reference/ocm_describe_plugins.md +++ b/docs/reference/ocm_describe_plugins.md @@ -30,6 +30,6 @@ $ ocm describe plugins demo ##### Parents -* [ocm describe](ocm_describe.md) — Describe artifacts +* [ocm describe](ocm_describe.md) — Describe various elements by using appropriate sub commands. * [ocm](ocm.md) — Open Component Model command line client diff --git a/docs/reference/ocm_hash.md b/docs/reference/ocm_hash.md new file mode 100644 index 000000000..a50398ae6 --- /dev/null +++ b/docs/reference/ocm_hash.md @@ -0,0 +1,25 @@ +## ocm hash — Hash And Normalization Operations + +### Synopsis + +``` +ocm hash [] ... +``` + +### Options + +``` + -h, --help help for hash +``` + +### SEE ALSO + +##### Parents + +* [ocm](ocm.md) — Open Component Model command line client + + +##### Sub Commands + +* [ocm hash componentversions](ocm_hash_componentversions.md) — hash component version + diff --git a/docs/reference/ocm_hash_componentversions.md b/docs/reference/ocm_hash_componentversions.md new file mode 100644 index 000000000..6b8697fef --- /dev/null +++ b/docs/reference/ocm_hash_componentversions.md @@ -0,0 +1,128 @@ +## ocm hash componentversions — Hash Component Version + +### Synopsis + +``` +ocm hash componentversions [] {} +``` + +### Options + +``` + --actual use actual component descriptor + -c, --constraints constraints version constraint + -H, --hash string hash algorithm (default "sha256") + -h, --help help for componentversions + --latest restrict component versions to latest + --lookup stringArray repository name or spec for closure lookup fallback + -N, --normalization string normalization algorithm (default "jsonNormalisation/v1") + -o, --output string output mode (JSON, json, wide, yaml) + -r, --recursive follow component reference nesting + --repo string repository name or spec + -s, --sort stringArray sort fields +``` + +### Description + + +Hash lists normalized forms for all component versions specified, if only a component is specified +all versions are listed. + +If the option --constraints is given, and no version is specified for a component, only versions matching +the given version constraints (semver https://github.com/Masterminds/semver) are selected. With --latest only +the latest matching versions will be selected. + +If the option --actual is given the component descriptor actually +found is used as it is, otherwise the required digests are calculated on-the-fly. + +With the option --recursive the complete reference tree of a component reference is traversed. + +If a component lookup for building a reference closure is required +the --lookup option can be used to specify a fallback +lookup repository. +By default the component versions are searched in the repository +holding the component version for which the closure is determined. +For *Component Archives* this is never possible, because it only +contains a single component version. Therefore, in this scenario +this option must always be specified to be able to follow component +references. + +The following normalization modes are supported with option --normalization: + + - jsonNormalisation/v1 (default): + - jsonNormalisation/v2: + + +The following hash modes are supported with option --hash: + + - NO-DIGEST: + - sha256 (default): + - sha512: + +If the --repo option is specified, the given names are interpreted +relative to the specified repository using the syntax + +
+ 
<component>[:<version>]
+
+ +If no --repo option is specified the given names are interpreted +as located OCM component version references: + +
+ 
[<repo type>::]<host>[:<port>][/<base path>]//<component>[:<version>]
+
+ +Additionally there is a variant to denote common transport archives +and general repository specifications + +
+ 
[<repo type>::]<filepath>|<spec json>[//<component>[:<version>]]
+
+ +The --repo option takes an OCM repository specification: + +
+ 
[<repo type>::]<configured name>|<file path>|<spec json>
+
+ +For the *Common Transport Format* the types directory, +tar or tgz is possible. + +Using the JSON variant any repository type supported by the +linked library can be used: + +Dedicated OCM repository types: +- `ComponentArchive` + +OCI Repository types (using standard component repository to OCI mapping): +- `ArtifactSet` +- `CommonTransportFormat` +- `DockerDaemon` +- `Empty` +- `OCIRegistry` +- `oci` +- `ociRegistry` + +With the option --output the output mode can be selected. +The following modes are supported: + - JSON + - json + - wide + - yaml + + +### Examples + +``` +$ ocm hash componentversion ghcr.io/mandelsoft/kubelink +$ ocm hash componentversion --repo OCIRegistry:ghcr.io mandelsoft/kubelink +``` + +### SEE ALSO + +##### Parents + +* [ocm hash](ocm_hash.md) — Hash and normalization operations +* [ocm](ocm.md) — Open Component Model command line client + diff --git a/hack/format.sh b/hack/format.sh index eebad5780..be324d634 100755 --- a/hack/format.sh +++ b/hack/format.sh @@ -16,7 +16,7 @@ pkgprefix="github.com/open-component-model/ocm" log "Format with gci" GCIFMT=( -s standard -s blank -s dot -s default -s="prefix(${pkgprefix})" --custom-order ) -gci diff --skip-generated "${GCIFMT[@]}" $@ \ +gci diff --skip-generated "${GCIFMT[@]}" $@ -h for additional help. +{{end}}{{if .HasExample}} Examples: {{.Example | indent 2}}{{end}}{{if .HasHelpSubCommands}} diff --git a/pkg/common/walk.go b/pkg/common/walk.go index a7f93a3e1..8e42885f7 100644 --- a/pkg/common/walk.go +++ b/pkg/common/walk.go @@ -8,9 +8,9 @@ import ( "github.com/open-component-model/ocm/pkg/utils" ) -type NameVersionInfo map[NameVersion]interface{} +type NameVersionInfo[T any] map[NameVersion]T -func (s NameVersionInfo) Add(nv NameVersion, data ...interface{}) bool { +func (s NameVersionInfo[T]) Add(nv NameVersion, data ...T) bool { if _, ok := s[nv]; !ok { s[nv] = utils.Optional(data...) return true @@ -18,28 +18,32 @@ func (s NameVersionInfo) Add(nv NameVersion, data ...interface{}) bool { return false } -func (s NameVersionInfo) Contains(nv NameVersion) bool { +func (s NameVersionInfo[T]) Contains(nv NameVersion) bool { _, ok := s[nv] return ok } -type WalkingState struct { - Closure NameVersionInfo +type WalkingState[T any] struct { + Closure NameVersionInfo[T] History History } -func NewWalkingState() WalkingState { - return WalkingState{Closure: NameVersionInfo{}} +func NewWalkingState[T any]() WalkingState[T] { + return WalkingState[T]{Closure: NameVersionInfo[T]{}} } -func (s *WalkingState) Add(kind string, nv NameVersion) (bool, error) { +func (s *WalkingState[T]) Add(kind string, nv NameVersion) (bool, error) { if err := s.History.Add(kind, nv); err != nil { return false, err } return s.Closure.Add(nv), nil } -func (s *WalkingState) Contains(nv NameVersion) bool { +func (s *WalkingState[T]) Contains(nv NameVersion) bool { _, ok := s.Closure[nv] return ok } + +func (s *WalkingState[T]) Get(nv NameVersion) T { + return s.Closure[nv] +} diff --git a/pkg/contexts/ocm/compdesc/signing.go b/pkg/contexts/ocm/compdesc/signing.go index 0487352ae..c4a8b810d 100644 --- a/pkg/contexts/ocm/compdesc/signing.go +++ b/pkg/contexts/ocm/compdesc/signing.go @@ -64,6 +64,23 @@ func Hash(cd *ComponentDescriptor, normAlgo string, hash hash.Hash) (string, err return hex.EncodeToString(hash.Sum(nil)), nil } +func NormHash(cd *ComponentDescriptor, normAlgo string, hash hash.Hash) ([]byte, string, error) { + if hash == nil { + return nil, metav1.NoDigest, nil + } + + normalized, err := Normalize(cd, normAlgo) + if err != nil { + return nil, "", fmt.Errorf("failed normalising component descriptor %w", err) + } + hash.Reset() + if _, err = hash.Write(normalized); err != nil { + return nil, "", fmt.Errorf("failed hashing the normalisedComponentDescriptorJson: %w", err) + } + + return normalized, hex.EncodeToString(hash.Sum(nil)), nil +} + // Sign signs the given component-descriptor with the signer. // The component-descriptor has to contain digests for componentReferences and resources. func Sign(cd *ComponentDescriptor, privateKey interface{}, signer signing.Signer, hasher signing.Hasher, signatureName, issuer string) error { diff --git a/pkg/contexts/ocm/compdesc/versions/ocm.software/v3alpha1/jsonscheme/bindata.go b/pkg/contexts/ocm/compdesc/versions/ocm.software/v3alpha1/jsonscheme/bindata.go index 254f2bbed..ec4279f05 100644 --- a/pkg/contexts/ocm/compdesc/versions/ocm.software/v3alpha1/jsonscheme/bindata.go +++ b/pkg/contexts/ocm/compdesc/versions/ocm.software/v3alpha1/jsonscheme/bindata.go @@ -94,7 +94,7 @@ func ResourcesComponentDescriptorOcmV3SchemaYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "../../../../../../../../resources/component-descriptor-ocm-v3-schema.yaml", size: 10607, mode: os.FileMode(436), modTime: time.Unix(1671033040, 0)} + info := bindataFileInfo{name: "../../../../../../../../resources/component-descriptor-ocm-v3-schema.yaml", size: 10607, mode: os.FileMode(436), modTime: time.Unix(1671551562, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/contexts/ocm/compdesc/versions/v2/jsonscheme/bindata.go b/pkg/contexts/ocm/compdesc/versions/v2/jsonscheme/bindata.go index 475c05057..cac6ef2dd 100644 --- a/pkg/contexts/ocm/compdesc/versions/v2/jsonscheme/bindata.go +++ b/pkg/contexts/ocm/compdesc/versions/v2/jsonscheme/bindata.go @@ -94,7 +94,7 @@ func ResourcesComponentDescriptorV2SchemaYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "../../../../../../../resources/component-descriptor-v2-schema.yaml", size: 10392, mode: os.FileMode(436), modTime: time.Unix(1671033040, 0)} + info := bindataFileInfo{name: "../../../../../../../resources/component-descriptor-v2-schema.yaml", size: 10392, mode: os.FileMode(436), modTime: time.Unix(1671551562, 0)} a := &asset{bytes: bytes, info: info} return a, nil } diff --git a/pkg/contexts/ocm/signing/handle.go b/pkg/contexts/ocm/signing/handle.go index 793959d32..ad3447c80 100644 --- a/pkg/contexts/ocm/signing/handle.go +++ b/pkg/contexts/ocm/signing/handle.go @@ -18,25 +18,44 @@ import ( "github.com/open-component-model/ocm/pkg/utils" ) +type VersionInfo struct { + Descriptor *compdesc.ComponentDescriptor + Digest *metav1.DigestSpec + Signed bool +} + func ToDigestSpec(v interface{}) *metav1.DigestSpec { if v == nil { return nil } - return v.(*metav1.DigestSpec) + return v.(*VersionInfo).Digest +} + +type WalkingState = common.WalkingState[*VersionInfo] + +func NewWalkingState() WalkingState { + return common.NewWalkingState[*VersionInfo]() } -func Apply(printer common.Printer, state *common.WalkingState, cv ocm.ComponentVersionAccess, opts *Options, closecv ...bool) (*metav1.DigestSpec, error) { +func Apply(printer common.Printer, state *WalkingState, cv ocm.ComponentVersionAccess, opts *Options, closecv ...bool) (*metav1.DigestSpec, error) { if printer == nil { printer = common.NewPrinter(nil) } if state == nil { - s := common.NewWalkingState() + s := common.NewWalkingState[*VersionInfo]() state = &s } return apply(printer, *state, cv, opts, utils.Optional(closecv...)) } -func apply(printer common.Printer, state common.WalkingState, cv ocm.ComponentVersionAccess, opts *Options, closecv bool) (d *metav1.DigestSpec, efferr error) { +func RequireReProcessing(vi *VersionInfo, opts *Options) bool { + if vi == nil { + return true + } + return opts.DoSign() && !vi.Signed +} + +func apply(printer common.Printer, state WalkingState, cv ocm.ComponentVersionAccess, opts *Options, closecv bool) (d *metav1.DigestSpec, efferr error) { var closer errors.ErrorFunction if closecv { closer = cv.Close @@ -44,16 +63,28 @@ func apply(printer common.Printer, state common.WalkingState, cv ocm.ComponentVe nv := common.VersionedElementKey(cv) defer errors.PropagateErrorf(&efferr, closer, "%s", state.History.Append(nv)) + vi := state.Get(nv) if ok, err := state.Add(ocm.KIND_COMPONENTVERSION, nv); !ok { - return ToDigestSpec(state.Closure[nv]), err + if err != nil || !RequireReProcessing(vi, opts) { + return ToDigestSpec(vi), err + } } - return _apply(printer, state, nv, cv, opts) + return _apply(printer, state, nv, cv, vi, opts) } -func _apply(printer common.Printer, state common.WalkingState, nv common.NameVersion, cv ocm.ComponentVersionAccess, opts *Options) (*metav1.DigestSpec, error) { - cd := cv.GetDescriptor().Copy() +func _apply(printer common.Printer, state WalkingState, nv common.NameVersion, cv ocm.ComponentVersionAccess, vi *VersionInfo, opts *Options) (*metav1.DigestSpec, error) { + var cd *compdesc.ComponentDescriptor + + prefix := "" + if vi == nil { + cd = cv.GetDescriptor().Copy() + vi = &VersionInfo{Descriptor: cd} + } else { + cd = vi.Descriptor + prefix = "re" + } octx := cv.GetContext() - printer.Printf("applying to version %q...\n", nv) + printer.Printf("%sapplying to version %q...\n", prefix, nv) signatureNames := opts.SignatureNames if len(signatureNames) == 0 { @@ -73,61 +104,64 @@ func _apply(printer common.Printer, state common.WalkingState, nv common.NameVer } } - if err := calculateReferenceDigests(printer, cd, state, opts); err != nil { - return nil, err - } - - blobdigesters := cv.GetContext().BlobDigesters() - for i, res := range cv.GetResources() { - raw := &cd.Resources[i] - acc, err := res.Access() - if err != nil { - return nil, errors.Wrapf(err, resMsg(raw, "", "failed getting access for resource")) - } - if _, ok := opts.SkipAccessTypes[acc.GetKind()]; ok { - // set the do not sign digest notation on skip-access-type resources - cd.Resources[i].Digest = metav1.NewExcludeFromSignatureDigest() - continue - } - // special digest notation indicates to not digest the content - if cd.Resources[i].Digest != nil && reflect.DeepEqual(cd.Resources[i].Digest, metav1.NewExcludeFromSignatureDigest()) { - continue + if vi.Digest == nil { + if err := calculateReferenceDigests(printer, cd, state, opts); err != nil { + return nil, err } - meth, err := acc.AccessMethod(cv) - if err != nil { - return nil, errors.Wrapf(err, resMsg(raw, acc.Describe(octx), "failed creating access for resource")) - } - var req []cpi.DigesterType - if raw.Digest != nil { - req = []cpi.DigesterType{ - { - HashAlgorithm: raw.Digest.HashAlgorithm, - NormalizationAlgorithm: raw.Digest.NormalisationAlgorithm, - }, + blobdigesters := cv.GetContext().BlobDigesters() + for i, res := range cv.GetResources() { + raw := &cd.Resources[i] + acc, err := res.Access() + if err != nil { + return nil, errors.Wrapf(err, resMsg(raw, "", "failed getting access for resource")) + } + if _, ok := opts.SkipAccessTypes[acc.GetKind()]; ok { + // set the do not sign digest notation on skip-access-type resources + cd.Resources[i].Digest = metav1.NewExcludeFromSignatureDigest() + continue + } + // special digest notation indicates to not digest the content + if cd.Resources[i].Digest != nil && reflect.DeepEqual(cd.Resources[i].Digest, metav1.NewExcludeFromSignatureDigest()) { + continue + } + + meth, err := acc.AccessMethod(cv) + if err != nil { + return nil, errors.Wrapf(err, resMsg(raw, acc.Describe(octx), "failed creating access for resource")) } + var req []cpi.DigesterType + if raw.Digest != nil { + req = []cpi.DigesterType{ + { + HashAlgorithm: raw.Digest.HashAlgorithm, + NormalizationAlgorithm: raw.Digest.NormalisationAlgorithm, + }, + } + } + digest, err := blobdigesters.DetermineDigests(res.Meta().GetType(), opts.Hasher, opts.Registry, meth, req...) + if err != nil { + return nil, errors.Wrapf(err, resMsg(raw, acc.Describe(octx), "failed determining digest for resource")) + } + if len(digest) == 0 { + return nil, errors.Newf(resMsg(raw, acc.Describe(octx), "no digester accepts resource")) + } + if raw.Digest != nil && !reflect.DeepEqual(*raw.Digest, digest[0]) { + return nil, errors.Newf(resMsg(raw, acc.Describe(octx), "calculated resource digest (%+v) mismatches existing digest (%+v) for", digest, raw.Digest)) + } + cd.Resources[i].Digest = &digest[0] + printer.Printf(" resource %d: %s: digest %s\n", i, res.Meta().GetIdentity(cv.GetDescriptor().Resources), &digest[0]) } - digest, err := blobdigesters.DetermineDigests(res.Meta().GetType(), opts.Hasher, opts.Registry, meth, req...) + digest, err := compdesc.Hash(cd, opts.NormalizationAlgo, opts.Hasher.Create()) if err != nil { - return nil, errors.Wrapf(err, resMsg(raw, acc.Describe(octx), "failed determining digest for resource")) - } - if len(digest) == 0 { - return nil, errors.Newf(resMsg(raw, acc.Describe(octx), "no digester accepts resource")) + return nil, errors.Wrapf(err, "failed hashing component descriptor") } - if raw.Digest != nil && !reflect.DeepEqual(*raw.Digest, digest[0]) { - return nil, errors.Newf(resMsg(raw, acc.Describe(octx), "calculated resource digest (%+v) mismatches existing digest (%+v) for", digest, raw.Digest)) + spec := &metav1.DigestSpec{ + HashAlgorithm: opts.Hasher.Algorithm(), + NormalisationAlgorithm: opts.NormalizationAlgo, + Value: digest, } - cd.Resources[i].Digest = &digest[0] - printer.Printf(" resource %d: %s: digest %s\n", i, res.Meta().GetIdentity(cv.GetDescriptor().Resources), &digest[0]) - } - digest, err := compdesc.Hash(cd, opts.NormalizationAlgo, opts.Hasher.Create()) - if err != nil { - return nil, errors.Wrapf(err, "failed hashing component descriptor") - } - spec := &metav1.DigestSpec{ - HashAlgorithm: opts.Hasher.Algorithm(), - NormalisationAlgorithm: opts.NormalizationAlgo, - Value: digest, + vi.Digest = spec } if opts.DoVerify() { @@ -138,7 +172,7 @@ func _apply(printer common.Printer, state common.WalkingState, nv common.NameVer found := cd.GetSignatureIndex(opts.SignatureName()) if opts.DoSign() && (!opts.DoVerify() || found == -1) { - sig, err := opts.Signer.Sign(digest, opts.Hasher.Crypto(), opts.Issuer, opts.PrivateKey()) + sig, err := opts.Signer.Sign(vi.Digest.Value, opts.Hasher.Crypto(), opts.Issuer, opts.PrivateKey()) if err != nil { return nil, errors.Wrapf(err, "failed signing component descriptor") } @@ -151,7 +185,7 @@ func _apply(printer common.Printer, state common.WalkingState, nv common.NameVer } signature := metav1.Signature{ Name: opts.SignatureName(), - Digest: *spec, + Digest: *vi.Digest, Signature: metav1.SignatureSpec{ Algorithm: sig.Algorithm, Value: sig.Value, @@ -175,10 +209,11 @@ func _apply(printer common.Printer, state common.WalkingState, nv common.NameVer } if opts.DoSign() { orig.Signatures = cd.Signatures + vi.Signed = true } } - state.Closure[nv] = spec - return spec, nil + state.Closure[nv] = vi + return vi.Digest, nil } func refMsg(ref compdesc.ComponentReference, msg string, args ...interface{}) string { @@ -192,7 +227,7 @@ func resMsg(ref *compdesc.Resource, acc string, msg string, args ...interface{}) return fmt.Sprintf("%s %s:%s", fmt.Sprintf(msg, args...), ref.Name, ref.Version) } -func doVerify(printer common.Printer, cd *compdesc.ComponentDescriptor, state common.WalkingState, signatureNames []string, opts *Options) error { +func doVerify(printer common.Printer, cd *compdesc.ComponentDescriptor, state WalkingState, signatureNames []string, opts *Options) error { var err error found := []string{} for _, n := range signatureNames { @@ -236,7 +271,7 @@ func doVerify(printer common.Printer, cd *compdesc.ComponentDescriptor, state co return nil } -func calculateReferenceDigests(printer common.Printer, cd *compdesc.ComponentDescriptor, state common.WalkingState, opts *Options) error { +func calculateReferenceDigests(printer common.Printer, cd *compdesc.ComponentDescriptor, state WalkingState, opts *Options) error { for i, reference := range cd.References { var calculatedDigest *metav1.DigestSpec if reference.Digest == nil && !opts.DoUpdate() { diff --git a/pkg/contexts/ocm/transfer/transfer.go b/pkg/contexts/ocm/transfer/transfer.go index 8e440d36b..1fe13d927 100644 --- a/pkg/contexts/ocm/transfer/transfer.go +++ b/pkg/contexts/ocm/transfer/transfer.go @@ -20,7 +20,9 @@ import ( "github.com/open-component-model/ocm/pkg/runtime" ) -type TransportClosure = common.NameVersionInfo +type WalkingState = common.WalkingState[*struct{}] + +type TransportClosure = common.NameVersionInfo[*struct{}] func TransferVersion(printer common.Printer, closure TransportClosure, src ocmcpi.ComponentVersionAccess, tgt ocmcpi.Repository, handler transferhandler.TransferHandler) error { if closure == nil { @@ -29,11 +31,11 @@ func TransferVersion(printer common.Printer, closure TransportClosure, src ocmcp if printer == nil { printer = common.NewPrinter(nil) } - state := common.WalkingState{Closure: closure} + state := WalkingState{Closure: closure} return transferVersion(printer, Logger(src), state, src, tgt, handler) } -func transferVersion(printer common.Printer, log logging.Logger, state common.WalkingState, src ocmcpi.ComponentVersionAccess, tgt ocmcpi.Repository, handler transferhandler.TransferHandler) error { +func transferVersion(printer common.Printer, log logging.Logger, state WalkingState, src ocmcpi.ComponentVersionAccess, tgt ocmcpi.Repository, handler transferhandler.TransferHandler) error { nv := common.VersionedElementKey(src) log = log.WithValues("history", state.History.String(), "version", nv) if ok, err := state.Add(ocm.KIND_COMPONENTVERSION, nv); !ok { diff --git a/pkg/docker/fetcher.go b/pkg/docker/fetcher.go index c1a2eb2f5..f2116163c 100644 --- a/pkg/docker/fetcher.go +++ b/pkg/docker/fetcher.go @@ -139,7 +139,11 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R } func (r dockerFetcher) open(ctx context.Context, req *request, mediatype string, offset int64) (_ io.ReadCloser, retErr error) { - req.header.Set("Accept", strings.Join([]string{mediatype, `*/*`}, ", ")) + mt := "*/*" + if mediatype != "" { + mt = mediatype + ", " + mt + } + req.header.Set("Accept", mt) if offset > 0 { // Note: "Accept-Ranges: bytes" cannot be trusted as some endpoints