From bffb1505674df31ead2443f0a9dd554cda799fc3 Mon Sep 17 00:00:00 2001 From: Balazs Nadasdi Date: Tue, 8 Nov 2022 12:45:57 +0100 Subject: [PATCH 1/2] fix: enable linter: maintidx Issues fixed: - Fix issues with auto fix (`--fix`). - Rename `this` receivers. - Fix language issues, most of the duplicated words like "the the". - Removed unnecessary type conversion. - Reduced complexity of the `_apply` function by extracting some parts into a separate function. - Reduced complexity of the `push` function by extracting some parts into a separate function. References: - https://github.com/open-component-model/ocm/issues/91 Closes #91 --- .golangci.yaml | 3 - cmds/demoplugin/accessmethods/demo.go | 7 +- cmds/demoplugin/uploaders/demo.go | 9 +- cmds/ocm/app/app.go | 4 +- .../ocicmds/artefacts/transfer/cmd.go | 2 +- cmds/ocm/commands/ocmcmds/common/addconfig.go | 2 +- .../ocmcmds/common/inputs/options/standard.go | 24 +- .../ocmcmds/common/inputs/types/spiff/type.go | 2 +- cmds/ocm/commands/ocmcmds/common/resources.go | 15 +- .../ocmcmds/references/add/provider.go | 9 +- cmds/ocm/pkg/processing/buffer.go | 288 +++++++++--------- cmds/ocm/pkg/template/template.go | 2 +- cmds/ocm/pkg/utils/command.go | 6 +- cmds/ocm/pkg/utils/handling.go | 2 +- cmds/ocm/topics/toi/bootstrapping/topic.go | 6 +- pkg/contexts/oci/ociutils/helm/ignore/doc.go | 2 +- .../ocm/accessmethods/github/method.go | 6 +- .../ocm/accessmethods/localblob/method.go | 6 +- .../ocm/accessmethods/localfsblob/method.go | 6 +- .../ocm/accessmethods/localociblob/method.go | 6 +- pkg/contexts/ocm/accessmethods/none/method.go | 6 +- .../ocm/accessmethods/ociartefact/method.go | 6 +- .../ocm/accessmethods/ociblob/method.go | 6 +- pkg/contexts/ocm/internal/uniform.go | 2 +- .../componentmapping/constants.go | 24 +- pkg/contexts/ocm/signing/handle.go | 152 ++++----- pkg/docker/pusher.go | 212 +++++++------ pkg/runtime/scheme.go | 2 +- pkg/runtime/unstructured.go | 2 +- .../handlers/rsa-signingservice/handler.go | 6 +- 30 files changed, 450 insertions(+), 375 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index fef17ab89b..5f76e434e9 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -47,9 +47,6 @@ linters: - nestif # Some nexted if statements are 8 or 9 deep. - dupl # Check code duplications. - cyclop # Complex functions are not good. - - maintidx # Too complex code is not good. - # name: push, Cyclomatic Complexity: 35 - # name: apply, Cyclomatic Complexity: 51 - gochecknoinits # Init functions cause an import to have side effects, # and side effects are hard to test, # reduce readability and increase the complexity of code. diff --git a/cmds/demoplugin/accessmethods/demo.go b/cmds/demoplugin/accessmethods/demo.go index 9d9b8accda..0d7008ae06 100644 --- a/cmds/demoplugin/accessmethods/demo.go +++ b/cmds/demoplugin/accessmethods/demo.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/mandelsoft/filepath/pkg/filepath" + "github.com/open-component-model/ocm/cmds/common" "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/oci/identity" @@ -18,8 +19,10 @@ import ( "github.com/open-component-model/ocm/pkg/runtime" ) -const NAME = "demo" -const VERSION = "v1" +const ( + NAME = "demo" + VERSION = "v1" +) type AccessSpec struct { runtime.ObjectVersionedType `json:",inline"` diff --git a/cmds/demoplugin/uploaders/demo.go b/cmds/demoplugin/uploaders/demo.go index 1751ea4707..6e013c2f5d 100644 --- a/cmds/demoplugin/uploaders/demo.go +++ b/cmds/demoplugin/uploaders/demo.go @@ -12,16 +12,17 @@ import ( "strings" "github.com/open-component-model/ocm/cmds/common" + "github.com/open-component-model/ocm/cmds/demoplugin/accessmethods" "github.com/open-component-model/ocm/pkg/contexts/credentials" "github.com/open-component-model/ocm/pkg/contexts/oci/identity" "github.com/open-component-model/ocm/pkg/contexts/ocm/plugin/ppi" "github.com/open-component-model/ocm/pkg/runtime" - - "github.com/open-component-model/ocm/cmds/demoplugin/accessmethods" ) -const NAME = "demo" -const VERSION = "v1" +const ( + NAME = "demo" + VERSION = "v1" +) type TargetSpec struct { runtime.ObjectVersionedType `json:",inline"` diff --git a/cmds/ocm/app/app.go b/cmds/ocm/app/app.go index 6cb2520f50..d34fd53870 100644 --- a/cmds/ocm/app/app.go +++ b/cmds/ocm/app/app.go @@ -317,7 +317,7 @@ func (o *CLIOptions) Complete() error { return errors.Wrapf(err, "attribute %s", s.Name) } } - err = ctx.ApplyConfig(spec, "cli") + _ = ctx.ApplyConfig(spec, "cli") } return plugincacheattr.Get(o.Context.OCMContext()).RegisterExtensions() } @@ -327,7 +327,7 @@ func NewVersionCommand(ctx clictx.Context) *cobra.Command { Use: "version", Aliases: []string{"v"}, Short: "displays the version", - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { v := version.Get() out.Outf(ctx, "%#v\n", v) }, diff --git a/cmds/ocm/commands/ocicmds/artefacts/transfer/cmd.go b/cmds/ocm/commands/ocicmds/artefacts/transfer/cmd.go index faa309b8ac..dd15021e6f 100644 --- a/cmds/ocm/commands/ocicmds/artefacts/transfer/cmd.go +++ b/cmds/ocm/commands/ocicmds/artefacts/transfer/cmd.go @@ -137,7 +137,7 @@ func NewAction(ctx clictx.Context, session oci.Session, target string, transferR } if ref.IsVersion() && transferRepo { - return nil, errors.Newf("repository names cannot be transferred for for a given target version") + return nil, errors.Newf("repository names cannot be transferred for a given target version") } if ref.IsRegistry() { transferRepo = true diff --git a/cmds/ocm/commands/ocmcmds/common/addconfig.go b/cmds/ocm/commands/ocmcmds/common/addconfig.go index dc738225d2..255f52f80b 100644 --- a/cmds/ocm/commands/ocmcmds/common/addconfig.go +++ b/cmds/ocm/commands/ocmcmds/common/addconfig.go @@ -129,7 +129,7 @@ func (o *ResourceConfigAdderCommand) ProcessResourceDescriptions(listkey string, } r = string(b) } - current += "\n---\n" + string(r) + current += "\n---\n" + r } source := NewModifiedResourceSpecificationsFile(current, o.ConfigFile, fs) diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/options/standard.go b/cmds/ocm/commands/ocmcmds/common/inputs/options/standard.go index ff27a224d0..8c65dab787 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/options/standard.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/options/standard.go @@ -9,19 +9,27 @@ import ( "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/options" ) -var HintOption = options.HintOption -var MediaTypeOption = options.MediatypeOption +var ( + HintOption = options.HintOption + MediaTypeOption = options.MediatypeOption +) var PathOption = flagsets.NewStringOptionType("inputPath", "path field for input") -var CompressOption = flagsets.NewBoolOptionType("inputCompress", "compress option for input") -var ExcludeOption = flagsets.NewStringArrayOptionType("inputExcludes", "excludes (path) for inputs") +var ( + CompressOption = flagsets.NewBoolOptionType("inputCompress", "compress option for input") + ExcludeOption = flagsets.NewStringArrayOptionType("inputExcludes", "excludes (path) for inputs") +) -var IncludeOption = flagsets.NewStringArrayOptionType("inputIncludes", "includes (path) for inputs") -var PreserveDirOption = flagsets.NewBoolOptionType("inputPreserveDir", "preserve directory in archive for inputs") +var ( + IncludeOption = flagsets.NewStringArrayOptionType("inputIncludes", "includes (path) for inputs") + PreserveDirOption = flagsets.NewBoolOptionType("inputPreserveDir", "preserve directory in archive for inputs") +) -var FollowSymlinksOption = flagsets.NewBoolOptionType("inputFollowSymlinks", "follow symbolic links during archive creation for inputs") -var VariantsOption = flagsets.NewStringArrayOptionType("inputVariants", "(platform) variants for inputs") +var ( + FollowSymlinksOption = flagsets.NewBoolOptionType("inputFollowSymlinks", "follow symbolic links during archive creation for inputs") + VariantsOption = flagsets.NewStringArrayOptionType("inputVariants", "(platform) variants for inputs") +) var LibrariesOption = flagsets.NewStringArrayOptionType("inputLibraries", "library path for inputs") diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/spiff/type.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/spiff/type.go index 96ea941b80..a6a1551312 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/spiff/type.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/spiff/type.go @@ -16,7 +16,7 @@ func init() { } func usage() string { - return file.Usage("The path must denote a [spiff](https://github.com/mandelsoft/spiff) template relative the the resources file.") + ` + return file.Usage("The path must denote a [spiff](https://github.com/mandelsoft/spiff) template relative the resources file.") + ` - **values** *map[string]any* This OPTIONAL property describes an additional value binding for the template processing. It will be available diff --git a/cmds/ocm/commands/ocmcmds/common/resources.go b/cmds/ocm/commands/ocmcmds/common/resources.go index 0bfd0660d5..b7f57bcdc7 100644 --- a/cmds/ocm/commands/ocmcmds/common/resources.go +++ b/cmds/ocm/commands/ocmcmds/common/resources.go @@ -233,8 +233,10 @@ type ContentResourceSpecificationsProvider struct { options flagsets.ConfigOptions } -var _ ResourceSpecificationsProvider = (*ContentResourceSpecificationsProvider)(nil) -var _ ResourceSpecifications = (*ContentResourceSpecificationsProvider)(nil) +var ( + _ ResourceSpecificationsProvider = (*ContentResourceSpecificationsProvider)(nil) + _ ResourceSpecifications = (*ContentResourceSpecificationsProvider)(nil) +) func NewContentResourceSpecificationProvider(ctx clictx.Context, name string, adder flagsets.ConfigAdder, deftype string, types ...flagsets.ConfigOptionType) *ContentResourceSpecificationsProvider { a := &ContentResourceSpecificationsProvider{ @@ -354,7 +356,8 @@ func (a *ContentResourceSpecificationsProvider) Get() (string, error) { return "", err } - r, err := json.Marshal(data) + //nolint:errchkjson // We don't care about this error. + r, _ := json.Marshal(data) return string(r), nil } @@ -442,12 +445,12 @@ func (o *ResourceAdderCommand) ProcessResourceDescriptions(listkey string, h Res defer obj.Close() for _, r := range resources { - ictx := ictx.Section("adding %s...", r.Spec().Info()) + isctx := ictx.Section("adding %s...", r.Spec().Info()) if h.RequireInputs() { if r.input.Input != nil { var acc ocm.AccessSpec // Local Blob - blob, hint, berr := r.input.Input.GetBlob(ictx, common.VersionedElementKey(obj), r.path) + blob, hint, berr := r.input.Input.GetBlob(isctx, common.VersionedElementKey(obj), r.path) if berr != nil { return errors.Wrapf(berr, "cannot get resource blob for %q(%s)", r.spec.GetName(), r.source) } @@ -478,7 +481,7 @@ func determineResources(printer common.Printer, ctx clictx.Context, ictx inputs. if err != nil { return nil, err } - parsed, err := templ.Execute(string(r)) + parsed, err := templ.Execute(r) if err != nil { return nil, errors.Wrapf(err, "error during variable substitution") } diff --git a/cmds/ocm/commands/ocmcmds/references/add/provider.go b/cmds/ocm/commands/ocmcmds/references/add/provider.go index 3230f3a1fc..f826fb287b 100644 --- a/cmds/ocm/commands/ocmcmds/references/add/provider.go +++ b/cmds/ocm/commands/ocmcmds/references/add/provider.go @@ -15,8 +15,10 @@ type ReferenceResourceSpecificationProvider struct { *ocmcomm.ResourceMetaDataSpecificationsProvider } -var _ ocmcomm.ResourceSpecificationsProvider = (*ReferenceResourceSpecificationProvider)(nil) -var _ ocmcomm.ResourceSpecifications = (*ReferenceResourceSpecificationProvider)(nil) +var ( + _ ocmcomm.ResourceSpecificationsProvider = (*ReferenceResourceSpecificationProvider)(nil) + _ ocmcomm.ResourceSpecifications = (*ReferenceResourceSpecificationProvider)(nil) +) func NewReferenceSpecificatonProvider() ocmcomm.ResourceSpecificationsProvider { a := &ReferenceResourceSpecificationProvider{ @@ -47,7 +49,8 @@ func (a *ReferenceResourceSpecificationProvider) Get() (string, error) { return "", err } - r, err := json.Marshal(data) + //nolint:errchkjson // We don't care about this error. + r, _ := json.Marshal(data) return string(r), nil } diff --git a/cmds/ocm/pkg/processing/buffer.go b/cmds/ocm/pkg/processing/buffer.go index bba9f63913..7fe2d9e810 100644 --- a/cmds/ocm/pkg/processing/buffer.go +++ b/cmds/ocm/pkg/processing/buffer.go @@ -161,54 +161,54 @@ func NewProcessingBuffer(log logging.Context, i BufferImplementation) Processing return (&_buffer{}).new(log, i) } -func (this *_buffer) new(log logging.Context, i BufferImplementation) *_buffer { - this.BufferImplementation = i - this.Cond = sync.NewCond(&this.Mutex) - this.complete = atomic.NewBool(false) - i.SetFrame(this) - this.log = log - return this -} - -func (this *_buffer) Add(e ProcessingEntry) ProcessingBuffer { - this.Lock() - notify := this.BufferImplementation.Add(e) - this.Unlock() +func (b *_buffer) new(log logging.Context, i BufferImplementation) *_buffer { + b.BufferImplementation = i + b.Cond = sync.NewCond(&b.Mutex) + b.complete = atomic.NewBool(false) + i.SetFrame(b) + b.log = log + return b +} + +func (b *_buffer) Add(e ProcessingEntry) ProcessingBuffer { + b.Lock() + notify := b.BufferImplementation.Add(e) + b.Unlock() if notify { - this.Broadcast() + b.Broadcast() } - return this + return b } -func (this *_buffer) Open() { - this.Lock() - this.BufferImplementation.Open() - this.complete.Unset() - this.Unlock() +func (b *_buffer) Open() { + b.Lock() + b.BufferImplementation.Open() + b.complete.Unset() + b.Unlock() } -func (this *_buffer) Close() { - this.Lock() - this.BufferImplementation.Close() - this.complete.Set() - this.Unlock() - this.Broadcast() +func (b *_buffer) Close() { + b.Lock() + b.BufferImplementation.Close() + b.complete.Set() + b.Unlock() + b.Broadcast() } -func (this *_buffer) IsClosed() bool { - return this.complete.IsSet() +func (b *_buffer) IsClosed() bool { + return b.complete.IsSet() } -func (this *_buffer) Len() int { - this.Lock() - defer this.Unlock() - return this.BufferImplementation.Len() +func (b *_buffer) Len() int { + b.Lock() + defer b.Unlock() + return b.BufferImplementation.Len() } -func (this *_buffer) Get(i int) interface{} { - this.Lock() - defer this.Unlock() - return this.BufferImplementation.Get(i) +func (b *_buffer) Get(i int) interface{} { + b.Lock() + defer b.Unlock() + return b.BufferImplementation.Get(i) } //////////////////////////////////////////////////////////////////////////////// @@ -223,41 +223,41 @@ func NewSimpleBuffer(log logging.Context) ProcessingBuffer { return NewProcessingBuffer(log, (&simpleBuffer{}).new(log)) } -func (this *simpleBuffer) new(log logging.Context) *simpleBuffer { - this.entries = []ProcessingEntry{} - this.log = log - return this +func (sb *simpleBuffer) new(log logging.Context) *simpleBuffer { + sb.entries = []ProcessingEntry{} + sb.log = log + return sb } -func (this *simpleBuffer) SetFrame(frame BufferFrame) { - this.frame = frame +func (sb *simpleBuffer) SetFrame(frame BufferFrame) { + sb.frame = frame } -func (this *simpleBuffer) Open() { +func (sb *simpleBuffer) Open() { } -func (this *simpleBuffer) Close() { +func (sb *simpleBuffer) Close() { } -func (this *simpleBuffer) Iterator() data.Iterator { - return (&simpleBufferIterator{}).new(this, true, this.log) +func (sb *simpleBuffer) Iterator() data.Iterator { + return (&simpleBufferIterator{}).new(sb, true, sb.log) } -func (this *simpleBuffer) ProcessingIterator() ProcessingIterator { - return (&simpleBufferIterator{}).new(this, false, this.log) +func (sb *simpleBuffer) ProcessingIterator() ProcessingIterator { + return (&simpleBufferIterator{}).new(sb, false, sb.log) } -func (this *simpleBuffer) Add(e ProcessingEntry) bool { - this.entries = append(this.entries, e) +func (sb *simpleBuffer) Add(e ProcessingEntry) bool { + sb.entries = append(sb.entries, e) return true } -func (this *simpleBuffer) Len() int { - return len(this.entries) +func (sb *simpleBuffer) Len() int { + return len(sb.entries) } -func (this *simpleBuffer) Get(i int) interface{} { - e := this.entries[i] +func (sb *simpleBuffer) Get(i int) interface{} { + e := sb.entries[i] if e.Valid { return e.Value } @@ -276,53 +276,53 @@ var ( _ data.Iterator = &simpleBufferIterator{} ) -func (this *simpleBufferIterator) new(buffer *simpleBuffer, valid bool, log logging.Context) *simpleBufferIterator { - this.valid = valid - this.current = -1 - this.buffer = buffer - this.log = log - return this +func (sbi *simpleBufferIterator) new(buffer *simpleBuffer, valid bool, log logging.Context) *simpleBufferIterator { + sbi.valid = valid + sbi.current = -1 + sbi.buffer = buffer + sbi.log = log + return sbi } -func (this *simpleBufferIterator) HasNext() bool { - this.buffer.frame.Lock() - defer this.buffer.frame.Unlock() +func (sbi *simpleBufferIterator) HasNext() bool { + sbi.buffer.frame.Lock() + defer sbi.buffer.frame.Unlock() for { - this.log.Logger().Debug("HasNext", "current", this.current) - if len(this.buffer.entries) > this.current+1 { - if !this.valid || this.buffer.entries[this.current+1].Valid { + sbi.log.Logger().Debug("HasNext", "current", sbi.current) + if len(sbi.buffer.entries) > sbi.current+1 { + if !sbi.valid || sbi.buffer.entries[sbi.current+1].Valid { return true } - this.current++ + sbi.current++ continue } - if this.buffer.frame.IsClosed() { + if sbi.buffer.frame.IsClosed() { return false } - this.buffer.frame.Wait() + sbi.buffer.frame.Wait() } } -func (this *simpleBufferIterator) Next() interface{} { - return this.NextProcessingEntry().Value +func (sbi *simpleBufferIterator) Next() interface{} { + return sbi.NextProcessingEntry().Value } -func (this *simpleBufferIterator) NextProcessingEntry() ProcessingEntry { - this.buffer.frame.Lock() - defer this.buffer.frame.Unlock() +func (sbi *simpleBufferIterator) NextProcessingEntry() ProcessingEntry { + sbi.buffer.frame.Lock() + defer sbi.buffer.frame.Unlock() for { - this.log.Logger().Debug("NextProcessingEntry", "current", this.current) - if len(this.buffer.entries) > this.current+1 { - this.current++ - if !this.valid || this.buffer.entries[this.current].Valid { - return this.buffer.entries[this.current] + sbi.log.Logger().Debug("NextProcessingEntry", "current", sbi.current) + if len(sbi.buffer.entries) > sbi.current+1 { + sbi.current++ + if !sbi.valid || sbi.buffer.entries[sbi.current].Valid { + return sbi.buffer.entries[sbi.current] } continue } - if this.buffer.frame.IsClosed() { + if sbi.buffer.frame.IsClosed() { return ProcessingEntry{} } - this.buffer.frame.Wait() + sbi.buffer.frame.Wait() } } @@ -350,26 +350,26 @@ func NewOrderedBuffer(log logging.Context) ProcessingBuffer { return NewProcessingBuffer(log, (&orderedBuffer{}).new(log)) } -func (this *orderedBuffer) new(log logging.Context) *orderedBuffer { - (&this.simple).new(log) - this.root.New(this) - this.valid = this.root.DLL() - this.last = this.valid - this.nextIndex = this.nextIndex.Next(-1, 0) - this.log = log - return this +func (ob *orderedBuffer) new(log logging.Context) *orderedBuffer { + (&ob.simple).new(log) + ob.root.New(ob) + ob.valid = ob.root.DLL() + ob.last = ob.valid + ob.nextIndex = ob.nextIndex.Next(-1, 0) + ob.log = log + return ob } -func (this *orderedBuffer) SetFrame(frame BufferFrame) { - this.simple.SetFrame(frame) +func (ob *orderedBuffer) SetFrame(frame BufferFrame) { + ob.simple.SetFrame(frame) } -func (this *orderedBuffer) Add(e ProcessingEntry) bool { +func (ob *orderedBuffer) Add(e ProcessingEntry) bool { e.Index.Validate(e.MaxIndex) - this.simple.Add(e) + ob.simple.Add(e) n := data.NewDLL(&e) - c := this.root.DLL() + c := ob.root.DLL() i := c.Next() for i != nil { v := i.Get().(*ProcessingEntry) @@ -379,54 +379,52 @@ func (this *orderedBuffer) Add(e ProcessingEntry) bool { c, i = i, i.Next() } c.Append(n) - this.size++ + ob.size++ if n.Next() == nil { - this.last = n + ob.last = n } increased := false - this.log.Logger().Debug("add index to cur value", "index", e.Index, "value", e.Value, "next-index", this.nextIndex) + ob.log.Logger().Debug("add index to cur value", "index", e.Index, "value", e.Value, "next-index", ob.nextIndex) - next := this.valid.Next() - for next != nil && !next.Get().(*ProcessingEntry).Index.After(this.nextIndex) { + next := ob.valid.Next() + for next != nil && !next.Get().(*ProcessingEntry).Index.After(ob.nextIndex) { n := next.Get().(*ProcessingEntry) - this.nextIndex = n.Index.Next(n.MaxIndex, n.MaxSub) - this.valid = next + ob.nextIndex = n.Index.Next(n.MaxIndex, n.MaxSub) + ob.valid = next next = next.Next() increased = true - this.log.Logger().Debug("increase to index to value", "index", n.Index, "value", n.Value) + ob.log.Logger().Debug("increase to index to value", "index", n.Index, "value", n.Value) } return increased } -func (this *orderedBuffer) Close() { - this.simple.Close() - if this.valid != this.last { - this.valid = this.last - this.nextIndex = this.valid.Get().(*ProcessingEntry).Index +func (ob *orderedBuffer) Close() { + ob.simple.Close() + if ob.valid != ob.last { + ob.valid = ob.last + ob.nextIndex = ob.valid.Get().(*ProcessingEntry).Index } } -func (this *orderedBuffer) Open() { - this.simple.Open() +func (ob *orderedBuffer) Open() { + ob.simple.Open() } -func (this *orderedBuffer) Iterator() data.Iterator { - // this this is another this than this in iter() in this.container - // still inherited to offer the unordered entries for processing - return (&orderedBufferIterator{}).new(this) +func (ob *orderedBuffer) Iterator() data.Iterator { + return (&orderedBufferIterator{}).new(ob) } -func (this *orderedBuffer) ProcessingIterator() ProcessingIterator { - return this.simple.ProcessingIterator() +func (ob *orderedBuffer) ProcessingIterator() ProcessingIterator { + return ob.simple.ProcessingIterator() } -func (this *orderedBuffer) Len() int { - return this.size +func (ob *orderedBuffer) Len() int { + return ob.size } -func (this *orderedBuffer) Get(i int) interface{} { - e := this.root.DLL() +func (ob *orderedBuffer) Get(i int) interface{} { + e := ob.root.DLL() for e != nil && i >= 0 { e = e.Next() i-- @@ -448,64 +446,64 @@ type orderedBufferIterator struct { var _ data.Iterator = (*orderedBufferIterator)(nil) -func (this *orderedBufferIterator) new(buffer *orderedBuffer) *orderedBufferIterator { - this.buffer = buffer - this.current = this.buffer.root.DLL() - return this +func (obi *orderedBufferIterator) new(buffer *orderedBuffer) *orderedBufferIterator { + obi.buffer = buffer + obi.current = obi.buffer.root.DLL() + return obi } -func (this *orderedBufferIterator) HasNext() bool { - this.buffer.simple.frame.Lock() - defer this.buffer.simple.frame.Unlock() +func (obi *orderedBufferIterator) HasNext() bool { + obi.buffer.simple.frame.Lock() + defer obi.buffer.simple.frame.Unlock() for { - n := this.current.Next() - if n != nil && this.current != this.buffer.valid { + n := obi.current.Next() + if n != nil && obi.current != obi.buffer.valid { if n.Get().(*ProcessingEntry).Valid { return true } - this.current = n // skip invalid entries + obi.current = n // skip invalid entries continue } - if this.buffer.simple.frame.IsClosed() { + if obi.buffer.simple.frame.IsClosed() { return false } - this.buffer.simple.frame.Wait() + obi.buffer.simple.frame.Wait() } } -func (this *orderedBufferIterator) CheckNext() bool { - this.buffer.simple.frame.Lock() - defer this.buffer.simple.frame.Unlock() +func (obi *orderedBufferIterator) CheckNext() bool { + obi.buffer.simple.frame.Lock() + defer obi.buffer.simple.frame.Unlock() for { - n := this.current.Next() - if n != nil && this.current != this.buffer.valid { + n := obi.current.Next() + if n != nil && obi.current != obi.buffer.valid { if n.Get().(*ProcessingEntry).Valid { return true } - this.current = n // skip invalid entries + obi.current = n // skip invalid entries continue } return false } } -func (this *orderedBufferIterator) Next() interface{} { - this.buffer.simple.frame.Lock() - defer this.buffer.simple.frame.Unlock() +func (obi *orderedBufferIterator) Next() interface{} { + obi.buffer.simple.frame.Lock() + defer obi.buffer.simple.frame.Unlock() for { - n := this.current.Next() - if n != nil && this.current != this.buffer.valid { + n := obi.current.Next() + if n != nil && obi.current != obi.buffer.valid { e := n.Get().(*ProcessingEntry) - this.current = n // always proceed + obi.current = n // always proceed if e.Valid { return e.Value } continue } - if this.buffer.simple.frame.IsClosed() { + if obi.buffer.simple.frame.IsClosed() { return ProcessingEntry{} } - this.buffer.simple.frame.Wait() + obi.buffer.simple.frame.Wait() } } diff --git a/cmds/ocm/pkg/template/template.go b/cmds/ocm/pkg/template/template.go index 2b66aa8523..a830306e3a 100644 --- a/cmds/ocm/pkg/template/template.go +++ b/cmds/ocm/pkg/template/template.go @@ -180,7 +180,7 @@ func ReadSimpleSettings(fs vfs.FileSystem, path string) (map[string]string, erro } func SplitYamlDocuments(data []byte) ([][]byte, error) { - decoder := yaml.NewDecoder(bytes.NewBuffer([]byte(data))) + decoder := yaml.NewDecoder(bytes.NewBuffer(data)) list := [][]byte{} i := 0 for { diff --git a/cmds/ocm/pkg/utils/command.go b/cmds/ocm/pkg/utils/command.go index d6d5e82bd8..b2e3ac2bbb 100644 --- a/cmds/ocm/pkg/utils/command.go +++ b/cmds/ocm/pkg/utils/command.go @@ -204,10 +204,8 @@ func SetupCommand(ocmcmd OCMCommand, names ...string) *cobra.Command { if cmd != root { if root.PersistentPreRunE != nil { root.PersistentPreRunE(cmd, args) - } else { - if root.PersistentPreRun != nil { - root.PersistentPreRun(cmd, args) - } + } else if root.PersistentPreRun != nil { + root.PersistentPreRun(cmd, args) } } break diff --git a/cmds/ocm/pkg/utils/handling.go b/cmds/ocm/pkg/utils/handling.go index e9c9cc6c14..1d77ce311b 100644 --- a/cmds/ocm/pkg/utils/handling.go +++ b/cmds/ocm/pkg/utils/handling.go @@ -30,7 +30,7 @@ func (s StringSpec) String() string { type TypeHandler interface { // All returns all elements according to its context All() ([]output.Object, error) - // Get returns the the elements for a dedicated specification + // Get returns the elements for a dedicated specification // according to the handlers context. Get(name ElemSpec) ([]output.Object, error) Close() error diff --git a/cmds/ocm/topics/toi/bootstrapping/topic.go b/cmds/ocm/topics/toi/bootstrapping/topic.go index b2a4febed3..1f2cb78e28 100644 --- a/cmds/ocm/topics/toi/bootstrapping/topic.go +++ b/cmds/ocm/topics/toi/bootstrapping/topic.go @@ -210,7 +210,7 @@ that contains the resource reference. It uses the following fields: #### *Identity* An identity specification is a map[string]string. It describes -the identity attributes of a desired resource in a a component version. +the identity attributes of a desired resource in a component version. It always has at least one identity attribute name, which is the resource name field of the desired resource. If this resource defines additional identity attributes, the complete set must be specified. @@ -254,7 +254,7 @@ It has the following format: Here the executor may request the provisioning of some credentials with a dedicated name/purpose and structure. If specified it will be propagated to a using package. It this uses an own credentials section, this one - will be filtered and checked for the the actual executor. + will be filtered and checked for the actual executor. - **outputs** (optional) *map[string]OutputSpecification* @@ -332,7 +332,7 @@ execution and reading provided executor outputs after the execution. / └── toi ├── inputs - │   ├── config config info from package specification + │   ├── config info from package specification │   ├── ocmrepo OCM filesystem repository containing the complete │   │ component version of the package │   └── parameters merged complete parameter file diff --git a/pkg/contexts/oci/ociutils/helm/ignore/doc.go b/pkg/contexts/oci/ociutils/helm/ignore/doc.go index 289d7713fa..9f27c143bd 100644 --- a/pkg/contexts/oci/ociutils/helm/ignore/doc.go +++ b/pkg/contexts/oci/ociutils/helm/ignore/doc.go @@ -28,7 +28,7 @@ The formatting rules are as follows: Example: - # Match any file named foo.txt + # Match exact file name foo.txt # Match any text file diff --git a/pkg/contexts/ocm/accessmethods/github/method.go b/pkg/contexts/ocm/accessmethods/github/method.go index b12a37e642..03ca81c9ba 100644 --- a/pkg/contexts/ocm/accessmethods/github/method.go +++ b/pkg/contexts/ocm/accessmethods/github/method.go @@ -31,8 +31,10 @@ import ( ) // Type is the access type of GitHub registry. -const Type = "gitHub" -const TypeV1 = Type + runtime.VersionSeparator + "v1" +const ( + Type = "gitHub" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) const ( LegacyType = "github" diff --git a/pkg/contexts/ocm/accessmethods/localblob/method.go b/pkg/contexts/ocm/accessmethods/localblob/method.go index 09f33326f1..74d654020b 100644 --- a/pkg/contexts/ocm/accessmethods/localblob/method.go +++ b/pkg/contexts/ocm/accessmethods/localblob/method.go @@ -14,8 +14,10 @@ import ( ) // Type is the access type of a blob local to a component. -const Type = "localBlob" -const TypeV1 = Type + runtime.VersionSeparator + "v1" +const ( + Type = "localBlob" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) func init() { cpi.RegisterAccessType(cpi.NewConvertedAccessSpecType(Type, LocalBlobV1, cpi.WithDescription(usage))) diff --git a/pkg/contexts/ocm/accessmethods/localfsblob/method.go b/pkg/contexts/ocm/accessmethods/localfsblob/method.go index 18ff12e5b9..2ef2efc9f5 100644 --- a/pkg/contexts/ocm/accessmethods/localfsblob/method.go +++ b/pkg/contexts/ocm/accessmethods/localfsblob/method.go @@ -11,8 +11,10 @@ import ( ) // Type is the access type of a blob in a local filesystem. -const Type = "localFilesystemBlob" -const TypeV1 = Type + "/v1" +const ( + Type = "localFilesystemBlob" + TypeV1 = Type + "/v1" +) // Keep old access method and map generic one to this implementation for component archives diff --git a/pkg/contexts/ocm/accessmethods/localociblob/method.go b/pkg/contexts/ocm/accessmethods/localociblob/method.go index 569af95742..8c234fce18 100644 --- a/pkg/contexts/ocm/accessmethods/localociblob/method.go +++ b/pkg/contexts/ocm/accessmethods/localociblob/method.go @@ -14,8 +14,10 @@ import ( ) // Type is the access type for a component version local blob in an OCI repository. -const Type = "localOciBlob" -const TypeV1 = Type + runtime.VersionSeparator + "v1" +const ( + Type = "localOciBlob" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) func init() { cpi.RegisterAccessType(cpi.NewAccessSpecType(Type, &AccessSpec{})) diff --git a/pkg/contexts/ocm/accessmethods/none/method.go b/pkg/contexts/ocm/accessmethods/none/method.go index 1c9d2031e9..5e37a45876 100644 --- a/pkg/contexts/ocm/accessmethods/none/method.go +++ b/pkg/contexts/ocm/accessmethods/none/method.go @@ -13,8 +13,10 @@ import ( ) // Type is the access type for no blob. -const Type = "none" -const TypeV1 = Type + runtime.VersionSeparator + "v1" +const ( + Type = "none" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) func init() { cpi.RegisterAccessType(cpi.NewAccessSpecType(Type, &AccessSpec{}, cpi.WithDescription("dummy resource with no access"))) diff --git a/pkg/contexts/ocm/accessmethods/ociartefact/method.go b/pkg/contexts/ocm/accessmethods/ociartefact/method.go index f6cb07e277..ad8c6f1007 100644 --- a/pkg/contexts/ocm/accessmethods/ociartefact/method.go +++ b/pkg/contexts/ocm/accessmethods/ociartefact/method.go @@ -23,8 +23,10 @@ import ( ) // Type is the access type of a oci registry. -const Type = "ociArtefact" -const TypeV1 = Type + runtime.VersionSeparator + "v1" +const ( + Type = "ociArtefact" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) const ( LegacyType = "ociRegistry" diff --git a/pkg/contexts/ocm/accessmethods/ociblob/method.go b/pkg/contexts/ocm/accessmethods/ociblob/method.go index 46d5699e52..c7a6fe0d93 100644 --- a/pkg/contexts/ocm/accessmethods/ociblob/method.go +++ b/pkg/contexts/ocm/accessmethods/ociblob/method.go @@ -20,8 +20,10 @@ import ( ) // Type is the access type for a blob in an OCI repository. -const Type = "ociBlob" -const TypeV1 = Type + runtime.VersionSeparator + "v1" +const ( + Type = "ociBlob" + TypeV1 = Type + runtime.VersionSeparator + "v1" +) func init() { cpi.RegisterAccessType(cpi.NewAccessSpecType(Type, &AccessSpec{}, cpi.WithDescription(usage))) diff --git a/pkg/contexts/ocm/internal/uniform.go b/pkg/contexts/ocm/internal/uniform.go index c468631c27..018ab37715 100644 --- a/pkg/contexts/ocm/internal/uniform.go +++ b/pkg/contexts/ocm/internal/uniform.go @@ -19,7 +19,7 @@ const ( dockerHubLegacyDomain = "index.docker.io" ) -// UniformRepositorySpec is is generic specification of the repository +// UniformRepositorySpec is generic specification of the repository // for handling as part of standard references. type UniformRepositorySpec struct { // Type diff --git a/pkg/contexts/ocm/repositories/genericocireg/componentmapping/constants.go b/pkg/contexts/ocm/repositories/genericocireg/componentmapping/constants.go index 3769b40181..680a1592cd 100644 --- a/pkg/contexts/ocm/repositories/genericocireg/componentmapping/constants.go +++ b/pkg/contexts/ocm/repositories/genericocireg/componentmapping/constants.go @@ -18,18 +18,24 @@ const ComponentDescriptorTarMimeType = "application/vnd.ocm.software.component-d // LegacyComponentDescriptorTarMimeType is the legacy mimetype for component-descriptor-blobs // that are stored as tar. -const LegacyComponentDescriptorTarMimeType = "application/vnd.oci.gardener.cloud.cnudie.component-descriptor.config.v2+yaml+tar" -const Legacy2ComponentDescriptorTarMimeType = "application/vnd.gardener.cloud.cnudie.component-descriptor.v2+yaml+tar" +const ( + LegacyComponentDescriptorTarMimeType = "application/vnd.oci.gardener.cloud.cnudie.component-descriptor.config.v2+yaml+tar" + Legacy2ComponentDescriptorTarMimeType = "application/vnd.gardener.cloud.cnudie.component-descriptor.v2+yaml+tar" +) // ComponentDescriptorJSONMimeType is the mimetype for component-descriptor-blobs // that are stored as JSON. -const ComponentDescriptorJSONMimeType = "application/vnd.ocm.software.component-descriptor.v2+json" -const LegacyComponentDescriptorJSONMimeType = "application/vnd.gardener.cloud.cnudie.component-descriptor.v2+json" +const ( + ComponentDescriptorJSONMimeType = "application/vnd.ocm.software.component-descriptor.v2+json" + LegacyComponentDescriptorJSONMimeType = "application/vnd.gardener.cloud.cnudie.component-descriptor.v2+json" +) // ComponentDescriptorJSONMimeType is the mimetype for component-descriptor-blobs // that are stored as YAML. -const ComponentDescriptorYAMLMimeType = "application/vnd.ocm.software.component-descriptor.v2+yaml" -const LegacyComponentDescriptorYAMLMimeType = "application/vnd.gardener.cloud.cnudie.component-descriptor.v2+yaml" +const ( + ComponentDescriptorYAMLMimeType = "application/vnd.ocm.software.component-descriptor.v2+yaml" + LegacyComponentDescriptorYAMLMimeType = "application/vnd.gardener.cloud.cnudie.component-descriptor.v2+yaml" +) // ComponentDescriptorMimeType are the mimetypes for component-descriptor-blobs. var ComponentDescriptorMimeType = []string{ @@ -43,8 +49,10 @@ var ComponentDescriptorMimeType = []string{ const ComponentDescriptorConfigMimeType = "application/vnd.ocm.software.component.config.v1+json" // LegacyComponentDescriptorConfigMimeType is the mimetype for the legacy component-descriptor-oci-cfg-blobs. -const LegacyComponentDescriptorConfigMimeType = "application/vnd.gardener.cloud.cnudie.component.config.v1+json" -const Legacy2ComponentDescriptorConfigMimeType = "application/vnd.oci.gardener.cloud.cnudie.component-descriptor-metadata.config.v2+json" +const ( + LegacyComponentDescriptorConfigMimeType = "application/vnd.gardener.cloud.cnudie.component.config.v1+json" + Legacy2ComponentDescriptorConfigMimeType = "application/vnd.oci.gardener.cloud.cnudie.component-descriptor-metadata.config.v2+json" +) // ComponentDescriptorNamespace is the subpath for all component descriptor artifacts in an oci registry.‚. const ComponentDescriptorNamespace = "component-descriptors" diff --git a/pkg/contexts/ocm/signing/handle.go b/pkg/contexts/ocm/signing/handle.go index fc44cb39ba..793959d326 100644 --- a/pkg/contexts/ocm/signing/handle.go +++ b/pkg/contexts/ocm/signing/handle.go @@ -73,37 +73,8 @@ func _apply(printer common.Printer, state common.WalkingState, nv common.NameVer } } - for i, reference := range cd.References { - var calculatedDigest *metav1.DigestSpec - if reference.Digest == nil && !opts.DoUpdate() { - printer.Printf(" no digest given for reference %s", reference) - } - if reference.Digest == nil || opts.Recursively || opts.Verify { - nested, err := opts.Resolver.LookupComponentVersion(reference.GetComponentName(), reference.GetVersion()) - if err != nil { - return nil, errors.Wrapf(err, refMsg(reference, "failed resolving component reference")) - } - closer := accessio.OnceCloser(nested) - defer closer.Close() - digestOpts, err := opts.For(reference.Digest) - if err != nil { - return nil, errors.Wrapf(err, refMsg(reference, "failed resolving hasher for existing digest for component reference")) - } - calculatedDigest, err = apply(printer.AddGap(" "), state, nested, digestOpts, true) - if err != nil { - return nil, errors.Wrapf(err, refMsg(reference, "failed applying to component reference")) - } - } else { - printer.Printf(" accepting digest from reference %s", reference) - calculatedDigest = reference.Digest - } - - if reference.Digest == nil { - cd.References[i].Digest = calculatedDigest - } else if calculatedDigest != nil && !reflect.DeepEqual(reference.Digest, calculatedDigest) { - return nil, errors.Newf(refMsg(reference, "calculated reference digest (%+v) mismatches existing digest (%+v) for", calculatedDigest, reference.Digest)) - } - printer.Printf(" reference %d: %s:%s: digest %s\n", i, reference.ComponentName, reference.Version, calculatedDigest) + if err := calculateReferenceDigests(printer, cd, state, opts); err != nil { + return nil, err } blobdigesters := cv.GetContext().BlobDigesters() @@ -160,43 +131,8 @@ func _apply(printer common.Printer, state common.WalkingState, nv common.NameVer } if opts.DoVerify() { - found := []string{} - for _, n := range signatureNames { - f := cd.GetSignatureIndex(n) - if f < 0 { - continue - } - pub := opts.PublicKey(n) - if pub == nil { - if opts.SignatureConfigured(n) { - return nil, errors.ErrNotFound(compdesc.KIND_PUBLIC_KEY, n) - } - printer.Printf("Warning: no public key for signature %q in %s\n", n, state.History) - continue - } - sig := &cd.Signatures[f] - verifier := opts.Registry.GetVerifier(sig.Signature.Algorithm) - if verifier == nil { - if opts.SignatureConfigured(n) { - return nil, errors.ErrUnknown(compdesc.KIND_VERIFY_ALGORITHM, n) - } - printer.Printf("Warning: no verifier (%s) found for signature %q in %s\n", sig.Signature.Algorithm, n, state.History) - continue - } - hasher := opts.Registry.GetHasher(sig.Digest.HashAlgorithm) - if hasher == nil { - return nil, errors.ErrUnknown(compdesc.KIND_HASH_ALGORITHM, sig.Digest.HashAlgorithm) - } - err = verifier.Verify(sig.Digest.Value, hasher.Crypto(), sig.ConvertToSigning(), pub) - if err != nil { - return nil, errors.ErrInvalidWrap(err, compdesc.KIND_SIGNATURE, sig.Signature.Algorithm) - } - found = append(found, n) - } - if len(found) == 0 { - if !opts.DoSign() { - return nil, errors.Newf("no verifiable signature found") - } + if err := doVerify(printer, cd, state, signatureNames, opts); err != nil { + return nil, err } } @@ -255,3 +191,83 @@ func resMsg(ref *compdesc.Resource, acc string, msg string, args ...interface{}) } return fmt.Sprintf("%s %s:%s", fmt.Sprintf(msg, args...), ref.Name, ref.Version) } + +func doVerify(printer common.Printer, cd *compdesc.ComponentDescriptor, state common.WalkingState, signatureNames []string, opts *Options) error { + var err error + found := []string{} + for _, n := range signatureNames { + f := cd.GetSignatureIndex(n) + if f < 0 { + continue + } + pub := opts.PublicKey(n) + if pub == nil { + if opts.SignatureConfigured(n) { + return errors.ErrNotFound(compdesc.KIND_PUBLIC_KEY, n) + } + printer.Printf("Warning: no public key for signature %q in %s\n", n, state.History) + continue + } + sig := &cd.Signatures[f] + verifier := opts.Registry.GetVerifier(sig.Signature.Algorithm) + if verifier == nil { + if opts.SignatureConfigured(n) { + return errors.ErrUnknown(compdesc.KIND_VERIFY_ALGORITHM, n) + } + printer.Printf("Warning: no verifier (%s) found for signature %q in %s\n", sig.Signature.Algorithm, n, state.History) + continue + } + hasher := opts.Registry.GetHasher(sig.Digest.HashAlgorithm) + if hasher == nil { + return errors.ErrUnknown(compdesc.KIND_HASH_ALGORITHM, sig.Digest.HashAlgorithm) + } + err = verifier.Verify(sig.Digest.Value, hasher.Crypto(), sig.ConvertToSigning(), pub) + if err != nil { + return errors.ErrInvalidWrap(err, compdesc.KIND_SIGNATURE, sig.Signature.Algorithm) + } + found = append(found, n) + } + if len(found) == 0 { + if !opts.DoSign() { + return errors.Newf("no verifiable signature found") + } + } + + return nil +} + +func calculateReferenceDigests(printer common.Printer, cd *compdesc.ComponentDescriptor, state common.WalkingState, opts *Options) error { + for i, reference := range cd.References { + var calculatedDigest *metav1.DigestSpec + if reference.Digest == nil && !opts.DoUpdate() { + printer.Printf(" no digest given for reference %s", reference) + } + if reference.Digest == nil || opts.Recursively || opts.Verify { + nested, err := opts.Resolver.LookupComponentVersion(reference.GetComponentName(), reference.GetVersion()) + if err != nil { + return errors.Wrapf(err, refMsg(reference, "failed resolving component reference")) + } + closer := accessio.OnceCloser(nested) + defer closer.Close() + digestOpts, err := opts.For(reference.Digest) + if err != nil { + return errors.Wrapf(err, refMsg(reference, "failed resolving hasher for existing digest for component reference")) + } + calculatedDigest, err = apply(printer.AddGap(" "), state, nested, digestOpts, true) + if err != nil { + return errors.Wrapf(err, refMsg(reference, "failed applying to component reference")) + } + } else { + printer.Printf(" accepting digest from reference %s", reference) + calculatedDigest = reference.Digest + } + + if reference.Digest == nil { + cd.References[i].Digest = calculatedDigest + } else if calculatedDigest != nil && !reflect.DeepEqual(reference.Digest, calculatedDigest) { + return errors.Newf(refMsg(reference, "calculated reference digest (%+v) mismatches existing digest (%+v) for", calculatedDigest, reference.Digest)) + } + printer.Printf(" reference %d: %s:%s: digest %s\n", i, reference.ComponentName, reference.Version, calculatedDigest) + } + return nil +} diff --git a/pkg/docker/pusher.go b/pkg/docker/pusher.go index dfc06d4eb5..798028bd40 100644 --- a/pkg/docker/pusher.go +++ b/pkg/docker/pusher.go @@ -140,102 +140,15 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src res req = p.request(host, http.MethodPut, putPath...) req.header.Add("Content-Type", desc.MediaType) } else { - // Start upload request - req = p.request(host, http.MethodPost, "blobs", "uploads/") - - var resp *http.Response - if fromRepo := selectRepositoryMountCandidate(p.refspec, desc.Annotations); fromRepo != "" { - preq := requestWithMountFrom(req, desc.Digest.String(), fromRepo) - pctx := ContextWithAppendPullRepositoryScope(ctx, fromRepo) - - // NOTE: the fromRepo might be private repo and - // auth service still can grant token without error. - // but the post request will fail because of 401. - // - // for the private repo, we should remove mount-from - // query and send the request again. - resp, err = preq.doWithRetries(pctx, nil) - if err != nil { - return nil, err - } - - if resp.StatusCode == http.StatusUnauthorized { - log.G(ctx).Debugf("failed to mount from repository %s", fromRepo) - - resp.Body.Close() - resp = nil - } - } - - if resp == nil { - resp, err = req.doWithRetries(ctx, nil) - if err != nil { - return nil, err - } - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusOK, http.StatusAccepted, http.StatusNoContent: - case http.StatusCreated: - p.tracker.SetStatus(ref, Status{ - Committed: true, - Status: content.Status{ - Ref: ref, - Total: desc.Size, - Offset: desc.Size, - }, - }) - return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest) - default: - err := remoteserrors.NewUnexpectedStatusErr(resp) - - var statusError remoteserrors.ErrUnexpectedStatus - if errors.As(err, &statusError) { - log.G(ctx). - WithField("resp", resp). - WithField("body", string(statusError.Body)). - Debug("unexpected response") - } - + req, err = upload(ctx, uploadOptions{ + Host: host, + Pusher: p, + Descriptor: desc, + Reference: ref, + }) + if err != nil { return nil, err } - - var ( - location = resp.Header.Get("Location") - lurl *url.URL - lhost = host - ) - // Support paths without host in location - if strings.HasPrefix(location, "/") { - lurl, err = url.Parse(lhost.Scheme + "://" + lhost.Host + location) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse location %v", location) - } - } else { - if !strings.Contains(location, "://") { - location = lhost.Scheme + "://" + location - } - lurl, err = url.Parse(location) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse location %v", location) - } - - if lurl.Host != lhost.Host || lhost.Scheme != lurl.Scheme { - lhost.Scheme = lurl.Scheme - lhost.Host = lurl.Host - log.G(ctx).WithField("host", lhost.Host).WithField("scheme", lhost.Scheme).Debug("upload changed destination") - - // Strip authorizer if change to host or scheme - lhost.Authorizer = nil - } - } - q := lurl.Query() - q.Add("digest", desc.Digest.String()) - - req = p.request(lhost, http.MethodPut) - req.header.Set("Content-Type", "application/octet-stream") - req.path = lurl.Path + "?" + q.Encode() } p.tracker.SetStatus(ref, Status{ Status: content.Status{ @@ -425,3 +338,114 @@ func requestWithMountFrom(req *request, mount, from string) *request { return &creq } + +type uploadOptions struct { + Host RegistryHost + Pusher dockerPusher + Descriptor ocispec.Descriptor + Reference string +} + +func upload(ctx context.Context, opts uploadOptions) (*request, error) { + req := opts.Pusher.request(opts.Host, http.MethodPost, "blobs", "uploads/") + + var ( + resp *http.Response + err error + ) + if fromRepo := selectRepositoryMountCandidate(opts.Pusher.refspec, opts.Descriptor.Annotations); fromRepo != "" { + preq := requestWithMountFrom(req, opts.Descriptor.Digest.String(), fromRepo) + pctx := ContextWithAppendPullRepositoryScope(ctx, fromRepo) + + // NOTE: the fromRepo might be private repo and + // auth service still can grant token without error. + // but the post request will fail because of 401. + // + // for the private repo, we should remove mount-from + // query and send the request again. + resp, err = preq.doWithRetries(pctx, nil) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusUnauthorized { + log.G(ctx).Debugf("failed to mount from repository %s", fromRepo) + + resp.Body.Close() + resp = nil + } + } + + if resp == nil { + resp, err = req.doWithRetries(ctx, nil) + if err != nil { + return nil, err + } + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK, http.StatusAccepted, http.StatusNoContent: + case http.StatusCreated: + opts.Pusher.tracker.SetStatus(opts.Reference, Status{ + Committed: true, + Status: content.Status{ + Ref: opts.Reference, + Total: opts.Descriptor.Size, + Offset: opts.Descriptor.Size, + }, + }) + return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", opts.Descriptor.Digest) + default: + err := remoteserrors.NewUnexpectedStatusErr(resp) + + var statusError remoteserrors.ErrUnexpectedStatus + if errors.As(err, &statusError) { + log.G(ctx). + WithField("resp", resp). + WithField("body", string(statusError.Body)). + Debug("unexpected response") + } + + return nil, err + } + + var ( + location = resp.Header.Get("Location") + lurl *url.URL + lhost = opts.Host + ) + // Support paths without host in location + if strings.HasPrefix(location, "/") { + lurl, err = url.Parse(lhost.Scheme + "://" + lhost.Host + location) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse location %v", location) + } + } else { + if !strings.Contains(location, "://") { + location = lhost.Scheme + "://" + location + } + lurl, err = url.Parse(location) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse location %v", location) + } + + if lurl.Host != lhost.Host || lhost.Scheme != lurl.Scheme { + lhost.Scheme = lurl.Scheme + lhost.Host = lurl.Host + log.G(ctx).WithField("host", lhost.Host).WithField("scheme", lhost.Scheme).Debug("upload changed destination") + + // Strip authorizer if change to host or scheme + lhost.Authorizer = nil + } + } + + q := lurl.Query() + q.Add("digest", opts.Descriptor.Digest.String()) + + req = opts.Pusher.request(lhost, http.MethodPut) + req.header.Set("Content-Type", "application/octet-stream") + req.path = lurl.Path + "?" + q.Encode() + + return req, nil +} diff --git a/pkg/runtime/scheme.go b/pkg/runtime/scheme.go index 20dbd91801..bc2ee24e75 100644 --- a/pkg/runtime/scheme.go +++ b/pkg/runtime/scheme.go @@ -40,7 +40,7 @@ type TypedObjectDecoder interface { } // TypedObjectEncoder is able to provide a versioned representation of -// of an effective TypedObject. +// an effective TypedObject. type TypedObjectEncoder interface { Encode(TypedObject, Marshaler) ([]byte, error) } diff --git a/pkg/runtime/unstructured.go b/pkg/runtime/unstructured.go index 4416afd6e5..5cef379bb9 100644 --- a/pkg/runtime/unstructured.go +++ b/pkg/runtime/unstructured.go @@ -17,7 +17,7 @@ import ( const ATTR_TYPE = "type" -// ATTENTION: UnstructuredTypedObject CANNOT be be used as anonymous +// ATTENTION: UnstructuredTypedObject CANNOT be used as anonymous // field together with the default struct marshalling with the // great json marshallers. // Anonymous inline struct fields are always marshaled by the default struct diff --git a/pkg/signing/handlers/rsa-signingservice/handler.go b/pkg/signing/handlers/rsa-signingservice/handler.go index 0f0bfc55c6..ff64c43083 100644 --- a/pkg/signing/handlers/rsa-signingservice/handler.go +++ b/pkg/signing/handlers/rsa-signingservice/handler.go @@ -15,8 +15,10 @@ import ( ) // Algorithm defines the type for the RSA PKCS #1 v1.5 signature algorithm. -const Algorithm = rsa.Algorithm -const Name = "rsa-signingsservice" +const ( + Algorithm = rsa.Algorithm + Name = "rsa-signingsservice" +) type Key struct { URL string `json:"url"` From dad53628cdb15fc7f45ad9cf7dfb899cbfeea0b6 Mon Sep 17 00:00:00 2001 From: Balazs Nadasdi Date: Thu, 10 Nov 2022 11:07:11 +0100 Subject: [PATCH 2/2] address pr comments --- .golangci.yaml | 5 + cmds/ocm/topics/toi/bootstrapping/topic.go | 2 +- pkg/contexts/oci/ociutils/helm/ignore/doc.go | 2 +- .../ocm/accessmethods/localfsblob/method.go | 2 +- pkg/docker/pusher.go | 212 ++++++++---------- 5 files changed, 102 insertions(+), 121 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 5f76e434e9..fd3b4e4f66 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -6,6 +6,8 @@ run: issues-exit-code: 2 skip-dirs: - "hack" + # External code from containerd/containerd + - "pkg/docker" linters: enable-all: true @@ -167,3 +169,6 @@ issues: - source: "// .* #\\d+" linters: - godox + - path: ignore/.*\.go + linters: + - dupword diff --git a/cmds/ocm/topics/toi/bootstrapping/topic.go b/cmds/ocm/topics/toi/bootstrapping/topic.go index 1f2cb78e28..548a119a02 100644 --- a/cmds/ocm/topics/toi/bootstrapping/topic.go +++ b/cmds/ocm/topics/toi/bootstrapping/topic.go @@ -332,7 +332,7 @@ execution and reading provided executor outputs after the execution. / └── toi ├── inputs - │   ├── config info from package specification + │   ├── config configuration from package specification │   ├── ocmrepo OCM filesystem repository containing the complete │   │ component version of the package │   └── parameters merged complete parameter file diff --git a/pkg/contexts/oci/ociutils/helm/ignore/doc.go b/pkg/contexts/oci/ociutils/helm/ignore/doc.go index 9f27c143bd..289d7713fa 100644 --- a/pkg/contexts/oci/ociutils/helm/ignore/doc.go +++ b/pkg/contexts/oci/ociutils/helm/ignore/doc.go @@ -28,7 +28,7 @@ The formatting rules are as follows: Example: - # Match exact file name + # Match any file named foo.txt foo.txt # Match any text file diff --git a/pkg/contexts/ocm/accessmethods/localfsblob/method.go b/pkg/contexts/ocm/accessmethods/localfsblob/method.go index 2ef2efc9f5..8221f003fd 100644 --- a/pkg/contexts/ocm/accessmethods/localfsblob/method.go +++ b/pkg/contexts/ocm/accessmethods/localfsblob/method.go @@ -13,7 +13,7 @@ import ( // Type is the access type of a blob in a local filesystem. const ( Type = "localFilesystemBlob" - TypeV1 = Type + "/v1" + TypeV1 = Type + runtime.VersionSeparator + "v1" ) // Keep old access method and map generic one to this implementation for component archives diff --git a/pkg/docker/pusher.go b/pkg/docker/pusher.go index 798028bd40..dfc06d4eb5 100644 --- a/pkg/docker/pusher.go +++ b/pkg/docker/pusher.go @@ -140,15 +140,102 @@ func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, src res req = p.request(host, http.MethodPut, putPath...) req.header.Add("Content-Type", desc.MediaType) } else { - req, err = upload(ctx, uploadOptions{ - Host: host, - Pusher: p, - Descriptor: desc, - Reference: ref, - }) - if err != nil { + // Start upload request + req = p.request(host, http.MethodPost, "blobs", "uploads/") + + var resp *http.Response + if fromRepo := selectRepositoryMountCandidate(p.refspec, desc.Annotations); fromRepo != "" { + preq := requestWithMountFrom(req, desc.Digest.String(), fromRepo) + pctx := ContextWithAppendPullRepositoryScope(ctx, fromRepo) + + // NOTE: the fromRepo might be private repo and + // auth service still can grant token without error. + // but the post request will fail because of 401. + // + // for the private repo, we should remove mount-from + // query and send the request again. + resp, err = preq.doWithRetries(pctx, nil) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusUnauthorized { + log.G(ctx).Debugf("failed to mount from repository %s", fromRepo) + + resp.Body.Close() + resp = nil + } + } + + if resp == nil { + resp, err = req.doWithRetries(ctx, nil) + if err != nil { + return nil, err + } + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK, http.StatusAccepted, http.StatusNoContent: + case http.StatusCreated: + p.tracker.SetStatus(ref, Status{ + Committed: true, + Status: content.Status{ + Ref: ref, + Total: desc.Size, + Offset: desc.Size, + }, + }) + return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest) + default: + err := remoteserrors.NewUnexpectedStatusErr(resp) + + var statusError remoteserrors.ErrUnexpectedStatus + if errors.As(err, &statusError) { + log.G(ctx). + WithField("resp", resp). + WithField("body", string(statusError.Body)). + Debug("unexpected response") + } + return nil, err } + + var ( + location = resp.Header.Get("Location") + lurl *url.URL + lhost = host + ) + // Support paths without host in location + if strings.HasPrefix(location, "/") { + lurl, err = url.Parse(lhost.Scheme + "://" + lhost.Host + location) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse location %v", location) + } + } else { + if !strings.Contains(location, "://") { + location = lhost.Scheme + "://" + location + } + lurl, err = url.Parse(location) + if err != nil { + return nil, errors.Wrapf(err, "unable to parse location %v", location) + } + + if lurl.Host != lhost.Host || lhost.Scheme != lurl.Scheme { + lhost.Scheme = lurl.Scheme + lhost.Host = lurl.Host + log.G(ctx).WithField("host", lhost.Host).WithField("scheme", lhost.Scheme).Debug("upload changed destination") + + // Strip authorizer if change to host or scheme + lhost.Authorizer = nil + } + } + q := lurl.Query() + q.Add("digest", desc.Digest.String()) + + req = p.request(lhost, http.MethodPut) + req.header.Set("Content-Type", "application/octet-stream") + req.path = lurl.Path + "?" + q.Encode() } p.tracker.SetStatus(ref, Status{ Status: content.Status{ @@ -338,114 +425,3 @@ func requestWithMountFrom(req *request, mount, from string) *request { return &creq } - -type uploadOptions struct { - Host RegistryHost - Pusher dockerPusher - Descriptor ocispec.Descriptor - Reference string -} - -func upload(ctx context.Context, opts uploadOptions) (*request, error) { - req := opts.Pusher.request(opts.Host, http.MethodPost, "blobs", "uploads/") - - var ( - resp *http.Response - err error - ) - if fromRepo := selectRepositoryMountCandidate(opts.Pusher.refspec, opts.Descriptor.Annotations); fromRepo != "" { - preq := requestWithMountFrom(req, opts.Descriptor.Digest.String(), fromRepo) - pctx := ContextWithAppendPullRepositoryScope(ctx, fromRepo) - - // NOTE: the fromRepo might be private repo and - // auth service still can grant token without error. - // but the post request will fail because of 401. - // - // for the private repo, we should remove mount-from - // query and send the request again. - resp, err = preq.doWithRetries(pctx, nil) - if err != nil { - return nil, err - } - - if resp.StatusCode == http.StatusUnauthorized { - log.G(ctx).Debugf("failed to mount from repository %s", fromRepo) - - resp.Body.Close() - resp = nil - } - } - - if resp == nil { - resp, err = req.doWithRetries(ctx, nil) - if err != nil { - return nil, err - } - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusOK, http.StatusAccepted, http.StatusNoContent: - case http.StatusCreated: - opts.Pusher.tracker.SetStatus(opts.Reference, Status{ - Committed: true, - Status: content.Status{ - Ref: opts.Reference, - Total: opts.Descriptor.Size, - Offset: opts.Descriptor.Size, - }, - }) - return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", opts.Descriptor.Digest) - default: - err := remoteserrors.NewUnexpectedStatusErr(resp) - - var statusError remoteserrors.ErrUnexpectedStatus - if errors.As(err, &statusError) { - log.G(ctx). - WithField("resp", resp). - WithField("body", string(statusError.Body)). - Debug("unexpected response") - } - - return nil, err - } - - var ( - location = resp.Header.Get("Location") - lurl *url.URL - lhost = opts.Host - ) - // Support paths without host in location - if strings.HasPrefix(location, "/") { - lurl, err = url.Parse(lhost.Scheme + "://" + lhost.Host + location) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse location %v", location) - } - } else { - if !strings.Contains(location, "://") { - location = lhost.Scheme + "://" + location - } - lurl, err = url.Parse(location) - if err != nil { - return nil, errors.Wrapf(err, "unable to parse location %v", location) - } - - if lurl.Host != lhost.Host || lhost.Scheme != lurl.Scheme { - lhost.Scheme = lurl.Scheme - lhost.Host = lurl.Host - log.G(ctx).WithField("host", lhost.Host).WithField("scheme", lhost.Scheme).Debug("upload changed destination") - - // Strip authorizer if change to host or scheme - lhost.Authorizer = nil - } - } - - q := lurl.Query() - q.Add("digest", opts.Descriptor.Digest.String()) - - req = opts.Pusher.request(lhost, http.MethodPut) - req.header.Set("Content-Type", "application/octet-stream") - req.path = lurl.Path + "?" + q.Encode() - - return req, nil -}