From ef5dacd9ce431055cfdd750ea602a764efb3b5b3 Mon Sep 17 00:00:00 2001 From: Xiaoxuan Wang Date: Fri, 9 Aug 2024 17:07:12 +0800 Subject: [PATCH] rebase Signed-off-by: Xiaoxuan Wang --- cmd/oras/root/manifest/index/cmd.go | 1 + cmd/oras/root/manifest/index/update.go | 197 ++++++++++++++++++ .../e2e/internal/testdata/multi_arch/const.go | 6 + test/e2e/suite/command/manifest_index.go | 90 +++++--- 4 files changed, 269 insertions(+), 25 deletions(-) create mode 100644 cmd/oras/root/manifest/index/update.go diff --git a/cmd/oras/root/manifest/index/cmd.go b/cmd/oras/root/manifest/index/cmd.go index 54391c123..3633faa03 100644 --- a/cmd/oras/root/manifest/index/cmd.go +++ b/cmd/oras/root/manifest/index/cmd.go @@ -25,6 +25,7 @@ func Cmd() *cobra.Command { cmd.AddCommand( createCmd(), + updateCmd(), ) return cmd } diff --git a/cmd/oras/root/manifest/index/update.go b/cmd/oras/root/manifest/index/update.go new file mode 100644 index 000000000..64bdf0978 --- /dev/null +++ b/cmd/oras/root/manifest/index/update.go @@ -0,0 +1,197 @@ +/* +Copyright The ORAS Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package index + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/spf13/cobra" + "oras.land/oras-go/v2" + "oras.land/oras/cmd/oras/internal/argument" + "oras.land/oras/cmd/oras/internal/command" + oerrors "oras.land/oras/cmd/oras/internal/errors" + "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/descriptor" +) + +type updateOptions struct { + option.Common + option.Target + + extraRefs []string + addArguments []string + mergeArguments []string + removeArguments []string +} + +func updateCmd() *cobra.Command { + var opts updateOptions + cmd := &cobra.Command{ + Use: "update {:|@} {--add/--merge/--remove} {|} [...]", + Short: "[Experimental] Update and push an image index", + Long: `[Experimental] Update and push an image index. All manifests should be in the same repository + +Example - add one manifest and remove two manifests from an index tagged 'latest': + oras manifest index update localhost:5000/hello:latest --add sha256:xxx --remove sha256:xxx + +Example - remove a manifest and merge manifests from indexes tagged as 'index01' and 'index02': + oras manifest index update localhost:5000/hello:latest --remove sha256:xxx --merge index01 --merge index02 + `, + Args: oerrors.CheckArgs(argument.AtLeast(1), "the destination index to update"), + PreRunE: func(cmd *cobra.Command, args []string) error { + refs := strings.Split(args[0], ",") + opts.RawReference = refs[0] + opts.extraRefs = refs[1:] + if err := option.Parse(cmd, &opts); err != nil { + return err + } + // if a digest is given as the index reference, we need to ignore it to successfully push + opts.RawReference, _, _ = strings.Cut(opts.RawReference, "@") + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + return updateIndex(cmd, opts) + }, + } + option.ApplyFlags(&opts, cmd.Flags()) + cmd.Flags().StringArrayVarP(&opts.addArguments, "add", "", nil, "add manifests to the index") + cmd.Flags().StringArrayVarP(&opts.mergeArguments, "merge", "", nil, "merge the manifests of another index") + cmd.Flags().StringArrayVarP(&opts.removeArguments, "remove", "", nil, "manifests to remove from the index") + return oerrors.Command(cmd, &opts.Target) +} + +func updateIndex(cmd *cobra.Command, opts updateOptions) error { + // if no update flag is used, do nothing + if !cmd.Flags().Changed("add") && !cmd.Flags().Changed("remove") && !cmd.Flags().Changed("merge") { + opts.Println("No update flag is used. There's nothing to update.") + return nil + } + ctx, logger := command.GetLogger(cmd, &opts.Common) + target, err := opts.NewTarget(opts.Common, logger) + if err != nil { + return err + } + if err := opts.EnsureReferenceNotEmpty(cmd, true); err != nil { + return err + } + index, err := fetchIndex(ctx, target, opts) + if err != nil { + return err + } + manifests, err := addManifests(ctx, index.Manifests, target, opts) + if err != nil { + return err + } + manifests, err = mergeIndexes(ctx, manifests, target, opts) + if err != nil { + return err + } + manifests, err = removeManifests(ctx, manifests, target, opts) + if err != nil { + return err + } + desc, content, err := packIndex(&index, manifests) + if err != nil { + return err + } + opts.Println("Updated the index") + return pushIndex(ctx, target, desc, content, opts.Reference, opts.extraRefs, opts.AnnotatedReference(), opts.Printer) +} + +func fetchIndex(ctx context.Context, target oras.ReadOnlyTarget, opts updateOptions) (ocispec.Index, error) { + _, content, err := oras.FetchBytes(ctx, target, opts.Reference, oras.DefaultFetchBytesOptions) + if err != nil { + return ocispec.Index{}, fmt.Errorf("could not find the index %s: %w", opts.Reference, err) + } + opts.Println("Resolved manifest", opts.Reference) + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return ocispec.Index{}, err + } + return index, nil +} + +func addManifests(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, opts updateOptions) ([]ocispec.Descriptor, error) { + for _, manifest := range opts.addArguments { + desc, content, err := oras.FetchBytes(ctx, target, manifest, oras.DefaultFetchBytesOptions) + if err != nil { + return nil, fmt.Errorf("could not find the manifest %s: %w", manifest, err) + } + opts.Println("Resolved manifest", manifest) + if descriptor.IsImageManifest(desc) { + desc.Platform, err = getPlatform(ctx, target, content) + if err != nil { + return nil, err + } + } + manifests = append(manifests, desc) + } + return manifests, nil +} + +func mergeIndexes(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, opts updateOptions) ([]ocispec.Descriptor, error) { + for _, index := range opts.mergeArguments { + desc, content, err := oras.FetchBytes(ctx, target, index, oras.DefaultFetchBytesOptions) + if err != nil { + return nil, fmt.Errorf("could not find the index %s: %w", index, err) + } + if desc.MediaType != ocispec.MediaTypeImageIndex { + return nil, fmt.Errorf("%s is not an image index", index) + } + opts.Println("Resolved index", index) + var index ocispec.Index + if err := json.Unmarshal(content, &index); err != nil { + return nil, err + } + manifests = append(manifests, index.Manifests...) + } + return manifests, nil +} + +func removeManifests(ctx context.Context, manifests []ocispec.Descriptor, target oras.ReadOnlyTarget, opts updateOptions) ([]ocispec.Descriptor, error) { + set := make(map[digest.Digest]int) + for _, manifest := range opts.removeArguments { + desc, _, err := oras.FetchBytes(ctx, target, manifest, oras.DefaultFetchBytesOptions) + if err != nil { + return nil, fmt.Errorf("could not find the manifest %s: %w", manifest, err) + } + set[desc.Digest] = set[desc.Digest] + 1 + } + pointer := len(manifests) - 1 + for i := len(manifests) - 1; i >= 0; i-- { + if _, exists := set[manifests[i].Digest]; exists { + val := manifests[i] + // move the item to the end of the slice + for j := i; j < pointer; j++ { + manifests[j] = manifests[j+1] + } + manifests[pointer] = val + pointer = pointer - 1 + set[val.Digest] = set[val.Digest] - 1 + if set[val.Digest] == 0 { + delete(set, val.Digest) + } + } + } + // shrink the slice to remove the manifests + manifests = manifests[:pointer+1] + return manifests, nil +} diff --git a/test/e2e/internal/testdata/multi_arch/const.go b/test/e2e/internal/testdata/multi_arch/const.go index b7ebde0e8..889e5eb1a 100644 --- a/test/e2e/internal/testdata/multi_arch/const.go +++ b/test/e2e/internal/testdata/multi_arch/const.go @@ -78,4 +78,10 @@ var ( Digest: digest.Digest("sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c"), Size: 458, } + + LinuxARMV7 = ocispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: digest.Digest("sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255"), + Size: 458, + } ) diff --git a/test/e2e/suite/command/manifest_index.go b/test/e2e/suite/command/manifest_index.go index 4eea635b9..5344fe4c7 100644 --- a/test/e2e/suite/command/manifest_index.go +++ b/test/e2e/suite/command/manifest_index.go @@ -26,8 +26,14 @@ import ( var _ = Describe("ORAS beginners:", func() { When("running manifest index command", func() { When("running `manifest index create`", func() { - It("should show help doc with alias", func() { - ORAS("manifest", "index", "create", "--help").MatchKeyWords("Aliases", "pack").Exec() + It("should show help doc with feature flags", func() { + ORAS("manifest", "index", "create", "--help").MatchKeyWords(ExampleDesc).Exec() + }) + }) + + When("running `manifest index update`", func() { + It("should show help doc with add flag", func() { + ORAS("manifest", "index", "update", "--help").MatchKeyWords("--add stringArray").Exec() }) }) }) @@ -39,7 +45,7 @@ var _ = Describe("1.1 registry users:", func() { It("should create index by using source manifest digests", func() { ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "index-create-by-digest"), string(multi_arch.LinuxAMD64.Digest), string(multi_arch.LinuxARM64.Digest)). - MatchKeyWords("Fetched", "Pushed", + MatchKeyWords("Resolved manifest", "Pushed [registry]", "sha256:cce9590b1193d8bcb70467e2381dc81e77869be4801c09abe9bc274b6a1d2001").Exec() // verify ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "index-create-by-digest")). @@ -52,7 +58,7 @@ var _ = Describe("1.1 registry users:", func() { ORAS("tag", RegistryRef(ZOTHost, ImageRepo, string(multi_arch.LinuxARM64.Digest)), "arm64").Exec() ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "index-create-by-tag"), "arm64", "amd64"). - MatchKeyWords("Fetched", "Pushed", + MatchKeyWords("Resolved manifest", "Pushed [registry]", "sha256:5c98cfc90e390c575679370a5dc5e37b52e854bbb7b9cb80cc1f30b56b8d183e").Exec() // verify ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "index-create-by-tag")). @@ -65,7 +71,7 @@ var _ = Describe("1.1 registry users:", func() { ORAS("tag", RegistryRef(ZOTHost, ImageRepo, string(multi_arch.LinuxARM64.Digest)), "arm64").Exec() ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, ""), "arm64", "amd64", "sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255"). - MatchKeyWords("Fetched", "Pushed", + MatchKeyWords("Resolved manifest", "Pushed [registry]", "sha256:820503ae4fecfdb841b5b6acc8718c8c5b298cf6b8f2259010f370052341cec8").Exec() // verify ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "sha256:820503ae4fecfdb841b5b6acc8718c8c5b298cf6b8f2259010f370052341cec8")). @@ -78,7 +84,7 @@ var _ = Describe("1.1 registry users:", func() { ORAS("tag", RegistryRef(ZOTHost, ImageRepo, string(multi_arch.LinuxARM64.Digest)), "arm64").Exec() ORAS("manifest", "index", "create", fmt.Sprintf("%s,tag1,tag2,tag3", RegistryRef(ZOTHost, ImageRepo, "tag0")), "sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255", "arm64", "amd64"). - MatchKeyWords("Fetched", "Pushed", "Tagged", + MatchKeyWords("Resolved manifest", "Pushed [registry]", "Tagged", "sha256:bfa1728d6292d5fa7689f8f4daa145ee6f067b5779528c6e059d1132745ef508").Exec() // verify ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "tag0")).Exec() @@ -87,32 +93,66 @@ var _ = Describe("1.1 registry users:", func() { ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "tag3")).Exec() }) - It("should create nested indexes", func() { - ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "index-0"), - string(multi_arch.LinuxAMD64.Digest)). - MatchKeyWords("sha256:c543059818cb70e6442597a33454ec1e3d3a2bdb526c17875578d33c2ddcf72e").Exec() - ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "index-1"), - "index-0").Exec() - // verify - ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "index-1")). - MatchKeyWords("sha256:c543059818cb70e6442597a33454ec1e3d3a2bdb526c17875578d33c2ddcf72e").Exec() - }) - - It("should fail if give a digest as index reference", func() { - ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "sha256:bfa1728d6292d5fa7689f8f4daa145ee6f067b5779528c6e059d1132745ef508"), - string(multi_arch.LinuxAMD64.Digest)).ExpectFailure().MatchErrKeyWords("not a valid tag").Exec() - }) - - It("should fail if give digests as among the index references", func() { + It("should ignore digests given as index reference", func() { ORAS("manifest", "index", "create", fmt.Sprintf("%s,another-tag,sha256:bfa1728d6292d5fa7689f8f4daa145ee6f067b5779528c6e059d1132745ef508", RegistryRef(ZOTHost, ImageRepo, "digest-test")), - string(multi_arch.LinuxAMD64.Digest)).ExpectFailure().MatchErrKeyWords("not a valid tag").Exec() + string(multi_arch.LinuxAMD64.Digest), string(multi_arch.LinuxARM64.Digest)).Exec() }) It("should fail if given a reference that does not exist in the repo", func() { ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, ""), "does-not-exist").ExpectFailure(). - MatchErrKeyWords("Error", "could not find", "does-not-exist").Exec() + MatchErrKeyWords("Error:", "could not resolve", "does-not-exist").Exec() + }) + }) + + When("running `manifest index update`", func() { + It("should add a manifest by digest", func() { + // create an index for testing purpose + ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "update-add"), + string(multi_arch.LinuxAMD64.Digest), string(multi_arch.LinuxARM64.Digest)).Exec() + // add a manifest to the index + ORAS("manifest", "index", "update", RegistryRef(ZOTHost, ImageRepo, "update-add"), + "--add", string(multi_arch.LinuxARMV7.Digest)). + MatchKeyWords("sha256:84887718c9e61daa0f1996aad3ae2eb10db15dcbdab394e4b2dfee7967c55f2c").Exec() + // verify + ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "update-add")). + MatchKeyWords("amd64", "arm64", "v7").Exec() + }) + It("should update by specifying the index digest", func() { + // create an index for testing purpose + ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, ""), + string(multi_arch.LinuxAMD64.Digest), string(multi_arch.LinuxARM64.Digest)).Exec() + // add a manifest to the index + ORAS("manifest", "index", "update", RegistryRef(ZOTHost, ImageRepo, "sha256:cce9590b1193d8bcb70467e2381dc81e77869be4801c09abe9bc274b6a1d2001"), + "--add", string(multi_arch.LinuxARMV7.Digest)). + MatchKeyWords("sha256:84887718c9e61daa0f1996aad3ae2eb10db15dcbdab394e4b2dfee7967c55f2c").Exec() + // verify + ORAS("manifest", "fetch", RegistryRef(ZOTHost, ImageRepo, "sha256:84887718c9e61daa0f1996aad3ae2eb10db15dcbdab394e4b2dfee7967c55f2c")). + MatchKeyWords("amd64", "arm64", "v7").Exec() + }) + It("should tell user nothing to update if no update flags are used", func() { + ORAS("manifest", "index", "update", RegistryRef(ZOTHost, ImageRepo, "nothing-to-update")). + MatchKeyWords("nothing to update").Exec() + }) + It("should fail if empty reference is given", func() { + ORAS("manifest", "index", "update", RegistryRef(ZOTHost, ImageRepo, ""), + "--add", string(multi_arch.LinuxARMV7.Digest)).ExpectFailure(). + MatchErrKeyWords("Error:", "no tag or digest specified").Exec() + }) + It("should fail if a wrong reference is given as the index to update", func() { + ORAS("manifest", "index", "update", RegistryRef(ZOTHost, ImageRepo, "does-not-exist"), + "--add", string(multi_arch.LinuxARMV7.Digest)).ExpectFailure(). + MatchErrKeyWords("Error:", "could not resolve", "does-not-exist").Exec() + }) + It("should fail if a wrong reference is given as the manifest to add", func() { + // create an index for testing purpose + ORAS("manifest", "index", "create", RegistryRef(ZOTHost, ImageRepo, "update-wrong-tag"), + string(multi_arch.LinuxAMD64.Digest), string(multi_arch.LinuxARM64.Digest)).Exec() + // add a manifest to the index + ORAS("manifest", "index", "update", RegistryRef(ZOTHost, ImageRepo, "update-wrong-tag"), + "--add", "does-not-exist").ExpectFailure(). + MatchErrKeyWords("Error:", "could not resolve", "does-not-exist").Exec() }) }) })