From db41779c0d7e93414cb7ec1776196f7239b2bab0 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Fri, 15 Sep 2023 09:17:22 +0000 Subject: [PATCH 01/15] feats: support copying multi-arch referrers Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 66 ++++++++++++++++++++++++++---------- internal/docker/mediatype.go | 3 +- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 90f7cb090..256e98557 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -17,6 +17,7 @@ package root import ( "context" + "encoding/json" "fmt" "strings" "sync" @@ -28,6 +29,7 @@ import ( "oras.land/oras-go/v2/content" "oras.land/oras/cmd/oras/internal/display" "oras.land/oras/cmd/oras/internal/option" + "oras.land/oras/internal/docker" "oras.land/oras/internal/graph" ) @@ -137,28 +139,19 @@ func runCopy(ctx context.Context, opts copyOptions) error { var desc ocispec.Descriptor rOpts := oras.DefaultResolveOptions rOpts.TargetPlatform = opts.Platform.Platform - if dstRef := opts.To.Reference; dstRef == "" { + if opts.recursive { desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts) if err != nil { return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) } - if opts.recursive { - err = oras.ExtendedCopyGraph(ctx, src, dst, desc, extendedCopyOptions.ExtendedCopyGraphOptions) - } else { - err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions) - } + err = RecursiveCopy(ctx, src, dst, opts.To.Reference, desc, extendedCopyOptions) } else { - if opts.recursive { - srcRef := opts.From.Reference - if rOpts.TargetPlatform != nil { - // resolve source reference to specified platform - desc, err := oras.Resolve(ctx, src, opts.From.Reference, rOpts) - if err != nil { - return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) - } - srcRef = desc.Digest.String() + if opts.To.Reference == "" { + desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts) + if err != nil { + return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) } - desc, err = oras.ExtendedCopy(ctx, src, srcRef, dst, dstRef, extendedCopyOptions) + err = oras.CopyGraph(ctx, src, dst, desc, extendedCopyOptions.CopyGraphOptions) } else { copyOptions := oras.CopyOptions{ CopyGraphOptions: extendedCopyOptions.CopyGraphOptions, @@ -166,9 +159,10 @@ func runCopy(ctx context.Context, opts copyOptions) error { if opts.Platform.Platform != nil { copyOptions.WithTargetPlatform(opts.Platform.Platform) } - desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, dstRef, copyOptions) + desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, opts.To.Reference, copyOptions) } } + if err != nil { return err } @@ -191,3 +185,41 @@ func runCopy(ctx context.Context, opts copyOptions) error { return nil } + +// RecursiveCopy copies an artifact and its referrers from one target to another. +// If the artifact is a manifet list or index, its predecessor's referrers are copied as well. +func RecursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error { + var err error + if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList { + var fetched []byte + fetched, err := content.FetchAll(ctx, src, root) + if err != nil { + return err + } + var index ocispec.Index + if err = json.Unmarshal(fetched, &index); err != nil { + return nil + } + + var wg sync.WaitGroup + for _, desc := range index.Manifests { + wg.Add(1) + go func(desc ocispec.Descriptor) { + defer wg.Done() + err = oras.ExtendedCopyGraph(ctx, src, dst, desc, opts.ExtendedCopyGraphOptions) + }(desc) + } + wg.Wait() + if err != nil { + return err + } + } + + if dstRef == "" || dstRef == root.Digest.String() { + err = oras.ExtendedCopyGraph(ctx, src, dst, root, opts.ExtendedCopyGraphOptions) + } else { + _, err = oras.ExtendedCopy(ctx, src, root.Digest.String(), dst, dstRef, opts) + } + return err + +} diff --git a/internal/docker/mediatype.go b/internal/docker/mediatype.go index 5f07ff54f..4ba809fea 100644 --- a/internal/docker/mediatype.go +++ b/internal/docker/mediatype.go @@ -17,5 +17,6 @@ package docker // docker media types const ( - MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" ) From 9413c3d4cefe9a1bfc352b4e3c27b0e3352988c8 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Mon, 18 Sep 2023 02:35:33 +0000 Subject: [PATCH 02/15] fix logging & e2e Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 21 ++++++++++++++++----- test/e2e/suite/command/cp.go | 30 +++++++++++++----------------- test/e2e/suite/command/pull.go | 2 +- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 256e98557..35da06ee9 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -117,23 +117,34 @@ func runCopy(ctx context.Context, opts copyOptions) error { } // Prepare copy options + started := &sync.Map{} committed := &sync.Map{} extendedCopyOptions := oras.DefaultExtendedCopyOptions extendedCopyOptions.Concurrency = opts.concurrency extendedCopyOptions.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { return graph.Referrers(ctx, src, desc, "") } - extendedCopyOptions.PreCopy = display.StatusPrinter("Copying", opts.Verbose) + extendedCopyOptions.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { + if _, loaded := started.LoadOrStore(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]); !loaded { + return display.PrintStatus(desc, "Copying", opts.Verbose) + } + return nil + } extendedCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { - committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - if err := display.PrintSuccessorStatus(ctx, desc, "Skipped", dst, committed, opts.Verbose); err != nil { + started.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + if _, loaded := committed.LoadOrStore(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]); loaded { + return nil + } + if err := display.PrintSuccessorStatus(ctx, desc, "Skipped", dst, started, opts.Verbose); err != nil { return err } return display.PrintStatus(desc, "Copied ", opts.Verbose) } extendedCopyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { - committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - return display.PrintStatus(desc, "Exists ", opts.Verbose) + if _, loaded := started.LoadOrStore(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]); !loaded { + return display.PrintStatus(desc, "Exists ", opts.Verbose) + } + return nil } var desc ocispec.Descriptor diff --git a/test/e2e/suite/command/cp.go b/test/e2e/suite/command/cp.go index 317d0f989..a23da84de 100644 --- a/test/e2e/suite/command/cp.go +++ b/test/e2e/suite/command/cp.go @@ -115,7 +115,7 @@ var _ = Describe("1.1 registry users:", func() { }) It("should copy an image and its referrers to a new repository", func() { - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag) dst := RegistryRef(ZOTHost, cpTestRepo("referrers"), foobar.Digest) ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() @@ -123,7 +123,7 @@ var _ = Describe("1.1 registry users:", func() { }) It("should copy a multi-arch image and its referrers to a new repository via tag", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) dstRepo := cpTestRepo("index-referrers") dst := RegistryRef(ZOTHost, dstRepo, "copiedTag") @@ -142,13 +142,12 @@ var _ = Describe("1.1 registry users:", func() { Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())). - WithDescription("not copy referrer of successor"). - ExpectFailure(). + WithDescription("copy referrer of successor"). Exec() }) It("should copy a multi-arch image and its referrers to a new repository via digest", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) dstRepo := cpTestRepo("index-referrers-digest") dst := RegistryRef(ZOTHost, dstRepo, ma.Digest) @@ -168,7 +167,6 @@ var _ = Describe("1.1 registry users:", func() { Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())). WithDescription("not copy referrer of successor"). - ExpectFailure(). Exec() }) @@ -270,7 +268,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() { When("running `cp`", func() { It("should copy an image artifact and its referrers from a registry to a fallback registry", func() { repo := cpTestRepo("to-fallback") - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String()) dst := RegistryRef(FallbackHost, repo, "") ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() @@ -280,7 +278,7 @@ var _ = Describe("OCI spec 1.0 registry users:", func() { }) It("should copy an image artifact and its referrers from a fallback registry to a registry", func() { repo := cpTestRepo("from-fallback") - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(FallbackHost, ArtifactRepo, foobar.SBOMImageReferrer.Digest.String()) dst := RegistryRef(ZOTHost, repo, "") ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() @@ -439,7 +437,7 @@ var _ = Describe("OCI layout users:", func() { }) It("should copy a tagged image and its referrers from a registry to an OCI image layout", func() { - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) dst := LayoutRef(GinkgoT().TempDir(), "copied") src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Tag) // test @@ -451,7 +449,7 @@ var _ = Describe("OCI layout users:", func() { }) It("should copy a image and its referrers from a registry to an OCI image layout via digest", func() { - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) toDir := GinkgoT().TempDir() src := RegistryRef(ZOTHost, ArtifactRepo, foobar.Digest) // test @@ -463,7 +461,7 @@ var _ = Describe("OCI layout users:", func() { }) It("should copy a multi-arch image and its referrers from a registry to an OCI image layout a via tag", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) toDir := GinkgoT().TempDir() dst := LayoutRef(toDir, "copied") @@ -485,13 +483,12 @@ var _ = Describe("OCI layout users:", func() { Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) ORAS("manifest", "fetch", Flags.Layout, LayoutRef(toDir, ma.LinuxAMD64Referrer.Digest.String())). - WithDescription("not copy referrer of successor"). - ExpectFailure(). + WithDescription("copy referrer of successor"). Exec() }) It("should copy a multi-arch image and its referrers from an OCI image layout to a registry via digest", func() { - stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.IndexReferrerConfigStateKey) + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) fromDir := GinkgoT().TempDir() src := LayoutRef(fromDir, ma.Tag) dst := RegistryRef(ZOTHost, cpTestRepo("recursive-from-layout"), "copied") @@ -514,9 +511,8 @@ var _ = Describe("OCI layout users:", func() { Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) Expect(len(index.Manifests)).To(Equal(1)) Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) - ORAS("manifest", "fetch", LayoutRef(fromDir, ma.LinuxAMD64Referrer.Digest.String())). - WithDescription("not copy referrer of successor"). - ExpectFailure(). + ORAS("manifest", "fetch", dst). + WithDescription("copy referrer of successor"). Exec() }) diff --git a/test/e2e/suite/command/pull.go b/test/e2e/suite/command/pull.go index 8bf6c1225..65d5e6eea 100644 --- a/test/e2e/suite/command/pull.go +++ b/test/e2e/suite/command/pull.go @@ -122,7 +122,7 @@ var _ = Describe("OCI spec 1.1 registry users:", func() { It("should copy an artifact with blob", func() { repo := cpTestRepo("artifact-with-blob") - stateKeys := append(append(foobarStates, foobar.ImageReferrersStateKeys...), foobar.ImageReferrerConfigStateKeys...) + stateKeys := append(append(foobar.ImageLayerStateKeys, foobar.ManifestStateKey, foobar.ImageReferrerConfigStateKeys[0]), foobar.ImageReferrersStateKeys...) src := RegistryRef(ZOTHost, ArtifactRepo, foobar.SignatureImageReferrer.Digest.String()) dst := RegistryRef(FallbackHost, repo, "") ORAS("cp", "-r", src, dst, "-v").MatchStatus(stateKeys, true, len(stateKeys)).Exec() From 50c55410c63ff9ee485399ed308cdeeddcf63f8b Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Mon, 18 Sep 2023 06:39:16 +0000 Subject: [PATCH 03/15] resolve comment Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 35da06ee9..7990cab51 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -173,7 +173,6 @@ func runCopy(ctx context.Context, opts copyOptions) error { desc, err = oras.Copy(ctx, src, opts.From.Reference, dst, opts.To.Reference, copyOptions) } } - if err != nil { return err } @@ -198,7 +197,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { } // RecursiveCopy copies an artifact and its referrers from one target to another. -// If the artifact is a manifet list or index, its predecessor's referrers are copied as well. +// If the artifact is a manifest list or index, referrers of its manifests are copied as well. func RecursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error { var err error if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList { @@ -232,5 +231,4 @@ func RecursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T _, err = oras.ExtendedCopy(ctx, src, root.Digest.String(), dst, dstRef, opts) } return err - } From d00a8de8f03dc54edb33ea50108995a2a9dc0475 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 01:05:50 +0000 Subject: [PATCH 04/15] rename function Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 7990cab51..a3903e49c 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -155,7 +155,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { if err != nil { return fmt.Errorf("failed to resolve %s: %w", opts.From.Reference, err) } - err = RecursiveCopy(ctx, src, dst, opts.To.Reference, desc, extendedCopyOptions) + err = recursiveCopy(ctx, src, dst, opts.To.Reference, desc, extendedCopyOptions) } else { if opts.To.Reference == "" { desc, err = oras.Resolve(ctx, src, opts.From.Reference, rOpts) @@ -196,9 +196,9 @@ func runCopy(ctx context.Context, opts copyOptions) error { return nil } -// RecursiveCopy copies an artifact and its referrers from one target to another. +// recursiveCopy copies an artifact and its referrers from one target to another. // If the artifact is a manifest list or index, referrers of its manifests are copied as well. -func RecursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error { +func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error { var err error if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList { var fetched []byte From e792317273211fd3e4870623f9dcb022282ab88e Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 07:35:34 +0000 Subject: [PATCH 05/15] re-implement `oras cp` Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 48 +++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index a3903e49c..211b9eeb3 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -117,34 +117,23 @@ func runCopy(ctx context.Context, opts copyOptions) error { } // Prepare copy options - started := &sync.Map{} committed := &sync.Map{} extendedCopyOptions := oras.DefaultExtendedCopyOptions extendedCopyOptions.Concurrency = opts.concurrency extendedCopyOptions.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { return graph.Referrers(ctx, src, desc, "") } - extendedCopyOptions.PreCopy = func(ctx context.Context, desc ocispec.Descriptor) error { - if _, loaded := started.LoadOrStore(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]); !loaded { - return display.PrintStatus(desc, "Copying", opts.Verbose) - } - return nil - } + extendedCopyOptions.PreCopy = display.StatusPrinter("Copying", opts.Verbose) extendedCopyOptions.PostCopy = func(ctx context.Context, desc ocispec.Descriptor) error { - started.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) - if _, loaded := committed.LoadOrStore(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]); loaded { - return nil - } - if err := display.PrintSuccessorStatus(ctx, desc, "Skipped", dst, started, opts.Verbose); err != nil { + committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + if err := display.PrintSuccessorStatus(ctx, desc, "Skipped", dst, committed, opts.Verbose); err != nil { return err } return display.PrintStatus(desc, "Copied ", opts.Verbose) } extendedCopyOptions.OnCopySkipped = func(ctx context.Context, desc ocispec.Descriptor) error { - if _, loaded := started.LoadOrStore(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]); !loaded { - return display.PrintStatus(desc, "Exists ", opts.Verbose) - } - return nil + committed.Store(desc.Digest.String(), desc.Annotations[ocispec.AnnotationTitle]) + return display.PrintStatus(desc, "Exists ", opts.Verbose) } var desc ocispec.Descriptor @@ -210,18 +199,25 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T if err = json.Unmarshal(fetched, &index); err != nil { return nil } - - var wg sync.WaitGroup + // point referrers child manifests to root + var referrers []ocispec.Descriptor for _, desc := range index.Manifests { - wg.Add(1) - go func(desc ocispec.Descriptor) { - defer wg.Done() - err = oras.ExtendedCopyGraph(ctx, src, dst, desc, opts.ExtendedCopyGraphOptions) - }(desc) + descs, err := graph.Referrers(ctx, src, desc, "") + if err != nil { + return err + } + referrers = append(referrers, descs...) } - wg.Wait() - if err != nil { - return err + findPredecessor := opts.FindPredecessors + opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + descs, err := findPredecessor(ctx, src, desc) + if err != nil { + return nil, err + } + if content.Equal(desc, root) { + descs = append(descs, referrers...) + } + return descs, nil } } From 5270af83afada49779fb94a3d8875a78a3c172f5 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 07:47:36 +0000 Subject: [PATCH 06/15] doc clean Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 211b9eeb3..223ad40d0 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -199,7 +199,7 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T if err = json.Unmarshal(fetched, &index); err != nil { return nil } - // point referrers child manifests to root + // point referrers of child manifests to root var referrers []ocispec.Descriptor for _, desc := range index.Manifests { descs, err := graph.Referrers(ctx, src, desc, "") From 0b8d844090116d9eabf461f67a40e59ae7dbe93d Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 08:02:09 +0000 Subject: [PATCH 07/15] bug fix Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 223ad40d0..e586d6254 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -199,16 +199,17 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T if err = json.Unmarshal(fetched, &index); err != nil { return nil } + // point referrers of child manifests to root + findPredecessor := opts.FindPredecessors var referrers []ocispec.Descriptor for _, desc := range index.Manifests { - descs, err := graph.Referrers(ctx, src, desc, "") + descs, err := findPredecessor(ctx, src, desc) if err != nil { return err } referrers = append(referrers, descs...) } - findPredecessor := opts.FindPredecessors opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { descs, err := findPredecessor(ctx, src, desc) if err != nil { From 62f244557abcf1edd05d4daf31f5f89affd52f69 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:04:49 +0000 Subject: [PATCH 08/15] get referrers with concurrency tuned Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 14 +++++--------- internal/graph/graph.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index e586d6254..7b2096a28 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -188,9 +188,7 @@ func runCopy(ctx context.Context, opts copyOptions) error { // recursiveCopy copies an artifact and its referrers from one target to another. // If the artifact is a manifest list or index, referrers of its manifests are copied as well. func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.Target, dstRef string, root ocispec.Descriptor, opts oras.ExtendedCopyOptions) error { - var err error if root.MediaType == ocispec.MediaTypeImageIndex || root.MediaType == docker.MediaTypeManifestList { - var fetched []byte fetched, err := content.FetchAll(ctx, src, root) if err != nil { return err @@ -202,14 +200,11 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T // point referrers of child manifests to root findPredecessor := opts.FindPredecessors - var referrers []ocispec.Descriptor - for _, desc := range index.Manifests { - descs, err := findPredecessor(ctx, src, desc) - if err != nil { - return err - } - referrers = append(referrers, descs...) + referrers, err := graph.FindPredecessorsCurrently(ctx, src, index.Manifests, opts) + if err != nil { + return err } + opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { descs, err := findPredecessor(ctx, src, desc) if err != nil { @@ -222,6 +217,7 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T } } + var err error if dstRef == "" || dstRef == root.Digest.String() { err = oras.ExtendedCopyGraph(ctx, src, dst, root, opts.ExtendedCopyGraphOptions) } else { diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 4269d36df..da8acee45 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -18,8 +18,11 @@ package graph import ( "context" "encoding/json" + "sync" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" + "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/registry" "oras.land/oras/internal/docker" @@ -188,3 +191,30 @@ func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descr defer rc.Close() return content.ReadAll(rc, desc) } + +// FindPredecessorsCurrently returns all predecessors of descs in src concurrently. +func FindPredecessorsCurrently(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) { + // point referrers of child manifests to root + var referrers []ocispec.Descriptor + g, ctx := errgroup.WithContext(ctx) + var m sync.Mutex + g.SetLimit(opts.Concurrency) + for _, desc := range descs { + g.Go(func(node ocispec.Descriptor) func() error { + return func() error { + descs, err := opts.FindPredecessors(ctx, src, node) + if err != nil { + return err + } + m.Lock() + defer m.Unlock() + referrers = append(referrers, descs...) + return nil + } + }(desc)) + } + if err := g.Wait(); err != nil { + return nil, err + } + return referrers, nil +} From 40aaf4efec9b4116ca01b209d0e2f8669005f6ac Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:05:59 +0000 Subject: [PATCH 09/15] doc clean Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 1 + internal/graph/graph.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 7b2096a28..11b95c95f 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -211,6 +211,7 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T return nil, err } if content.Equal(desc, root) { + // make sure referrers of child manifests are copied by pointing them to root descs = append(descs, referrers...) } return descs, nil diff --git a/internal/graph/graph.go b/internal/graph/graph.go index da8acee45..3b8aaa29f 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -194,7 +194,6 @@ func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descr // FindPredecessorsCurrently returns all predecessors of descs in src concurrently. func FindPredecessorsCurrently(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) { - // point referrers of child manifests to root var referrers []ocispec.Descriptor g, ctx := errgroup.WithContext(ctx) var m sync.Mutex From f15b66e79410686fb43d0ce10f2a6cb3c33e5a37 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:16:13 +0000 Subject: [PATCH 10/15] code clean Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 11b95c95f..80f880e9f 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -199,12 +199,12 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T } // point referrers of child manifests to root - findPredecessor := opts.FindPredecessors referrers, err := graph.FindPredecessorsCurrently(ctx, src, index.Manifests, opts) if err != nil { return err } + findPredecessor := opts.FindPredecessors opts.FindPredecessors = func(ctx context.Context, src content.ReadOnlyGraphStorage, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { descs, err := findPredecessor(ctx, src, desc) if err != nil { From 7d2bd1130324f16f55cd4dc347bead0f2d2bb5dd Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:16:40 +0000 Subject: [PATCH 11/15] code clean Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 80f880e9f..2ff578149 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -198,7 +198,6 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T return nil } - // point referrers of child manifests to root referrers, err := graph.FindPredecessorsCurrently(ctx, src, index.Manifests, opts) if err != nil { return err From a958e367411929f6489503ac25045cf5c83b5ffe Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:19:52 +0000 Subject: [PATCH 12/15] rename function Signed-off-by: Billy Zha --- cmd/oras/root/cp.go | 2 +- internal/graph/graph.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/oras/root/cp.go b/cmd/oras/root/cp.go index 2ff578149..908493618 100644 --- a/cmd/oras/root/cp.go +++ b/cmd/oras/root/cp.go @@ -198,7 +198,7 @@ func recursiveCopy(ctx context.Context, src oras.ReadOnlyGraphTarget, dst oras.T return nil } - referrers, err := graph.FindPredecessorsCurrently(ctx, src, index.Manifests, opts) + referrers, err := graph.FindPredecessors(ctx, src, index.Manifests, opts) if err != nil { return err } diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 3b8aaa29f..856a67571 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -192,8 +192,8 @@ func fetchBytes(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descr return content.ReadAll(rc, desc) } -// FindPredecessorsCurrently returns all predecessors of descs in src concurrently. -func FindPredecessorsCurrently(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) { +// FindPredecessors returns all predecessors of descs in src concurrently. +func FindPredecessors(ctx context.Context, src oras.ReadOnlyGraphTarget, descs []ocispec.Descriptor, opts oras.ExtendedCopyOptions) ([]ocispec.Descriptor, error) { var referrers []ocispec.Descriptor g, ctx := errgroup.WithContext(ctx) var m sync.Mutex From 738fbd77306811e059b32208fd5c88ef8f8b2efd Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:27:07 +0000 Subject: [PATCH 13/15] add concurrency test Signed-off-by: Billy Zha --- test/e2e/suite/command/cp.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/e2e/suite/command/cp.go b/test/e2e/suite/command/cp.go index a23da84de..b9401eb6c 100644 --- a/test/e2e/suite/command/cp.go +++ b/test/e2e/suite/command/cp.go @@ -146,6 +146,31 @@ var _ = Describe("1.1 registry users:", func() { Exec() }) + It("should copy a multi-arch image and its referrers without concurrency limitation", func() { + stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) + src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) + dstRepo := cpTestRepo("index-referrers-concurrent") + dst := RegistryRef(ZOTHost, dstRepo, "copiedTag") + // test + ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0"). + MatchStatus(stateKeys, true, len(stateKeys)). + MatchKeyWords("Digest: " + ma.Digest). + Exec() + // validate + CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.Digest), dst) + var index ocispec.Index + bytes := ORAS("discover", dst, "-o", "json", "--artifact-type", ma.IndexReferrerConfigStateKey.Name). + MatchKeyWords(ma.IndexReferrerDigest). + WithDescription("copy image referrer"). + Exec().Out.Contents() + Expect(json.Unmarshal(bytes, &index)).ShouldNot(HaveOccurred()) + Expect(len(index.Manifests)).To(Equal(1)) + Expect(index.Manifests[0].Digest.String()).To(Equal(ma.IndexReferrerDigest)) + ORAS("manifest", "fetch", RegistryRef(ZOTHost, dstRepo, ma.LinuxAMD64Referrer.Digest.String())). + WithDescription("copy referrer of successor"). + Exec() + }) + It("should copy a multi-arch image and its referrers to a new repository via digest", func() { stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) From fffc3b1382243efeb5adc614e5aeea4be9aa02b6 Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:38:34 +0000 Subject: [PATCH 14/15] fix concurrency bug Signed-off-by: Billy Zha --- internal/graph/graph.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 856a67571..e5323124d 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -197,7 +197,9 @@ func FindPredecessors(ctx context.Context, src oras.ReadOnlyGraphTarget, descs [ var referrers []ocispec.Descriptor g, ctx := errgroup.WithContext(ctx) var m sync.Mutex - g.SetLimit(opts.Concurrency) + if opts.Concurrency != 0 { + g.SetLimit(opts.Concurrency) + } for _, desc := range descs { g.Go(func(node ocispec.Descriptor) func() error { return func() error { From cd843215beb1b5a41c21e9241fb612331d38a2fc Mon Sep 17 00:00:00 2001 From: Billy Zha Date: Tue, 19 Sep 2023 09:47:42 +0000 Subject: [PATCH 15/15] add test for empty index Signed-off-by: Billy Zha --- test/e2e/internal/testdata/multi_arch/const.go | 1 + test/e2e/suite/command/cp.go | 10 ++++++++++ ...de647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 | 1 + test/e2e/testdata/zot/command/images/index.json | 8 ++++++++ 4 files changed, 20 insertions(+) create mode 100644 test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 diff --git a/test/e2e/internal/testdata/multi_arch/const.go b/test/e2e/internal/testdata/multi_arch/const.go index 119764386..5915b5942 100644 --- a/test/e2e/internal/testdata/multi_arch/const.go +++ b/test/e2e/internal/testdata/multi_arch/const.go @@ -23,6 +23,7 @@ import ( var ( Tag = "multi" + EmptyTag = "empty_index" Digest = "sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f" Manifest = `{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:9d84a5716c66a1d1b9c13f8ed157ba7d1edfe7f9b8766728b8a1f25c0d9c14c1","size":458,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4f93460061882467e6fb3b772dc6ab72130d9ac1906aed2fc7589a5cd145433c","size":458,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:58efe73e78fe043ca31b89007a025c594ce12aa7e6da27d21c7b14b50112e255","size":458,"platform":{"architecture":"arm","os":"linux","variant":"v7"}}]}` Descriptor = `{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:e2bfc9cc6a84ec2d7365b5a28c6bc5806b7fa581c9ad7883be955a64e3cc034f","size":706}` diff --git a/test/e2e/suite/command/cp.go b/test/e2e/suite/command/cp.go index b9401eb6c..5e0893d73 100644 --- a/test/e2e/suite/command/cp.go +++ b/test/e2e/suite/command/cp.go @@ -171,6 +171,16 @@ var _ = Describe("1.1 registry users:", func() { Exec() }) + It("should copy an empty index", func() { + src := RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag) + dstRepo := cpTestRepo("empty-index") + dst := RegistryRef(ZOTHost, dstRepo, "copiedTag") + // test + ORAS("cp", src, dst, "-r", "-v", "--concurrency", "0").Exec() + // validate + CompareRef(RegistryRef(ZOTHost, ImageRepo, ma.EmptyTag), dst) + }) + It("should copy a multi-arch image and its referrers to a new repository via digest", func() { stateKeys := append(ma.IndexStateKeys, ma.IndexZOTReferrerStateKey, ma.LinuxAMD64ReferrerConfigStateKey) src := RegistryRef(ZOTHost, ArtifactRepo, ma.Tag) diff --git a/test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 b/test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 new file mode 100644 index 000000000..3df36494f --- /dev/null +++ b/test/e2e/testdata/zot/command/images/blobs/sha256/b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286 @@ -0,0 +1 @@ +{"mediaType":"application/vnd.oci.image.index.v1+json","schemaVersion":2,"manifests":[]} diff --git a/test/e2e/testdata/zot/command/images/index.json b/test/e2e/testdata/zot/command/images/index.json index 4b2b1af71..2162bc883 100644 --- a/test/e2e/testdata/zot/command/images/index.json +++ b/test/e2e/testdata/zot/command/images/index.json @@ -44,6 +44,14 @@ "architecture": "amd64", "os": "linux" } + }, + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": "sha256:b2a5fcfb112ccde647a5a3dc0215c2c9e7d0ce598924a5ec48aa85beca048286", + "size": 89, + "annotations": { + "org.opencontainers.image.ref.name": "empty_index" + } } ] } \ No newline at end of file