diff --git a/cmd/skopeo/sync.go b/cmd/skopeo/sync.go index 138ff9beca..048b03ec5a 100644 --- a/cmd/skopeo/sync.go +++ b/cmd/skopeo/sync.go @@ -18,6 +18,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -35,13 +36,14 @@ type syncOptions struct { source string // Source repository name destination string // Destination registry name scoped bool // When true, namespace copied images at destination using the source repository name + all bool // Copy all of the images if an image in the source is a list } // repoDescriptor contains information of a single repository used as a sync source. type repoDescriptor struct { - DirBasePath string // base path when source is 'dir' - TaggedImages []types.ImageReference // List of tagged image found for the repository - Context *types.SystemContext // SystemContext for the sync command + DirBasePath string // base path when source is 'dir' + ImageRefs []types.ImageReference // List of tagged image found for the repository + Context *types.SystemContext // SystemContext for the sync command } // tlsVerify is an implementation of the Unmarshaler interface, used to @@ -53,7 +55,7 @@ type tlsVerifyConfig struct { // registrySyncConfig contains information about a single registry, read from // the source YAML file type registrySyncConfig struct { - Images map[string][]string // Images map images name to slices with the images' tags + Images map[string][]string // Images map images name to slices with the images' references (tags, digests) ImagesByTagRegex map[string]string `yaml:"images-by-tag-regex"` // Images map images name to regular expression with the images' tags Credentials types.DockerAuthConfig // Username and password used to authenticate with the registry TLSVerify tlsVerifyConfig `yaml:"tls-verify"` // TLS verification mode (enabled by default) @@ -96,6 +98,7 @@ See skopeo-sync(1) for details. flags.StringVarP(&opts.source, "src", "s", "", "SOURCE transport type") flags.StringVarP(&opts.destination, "dest", "d", "", "DESTINATION transport type") flags.BoolVar(&opts.scoped, "scoped", false, "Images at DESTINATION are prefix using the full source image path as scope") + flags.BoolVarP(&opts.all, "all", "a", false, "Copy all images if SOURCE-IMAGE is a list") flags.AddFlagSet(&sharedFlags) flags.AddFlagSet(&srcFlags) flags.AddFlagSet(&destFlags) @@ -269,7 +272,7 @@ func imagesToCopyFromDir(dirPath string) ([]types.ImageReference, error) { // in a registry configuration. // It returns a repository descriptors slice with as many elements as the images // found and any error encountered. Each element of the slice is a list of -// tagged image references, to be used as sync source. +// image references, to be used as sync source. func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourceCtx types.SystemContext) ([]repoDescriptor, error) { serverCtx := &sourceCtx // override ctx with per-registryName options @@ -280,7 +283,7 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc serverCtx.DockerAuthConfig = &cfg.Credentials var repoDescList []repoDescriptor - for imageName, tags := range cfg.Images { + for imageName, refs := range cfg.Images { repoLogger := logrus.WithFields(logrus.Fields{ "repo": imageName, "registry": registryName, @@ -295,24 +298,37 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc repoLogger.Info("Processing repo") var sourceReferences []types.ImageReference - if len(tags) != 0 { - for _, tag := range tags { - tagLogger := logrus.WithFields(logrus.Fields{"tag": tag}) - taggedRef, err := reference.WithTag(repoRef, tag) - if err != nil { - tagLogger.Error("Error parsing tag, skipping") - logrus.Error(err) - continue + if len(refs) != 0 { + for _, ref := range refs { + tagLogger := logrus.WithFields(logrus.Fields{"ref": ref}) + var named reference.Named + // first try as digest + if d, err := digest.Parse(ref); err == nil { + named, err = reference.WithDigest(repoRef, d) + if err != nil { + tagLogger.Error("Error processing ref, skipping") + logrus.Error(err) + continue + } + } else { + tagLogger.Debugf("Ref was not a digest, trying as a tag: %s", err) + named, err = reference.WithTag(repoRef, ref) + if err != nil { + tagLogger.Error("Error parsing ref, skipping") + logrus.Error(err) + continue + } } - imageRef, err := docker.NewReference(taggedRef) + + imageRef, err := docker.NewReference(named) if err != nil { - tagLogger.Error("Error processing tag, skipping") + tagLogger.Error("Error processing ref, skipping") logrus.Errorf("Error getting image reference: %s", err) continue } sourceReferences = append(sourceReferences, imageRef) } - } else { // len(tags) == 0 + } else { // len(refs) == 0 repoLogger.Info("Querying registry for image tags") sourceReferences, err = imagesToCopyFromRepo(serverCtx, repoRef) if err != nil { @@ -323,12 +339,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc } if len(sourceReferences) == 0 { - repoLogger.Warnf("No tags to sync found") + repoLogger.Warnf("No refs to sync found") continue } repoDescList = append(repoDescList, repoDescriptor{ - TaggedImages: sourceReferences, - Context: serverCtx}) + ImageRefs: sourceReferences, + Context: serverCtx}) } for imageName, tagRegex := range cfg.ImagesByTagRegex { @@ -377,12 +393,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc } if len(sourceReferences) == 0 { - repoLogger.Warnf("No tags to sync found") + repoLogger.Warnf("No refs to sync found") continue } repoDescList = append(repoDescList, repoDescriptor{ - TaggedImages: sourceReferences, - Context: serverCtx}) + ImageRefs: sourceReferences, + Context: serverCtx}) } return repoDescList, nil @@ -415,13 +431,13 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex if err != nil { return nil, errors.Wrapf(err, "Cannot obtain a valid image reference for transport %q and reference %q", docker.Transport.Name(), named.String()) } - desc.TaggedImages = []types.ImageReference{srcRef} + desc.ImageRefs = []types.ImageReference{srcRef} } else { - desc.TaggedImages, err = imagesToCopyFromRepo(sourceCtx, named) + desc.ImageRefs, err = imagesToCopyFromRepo(sourceCtx, named) if err != nil { return descriptors, err } - if len(desc.TaggedImages) == 0 { + if len(desc.ImageRefs) == 0 { return descriptors, errors.Errorf("No images to sync found in %q", source) } } @@ -437,11 +453,11 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex } desc.DirBasePath = source var err error - desc.TaggedImages, err = imagesToCopyFromDir(source) + desc.ImageRefs, err = imagesToCopyFromDir(source) if err != nil { return descriptors, err } - if len(desc.TaggedImages) == 0 { + if len(desc.ImageRefs) == 0 { return descriptors, errors.Errorf("No images to sync found in %q", source) } descriptors = append(descriptors, desc) @@ -509,6 +525,11 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error { return errors.New("sync from 'dir' to 'dir' not implemented, consider using rsync instead") } + imageListSelection := copy.CopySystemImage + if opts.all { + imageListSelection = copy.CopyAllImages + } + sourceCtx, err := opts.srcImage.newSystemContext() if err != nil { return err @@ -534,15 +555,16 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error { imagesNumber := 0 options := copy.Options{ - RemoveSignatures: opts.removeSignatures, - SignBy: opts.signByFingerprint, - ReportWriter: os.Stdout, - DestinationCtx: destinationCtx, + RemoveSignatures: opts.removeSignatures, + SignBy: opts.signByFingerprint, + ReportWriter: os.Stdout, + DestinationCtx: destinationCtx, + ImageListSelection: imageListSelection, } for _, srcRepo := range srcRepoList { options.SourceCtx = srcRepo.Context - for counter, ref := range srcRepo.TaggedImages { + for counter, ref := range srcRepo.ImageRefs { var destSuffix string switch ref.Transport() { case docker.Transport: @@ -569,13 +591,13 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error { logrus.WithFields(logrus.Fields{ "from": transports.ImageName(ref), "to": transports.ImageName(destRef), - }).Infof("Copying image tag %d/%d", counter+1, len(srcRepo.TaggedImages)) + }).Infof("Copying image ref %d/%d", counter+1, len(srcRepo.ImageRefs)) if err = retry.RetryIfNecessary(ctx, func() error { _, err = copy.Image(ctx, policyContext, destRef, ref, &options) return err }, opts.retryOpts); err != nil { - return errors.Wrapf(err, "Error copying tag %q", transports.ImageName(ref)) + return errors.Wrapf(err, "Error copying ref %q", transports.ImageName(ref)) } imagesNumber++ } diff --git a/docs/skopeo-sync.1.md b/docs/skopeo-sync.1.md index 4e603feb03..21570cba95 100644 --- a/docs/skopeo-sync.1.md +++ b/docs/skopeo-sync.1.md @@ -32,6 +32,11 @@ When the `--scoped` option is specified, images are prefixed with the source ima name can be stored at _destination_. ## OPTIONS +**--all** +If one of the images in __src__ refers to a list of images, instead of copying just the image which matches the current OS and +architecture (subject to the use of the global --override-os, --override-arch and --override-variant options), attempt to copy all of +the images in the list, and the list itself. + **--authfile** _path_ Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`. @@ -147,6 +152,7 @@ registry.example.com: redis: - "1.0" - "2.0" + - "sha256:0000000000000000000000000000000011111111111111111111111111111111" images-by-tag-regex: nginx: ^1\.13\.[12]-alpine-perl$ credentials: @@ -166,7 +172,7 @@ skopeo sync --src yaml --dest docker sync.yml my-registry.local.lan/repo/ ``` This will copy the following images: - Repository `registry.example.com/busybox`: all images, as no tags are specified. -- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0". +- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0" along with image with digest "sha256:0000000000000000000000000000000011111111111111111111111111111111". - Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl". - Repository `quay.io/coreos/etcd`: images tagged "latest". diff --git a/integration/sync_test.go b/integration/sync_test.go index 43d9ada1f1..17e93fb751 100644 --- a/integration/sync_test.go +++ b/integration/sync_test.go @@ -117,6 +117,34 @@ func (s *SyncSuite) TestDocker2DirTagged(c *check.C) { c.Assert(out, check.Equals, "") } +func (s *SyncSuite) TestDocker2DirTaggedAll(c *check.C) { + tmpDir, err := ioutil.TempDir("", "skopeo-sync-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(tmpDir) + + // FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection. + image := "busybox:latest" + imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image)) + c.Assert(err, check.IsNil) + imagePath := imageRef.DockerReference().String() + + dir1 := path.Join(tmpDir, "dir1") + dir2 := path.Join(tmpDir, "dir2") + + // sync docker => dir + assertSkopeoSucceeds(c, "", "sync", "--all", "--scoped", "--src", "docker", "--dest", "dir", image, dir1) + _, err = os.Stat(path.Join(dir1, imagePath, "manifest.json")) + c.Assert(err, check.IsNil) + + // copy docker => dir + assertSkopeoSucceeds(c, "", "copy", "--all", "docker://"+image, "dir:"+dir2) + _, err = os.Stat(path.Join(dir2, "manifest.json")) + c.Assert(err, check.IsNil) + + out := combinedOutputOfCommand(c, "diff", "-urN", path.Join(dir1, imagePath), dir2) + c.Assert(out, check.Equals, "") +} + func (s *SyncSuite) TestScoped(c *check.C) { // FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection. image := "busybox:latest" @@ -289,6 +317,37 @@ docker.io: c.Assert(nManifests, check.Equals, nTags) } +func (s *SyncSuite) TestYamlDigest2Dir(c *check.C) { + tmpDir, err := ioutil.TempDir("", "skopeo-sync-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(tmpDir) + dir1 := path.Join(tmpDir, "dir1") + + yamlConfig := ` +docker.io: + images: + redis: + - sha256:61ce79d60150379787d7da677dcb89a7a047ced63406e29d6b2677b2b2163e92 +` + yamlFile := path.Join(tmpDir, "registries.yaml") + ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644) + assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "dir", yamlFile, dir1) + + nManifests := 0 + err = filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && info.Name() == "manifest.json" { + nManifests++ + return filepath.SkipDir + } + return nil + }) + c.Assert(err, check.IsNil) + c.Assert(nManifests, check.Equals, 1) +} + func (s *SyncSuite) TestYaml2Dir(c *check.C) { tmpDir, err := ioutil.TempDir("", "skopeo-sync-test") c.Assert(err, check.IsNil)