Merge pull request #880 from muff1nman/sync-digests

Fix #858 Add support for digests in sync

cmd/skopeo/sync.go

@@ -18,6 +18,7 @@ import (
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/transports"
 	"github.com/containers/image/v5/types"
+	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -35,13 +36,14 @@ type syncOptions struct {
 	source            string // Source repository name
 	destination       string // Destination registry name
 	scoped            bool   // When true, namespace copied images at destination using the source repository name
+	all               bool   // Copy all of the images if an image in the source is a list
 }
 
 // repoDescriptor contains information of a single repository used as a sync source.
 type repoDescriptor struct {
-	DirBasePath  string                 // base path when source is 'dir'
-	TaggedImages []types.ImageReference // List of tagged image found for the repository
-	Context      *types.SystemContext   // SystemContext for the sync command
+	DirBasePath string                 // base path when source is 'dir'
+	ImageRefs   []types.ImageReference // List of tagged image found for the repository
+	Context     *types.SystemContext   // SystemContext for the sync command
 }
 
 // tlsVerify is an implementation of the Unmarshaler interface, used to
@@ -53,7 +55,7 @@ type tlsVerifyConfig struct {
 // registrySyncConfig contains information about a single registry, read from
 // the source YAML file
 type registrySyncConfig struct {
-	Images           map[string][]string    // Images map images name to slices with the images' tags
+	Images           map[string][]string    // Images map images name to slices with the images' references (tags, digests)
 	ImagesByTagRegex map[string]string      `yaml:"images-by-tag-regex"` // Images map images name to regular expression with the images' tags
 	Credentials      types.DockerAuthConfig // Username and password used to authenticate with the registry
 	TLSVerify        tlsVerifyConfig        `yaml:"tls-verify"` // TLS verification mode (enabled by default)
@@ -96,6 +98,7 @@ See skopeo-sync(1) for details.
 	flags.StringVarP(&opts.source, "src", "s", "", "SOURCE transport type")
 	flags.StringVarP(&opts.destination, "dest", "d", "", "DESTINATION transport type")
 	flags.BoolVar(&opts.scoped, "scoped", false, "Images at DESTINATION are prefix using the full source image path as scope")
+	flags.BoolVarP(&opts.all, "all", "a", false, "Copy all images if SOURCE-IMAGE is a list")
 	flags.AddFlagSet(&sharedFlags)
 	flags.AddFlagSet(&srcFlags)
 	flags.AddFlagSet(&destFlags)
@@ -269,7 +272,7 @@ func imagesToCopyFromDir(dirPath string) ([]types.ImageReference, error) {
 // in a registry configuration.
 // It returns a repository descriptors slice with as many elements as the images
 // found and any error encountered. Each element of the slice is a list of
-// tagged image references, to be used as sync source.
+// image references, to be used as sync source.
 func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourceCtx types.SystemContext) ([]repoDescriptor, error) {
 	serverCtx := &sourceCtx
 	// override ctx with per-registryName options
@@ -280,7 +283,7 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 	serverCtx.DockerAuthConfig = &cfg.Credentials
 
 	var repoDescList []repoDescriptor
-	for imageName, tags := range cfg.Images {
+	for imageName, refs := range cfg.Images {
 		repoLogger := logrus.WithFields(logrus.Fields{
 			"repo":     imageName,
 			"registry": registryName,
@@ -295,24 +298,37 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 		repoLogger.Info("Processing repo")
 
 		var sourceReferences []types.ImageReference
-		if len(tags) != 0 {
-			for _, tag := range tags {
-				tagLogger := logrus.WithFields(logrus.Fields{"tag": tag})
-				taggedRef, err := reference.WithTag(repoRef, tag)
-				if err != nil {
-					tagLogger.Error("Error parsing tag, skipping")
-					logrus.Error(err)
-					continue
+		if len(refs) != 0 {
+			for _, ref := range refs {
+				tagLogger := logrus.WithFields(logrus.Fields{"ref": ref})
+				var named reference.Named
+				// first try as digest
+				if d, err := digest.Parse(ref); err == nil {
+					named, err = reference.WithDigest(repoRef, d)
+					if err != nil {
+						tagLogger.Error("Error processing ref, skipping")
+						logrus.Error(err)
+						continue
+					}
+				} else {
+					tagLogger.Debugf("Ref was not a digest, trying as a tag: %s", err)
+					named, err = reference.WithTag(repoRef, ref)
+					if err != nil {
+						tagLogger.Error("Error parsing ref, skipping")
+						logrus.Error(err)
+						continue
+					}
 				}
-				imageRef, err := docker.NewReference(taggedRef)
+
+				imageRef, err := docker.NewReference(named)
 				if err != nil {
-					tagLogger.Error("Error processing tag, skipping")
+					tagLogger.Error("Error processing ref, skipping")
 					logrus.Errorf("Error getting image reference: %s", err)
 					continue
 				}
 				sourceReferences = append(sourceReferences, imageRef)
 			}
-		} else { // len(tags) == 0
+		} else { // len(refs) == 0
 			repoLogger.Info("Querying registry for image tags")
 			sourceReferences, err = imagesToCopyFromRepo(serverCtx, repoRef)
 			if err != nil {
@@ -323,12 +339,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 		}
 
 		if len(sourceReferences) == 0 {
-			repoLogger.Warnf("No tags to sync found")
+			repoLogger.Warnf("No refs to sync found")
 			continue
 		}
 		repoDescList = append(repoDescList, repoDescriptor{
-			TaggedImages: sourceReferences,
-			Context:      serverCtx})
+			ImageRefs: sourceReferences,
+			Context:   serverCtx})
 	}
 
 	for imageName, tagRegex := range cfg.ImagesByTagRegex {
@@ -377,12 +393,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
 		}
 
 		if len(sourceReferences) == 0 {
-			repoLogger.Warnf("No tags to sync found")
+			repoLogger.Warnf("No refs to sync found")
 			continue
 		}
 		repoDescList = append(repoDescList, repoDescriptor{
-			TaggedImages: sourceReferences,
-			Context:      serverCtx})
+			ImageRefs: sourceReferences,
+			Context:   serverCtx})
 	}
 
 	return repoDescList, nil
@@ -415,13 +431,13 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
 			if err != nil {
 				return nil, errors.Wrapf(err, "Cannot obtain a valid image reference for transport %q and reference %q", docker.Transport.Name(), named.String())
 			}
-			desc.TaggedImages = []types.ImageReference{srcRef}
+			desc.ImageRefs = []types.ImageReference{srcRef}
 		} else {
-			desc.TaggedImages, err = imagesToCopyFromRepo(sourceCtx, named)
+			desc.ImageRefs, err = imagesToCopyFromRepo(sourceCtx, named)
 			if err != nil {
 				return descriptors, err
 			}
-			if len(desc.TaggedImages) == 0 {
+			if len(desc.ImageRefs) == 0 {
 				return descriptors, errors.Errorf("No images to sync found in %q", source)
 			}
 		}
@@ -437,11 +453,11 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
 		}
 		desc.DirBasePath = source
 		var err error
-		desc.TaggedImages, err = imagesToCopyFromDir(source)
+		desc.ImageRefs, err = imagesToCopyFromDir(source)
 		if err != nil {
 			return descriptors, err
 		}
-		if len(desc.TaggedImages) == 0 {
+		if len(desc.ImageRefs) == 0 {
 			return descriptors, errors.Errorf("No images to sync found in %q", source)
 		}
 		descriptors = append(descriptors, desc)
@@ -509,6 +525,11 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
 		return errors.New("sync from 'dir' to 'dir' not implemented, consider using rsync instead")
 	}
 
+	imageListSelection := copy.CopySystemImage
+	if opts.all {
+		imageListSelection = copy.CopyAllImages
+	}
+
 	sourceCtx, err := opts.srcImage.newSystemContext()
 	if err != nil {
 		return err
@@ -534,15 +555,16 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
 
 	imagesNumber := 0
 	options := copy.Options{
-		RemoveSignatures: opts.removeSignatures,
-		SignBy:           opts.signByFingerprint,
-		ReportWriter:     os.Stdout,
-		DestinationCtx:   destinationCtx,
+		RemoveSignatures:   opts.removeSignatures,
+		SignBy:             opts.signByFingerprint,
+		ReportWriter:       os.Stdout,
+		DestinationCtx:     destinationCtx,
+		ImageListSelection: imageListSelection,
 	}
 
 	for _, srcRepo := range srcRepoList {
 		options.SourceCtx = srcRepo.Context
-		for counter, ref := range srcRepo.TaggedImages {
+		for counter, ref := range srcRepo.ImageRefs {
 			var destSuffix string
 			switch ref.Transport() {
 			case docker.Transport:
@@ -569,13 +591,13 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
 			logrus.WithFields(logrus.Fields{
 				"from": transports.ImageName(ref),
 				"to":   transports.ImageName(destRef),
-			}).Infof("Copying image tag %d/%d", counter+1, len(srcRepo.TaggedImages))
+			}).Infof("Copying image ref %d/%d", counter+1, len(srcRepo.ImageRefs))
 
 			if err = retry.RetryIfNecessary(ctx, func() error {
 				_, err = copy.Image(ctx, policyContext, destRef, ref, &options)
 				return err
 			}, opts.retryOpts); err != nil {
-				return errors.Wrapf(err, "Error copying tag %q", transports.ImageName(ref))
+				return errors.Wrapf(err, "Error copying ref %q", transports.ImageName(ref))
 			}
 			imagesNumber++
 		}

docs/skopeo-sync.1.md

@@ -32,6 +32,11 @@ When the `--scoped` option is specified, images are prefixed with the source ima
 name can be stored at _destination_.
 
 ## OPTIONS
+**--all**
+If one of the images in __src__ refers to a list of images, instead of copying just the image which matches the current OS and
+architecture (subject to the use of the global --override-os, --override-arch and --override-variant options), attempt to copy all of
+the images in the list, and the list itself.
+
 **--authfile** _path_
 
 Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `skopeo login`.
@@ -147,6 +152,7 @@ registry.example.com:
         redis:
             - "1.0"
             - "2.0"
+            - "sha256:0000000000000000000000000000000011111111111111111111111111111111"
     images-by-tag-regex:
         nginx: ^1\.13\.[12]-alpine-perl$
     credentials:
@@ -166,7 +172,7 @@ skopeo sync --src yaml --dest docker sync.yml my-registry.local.lan/repo/
 ```
 This will copy the following images:
 - Repository `registry.example.com/busybox`: all images, as no tags are specified.
-- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0".
+- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0" along with image with digest "sha256:0000000000000000000000000000000011111111111111111111111111111111".
 - Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl".
 - Repository `quay.io/coreos/etcd`: images tagged "latest".
 

integration/sync_test.go

@@ -117,6 +117,34 @@ func (s *SyncSuite) TestDocker2DirTagged(c *check.C) {
 	c.Assert(out, check.Equals, "")
 }
 
+func (s *SyncSuite) TestDocker2DirTaggedAll(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(tmpDir)
+
+	// FIXME: It would be nice to use one of the local Docker registries instead of neeeding an Internet connection.
+	image := "busybox:latest"
+	imageRef, err := docker.ParseReference(fmt.Sprintf("//%s", image))
+	c.Assert(err, check.IsNil)
+	imagePath := imageRef.DockerReference().String()
+
+	dir1 := path.Join(tmpDir, "dir1")
+	dir2 := path.Join(tmpDir, "dir2")
+
+	// sync docker => dir
+	assertSkopeoSucceeds(c, "", "sync", "--all", "--scoped", "--src", "docker", "--dest", "dir", image, dir1)
+	_, err = os.Stat(path.Join(dir1, imagePath, "manifest.json"))
+	c.Assert(err, check.IsNil)
+
+	// copy docker => dir
+	assertSkopeoSucceeds(c, "", "copy", "--all", "docker://"+image, "dir:"+dir2)
+	_, err = os.Stat(path.Join(dir2, "manifest.json"))
+	c.Assert(err, check.IsNil)
+
+	out := combinedOutputOfCommand(c, "diff", "-urN", path.Join(dir1, imagePath), dir2)
+	c.Assert(out, check.Equals, "")
+}
+
 func (s *SyncSuite) TestScoped(c *check.C) {
 	// FIXME: It would be nice to use one of the local Docker registries instead of needing an Internet connection.
 	image := "busybox:latest"
@@ -289,6 +317,37 @@ docker.io:
 	c.Assert(nManifests, check.Equals, nTags)
 }
 
+func (s *SyncSuite) TestYamlDigest2Dir(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
+	c.Assert(err, check.IsNil)
+	defer os.RemoveAll(tmpDir)
+	dir1 := path.Join(tmpDir, "dir1")
+
+	yamlConfig := `
+docker.io:
+  images:
+    redis:
+    - sha256:61ce79d60150379787d7da677dcb89a7a047ced63406e29d6b2677b2b2163e92
+`
+	yamlFile := path.Join(tmpDir, "registries.yaml")
+	ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644)
+	assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "dir", yamlFile, dir1)
+
+	nManifests := 0
+	err = filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && info.Name() == "manifest.json" {
+			nManifests++
+			return filepath.SkipDir
+		}
+		return nil
+	})
+	c.Assert(err, check.IsNil)
+	c.Assert(nManifests, check.Equals, 1)
+}
+
 func (s *SyncSuite) TestYaml2Dir(c *check.C) {
 	tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
 	c.Assert(err, check.IsNil)