Merge pull request #741 from nalind/manifest-lists

copy: add a --all/-a flag

cmd/skopeo/copy.go

@@ -24,7 +24,7 @@ type copyOptions struct {
 	signByFingerprint string          // Sign the image using a GPG key with the specified fingerprint
 	format            optionalString  // Force conversion of the image to a specified format
 	quiet             bool            // Suppress output information when copying images
-
+	all               bool            // Copy all of the images if the source is a list
 }
 
 func copyCmd(global *globalOptions) cli.Command {
@@ -62,6 +62,11 @@ func copyCmd(global *globalOptions) cli.Command {
 				Usage:       "Suppress output information when copying images",
 				Destination: &opts.quiet,
 			},
+			cli.BoolFlag{
+				Name:        "all, a",
+				Usage:       "Copy all images if SOURCE-IMAGE is a list",
+				Destination: &opts.all,
+			},
 			cli.BoolFlag{
 				Name:        "remove-signatures",
 				Usage:       "Do not copy signatures from SOURCE-IMAGE",
@@ -147,13 +152,18 @@ func (opts *copyOptions) run(args []string, stdout io.Writer) error {
 	if opts.quiet {
 		stdout = nil
 	}
+	imageListSelection := copy.CopySystemImage
+	if opts.all {
+		imageListSelection = copy.CopyAllImages
+	}
 	_, err = copy.Image(ctx, policyContext, destRef, srcRef, &copy.Options{
 		RemoveSignatures:      opts.removeSignatures,
 		SignBy:                opts.signByFingerprint,
 		ReportWriter:          stdout,
 		SourceCtx:             sourceCtx,
 		DestinationCtx:        destinationCtx,
 		ForceManifestMIMEType: manifestType,
+		ImageListSelection:    imageListSelection,
 	})
 	return err
 }

cmd/skopeo/layers.go

@@ -142,9 +142,9 @@ func (opts *layersOptions) run(args []string, stdout io.Writer) (retErr error) {
 	if err != nil {
 		return err
 	}
-	if err := dest.PutManifest(ctx, manifest); err != nil {
+	if err := dest.PutManifest(ctx, manifest, nil); err != nil {
 		return err
 	}
 
-	return dest.Commit(ctx)
+	return dest.Commit(ctx, image.UnparsedInstance(rawSource, nil))
 }

completions/bash/skopeo

@@ -51,6 +51,7 @@ _skopeo_copy() {
     "
 
     local boolean_options="
+    --all
     --dest-compress
     --remove-signatures
     --src-no-creds

docs/skopeo-copy.1.md

@@ -17,6 +17,12 @@ Uses the system's trust policy to validate images, rejects images not trusted by
 
 ## OPTIONS
 
+**--all**
+
+If _source-image_ refers to a list of images, instead of copying just the image which matches the current OS and
+architecture (subject to the use of the global --override-os and --override-arch options), attempt to copy all of
+the images in the list, and the list itself.
+
 **--authfile** _path_
 
 Path of the authentication file. Default is ${XDG_RUNTIME\_DIR}/containers/auth.json, which is set using `podman login`.

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/containers/buildah v1.8.4
-	github.com/containers/image/v4 v4.0.1
+	github.com/containers/image/v4 v4.0.2-0.20191021195858-69340234bfc6
 	github.com/containers/storage v1.13.4
 	github.com/docker/docker v0.0.0-20180522102801-da99009bbb11
 	github.com/dsnet/compress v0.0.1 // indirect

go.sum

@@ -15,6 +15,8 @@ github.com/containers/buildah v1.8.4 h1:06c+UNeEWMa2wA1Z7muZ0ZqUzE91sDuZJbB0BiZa
 github.com/containers/buildah v1.8.4/go.mod h1:1CsiLJvyU+h+wOjnqJJOWuJCVcMxZOr5HN/gHGdzJxY=
 github.com/containers/image/v4 v4.0.1 h1:idNGHChj0Pyv3vLrxul2oSVMZLeFqpoq3CjLeVgapSQ=
 github.com/containers/image/v4 v4.0.1/go.mod h1:0ASJH1YgJiX/eqFZObqepgsvIA4XjCgpyfwn9pDGafA=
+github.com/containers/image/v4 v4.0.2-0.20191021195858-69340234bfc6 h1:sFL2cwC0xjphJHpa6DXhka2jTLGI5HwbnAUSAKFhg2M=
+github.com/containers/image/v4 v4.0.2-0.20191021195858-69340234bfc6/go.mod h1:0ASJH1YgJiX/eqFZObqepgsvIA4XjCgpyfwn9pDGafA=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/storage v1.13.4 h1:j0bBaJDKbUHtAW1MXPFnwXJtqcH+foWeuXK1YaBV5GA=

integration/decompress-dirs.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+# Account for differences between dir: images that are solely due to one being
+# compressed (fresh from a registry) and the other not being compressed (read
+# from storage, which decompressed it and had to reassemble the layer blobs).
+for dir in "$@" ; do
+    # Updating the manifest's blob digests may change the formatting, so
+    # use jq to get them into similar shape.
+    jq -M . "${dir}"/manifest.json > "${dir}"/manifest.json.tmp && mv "${dir}"/manifest.json.tmp "${dir}"/manifest.json
+    for candidate in "${dir}"/???????????????????????????????????????????????????????????????? ; do
+        # If a digest-identified file looks like it was compressed,
+        # decompress it, and replace its hash and size in the manifest
+        # with the values for their decompressed versions.
+        uncompressed=`zcat "${candidate}" 2> /dev/null | sha256sum | cut -c1-64`
+        if test $? -eq 0 ; then
+            if test "$uncompressed" != e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ; then
+                zcat "${candidate}" > "${dir}"/${uncompressed}
+                sed -r -i -e "s#sha256:$(basename ${candidate})#sha256:${uncompressed}#g" "${dir}"/manifest.json
+                sed -r -i -e "s#\"size\": $(wc -c < ${candidate}),#\"size\": $(wc -c < ${dir}/${uncompressed}),#g" "${dir}"/manifest.json
+                rm -f "${candidate}"
+            fi
+        fi
+    done
+done

integration/utils.go

@@ -6,13 +6,15 @@ import (
 	"io/ioutil"
 	"net"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/go-check/check"
 )
 
 const skopeoBinary = "skopeo"
+const decompressDirsBinary = "./decompress-dirs.sh"
 
 // consumeAndLogOutputStream takes (f, err) from an exec.*Pipe(), and causes all output to it to be logged to c.
 func consumeAndLogOutputStream(c *check.C, id string, f io.ReadCloser, err error) {
@@ -174,3 +176,27 @@ func fileFromFixture(c *check.C, inputPath string, edits map[string]string) stri
 	c.Assert(err, check.IsNil)
 	return path
 }
+
+// runDecompressDirs runs decompress-dirs.sh using exec.Command().CombinedOutput, verifies that the exit status is 0,
+// and optionally that the output matches a multi-line regexp if it is nonempty; or terminates c on failure
+func runDecompressDirs(c *check.C, regexp string, args ...string) {
+	c.Logf("Running %s %s", decompressDirsBinary, strings.Join(args, " "))
+	for i, dir := range args {
+		m, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json"))
+		c.Assert(err, check.IsNil)
+		c.Logf("manifest %d before: %s", i+1, string(m))
+	}
+	out, err := exec.Command(decompressDirsBinary, args...).CombinedOutput()
+	c.Assert(err, check.IsNil, check.Commentf("%s", out))
+	for i, dir := range args {
+		if len(out) > 0 {
+			c.Logf("output: %s", out)
+		}
+		m, err := ioutil.ReadFile(filepath.Join(dir, "manifest.json"))
+		c.Assert(err, check.IsNil)
+		c.Logf("manifest %d after: %s", i+1, string(m))
+	}
+	if regexp != "" {
+		c.Assert(string(out), check.Matches, "(?s)"+regexp) // (?s) : '.' will also match newlines
+	}
+}