Implement annotation-based rebase hints

`crane rebase` now inspects the original image for annotations to
identify its old base and new base images. If those are found, and
--old_base and --new_base flags aren't specified, those will be used.

If the --rebased flag is not passed, the rebased image will be the
tagged with the original image reference; if it was originally specified
by digest, the rebased image will be tagged with :rebased (instead of
stripping the digest and pushing to :latest)

`crane rebase` now prints the full pushed image reference, instead of
just the digest (aiding embedding in other bash pipelines), and adds
annotations to aid future rebasings.

This also adds a bash test that covers the rebase case for detected base
image information.

cmd/crane/cmd/rebase.go

@@ -15,10 +15,17 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/logs"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	specsv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 )
 
@@ -29,48 +36,191 @@ func NewCmdRebase(options *[]crane.Option) *cobra.Command {
 	rebaseCmd := &cobra.Command{
 		Use:   "rebase",
 		Short: "Rebase an image onto a new base image",
-		Args:  cobra.NoArgs,
-		RunE: func(*cobra.Command, []string) error {
-			origImg, err := crane.Pull(orig, *options...)
+		Args:  cobra.MaximumNArgs(1),
+		RunE: func(_ *cobra.Command, args []string) error {
+			if orig == "" {
+				orig = args[0]
+			} else if len(args) != 0 || args[0] != "" {
+				return fmt.Errorf("cannot use --original with positional argument")
+			}
+
+			// Parse and fetch the original image or index.
+			origRef, err := name.ParseReference(orig)
+			if err != nil {
+				return fmt.Errorf("parsing tag %q: %v", orig, err)
+			}
+			origDesc, err := remote.Get(origRef)
+			if err != nil {
+				return err
+			}
+			if origDesc.Descriptor.MediaType.IsIndex() {
+				return errors.New("rebasing indexes is not currently supported")
+			}
+
+			// TODO: This will bias toward rebasing the linux/amd64
+			// image, instead of the whole index.  Support rebasing
+			// indexes, and support the --platform flag here.
+			origImg, err := origDesc.Image()
+			if err != nil {
+				return err
+			}
+			origMf, err := origImg.Manifest()
+			if err != nil {
+				return err
+			}
+			anns := origMf.Annotations
+			if newBase == "" && anns != nil {
+				newBase = anns[specsv1.AnnotationBaseImageName]
+			}
+			if newBase == "" {
+				return errors.New("could not determine new base image from annotations")
+			}
+			newBaseRef, err := name.ParseReference(newBase)
 			if err != nil {
-				return fmt.Errorf("pulling %s: %v", orig, err)
+				return err
+			}
+			if oldBase == "" && anns != nil {
+				oldBaseDigest := anns[specsv1.AnnotationBaseImageDigest]
+				oldBase = newBaseRef.Context().Digest(oldBaseDigest).String()
+			}
+			if oldBase == "" {
+				return errors.New("could not determine old base image by digest from annotations")
 			}
 
-			oldBaseImg, err := crane.Pull(oldBase, *options...)
+			rebasedImg, err := rebaseImage(origImg, oldBase, newBase, *options...)
 			if err != nil {
-				return fmt.Errorf("pulling %s: %v", oldBase, err)
+				return fmt.Errorf("rebasing image: %v", err)
 			}
 
-			newBaseImg, err := crane.Pull(newBase, *options...)
+			// If the new ref isn't provided, write over the original image.
+			// If that ref was provided by digest (e.g., output from
+			// another crane command), then strip that and push to
+			// a ":rebased" tag instead.
+			if rebased == "" {
+				if _, ok := origRef.(name.Digest); ok {
+					rebased = origRef.Context().Tag("rebased").String()
+				} else {
+					rebased = orig
+				}
+			}
+			logs.Progress.Println("pushing rebased image as", rebased)
+			rebasedDigest, err := rebasedImg.Digest()
 			if err != nil {
-				return fmt.Errorf("pulling %s: %v", newBase, err)
+				return fmt.Errorf("digesting new image: %v", err)
 			}
+			origDigest, err := origImg.Digest()
+			if err != nil {
+				return err
+			}
+			if rebasedDigest == origDigest {
+				logs.Warn.Println("rebasing was no-op")
+			} else {
+				logs.Debug.Println("rebased digest: ", rebasedDigest)
+				logs.Debug.Println("original digest:", origDigest)
 
-			img, err := mutate.Rebase(origImg, oldBaseImg, newBaseImg)
+				origManifest, _ := origImg.Manifest()
+				rebasedManifest, _ := rebasedImg.Manifest()
+				logs.Debug.Println("DIFF:", cmp.Diff(origManifest, rebasedManifest))
+			}
+
+			r, err := name.ParseReference(rebased)
 			if err != nil {
 				return fmt.Errorf("rebasing: %v", err)
 			}
 
-			if err := crane.Push(img, rebased, *options...); err != nil {
+			if err := crane.Push(rebasedImg, rebased, *options...); err != nil {
 				return fmt.Errorf("pushing %s: %v", rebased, err)
 			}
+			if _, ok := r.(name.Digest); ok {
+				rebased = r.Context().Digest(rebasedDigest.String()).String()
+				if err := crane.Push(rebasedImg, rebased, *options...); err != nil {
+					return fmt.Errorf("pushing %s: %v", rebased, err)
+				}
+			}
 
-			digest, err := img.Digest()
+			rebasedRef, err := name.ParseReference(rebased)
 			if err != nil {
-				return fmt.Errorf("digesting rebased: %v", err)
+				return fmt.Errorf("parsing %q: %v", rebased, err)
 			}
-			fmt.Println(digest.String())
+
+			fmt.Println(rebasedRef.Context().Digest(rebasedDigest.String()))
 			return nil
 		},
 	}
-	rebaseCmd.Flags().StringVarP(&orig, "original", "", "", "Original image to rebase")
-	rebaseCmd.Flags().StringVarP(&oldBase, "old_base", "", "", "Old base image to remove")
-	rebaseCmd.Flags().StringVarP(&newBase, "new_base", "", "", "New base image to insert")
-	rebaseCmd.Flags().StringVarP(&rebased, "rebased", "", "", "Tag to apply to rebased image")
-
-	rebaseCmd.MarkFlagRequired("original")
-	rebaseCmd.MarkFlagRequired("old_base")
-	rebaseCmd.MarkFlagRequired("new_base")
-	rebaseCmd.MarkFlagRequired("rebased")
+	rebaseCmd.Flags().StringVar(&orig, "original", "", "Original image to rebase; use positional arg instead")
+	rebaseCmd.Flags().StringVar(&oldBase, "old_base", "", "Old base image to remove")
+	rebaseCmd.Flags().StringVar(&newBase, "new_base", "", "New base image to insert")
+	rebaseCmd.Flags().StringVarP(&rebased, "tag", "t", "", "Tag to apply to rebased image")
 	return rebaseCmd
 }
+
+// rebaseImage parses the references and uses them to perform a rebase on the
+// original image.
+//
+// If oldBase or newBase are "", rebaseImage attempts to derive them using
+// annotations in the original image. If those annotations are not found,
+// rebaseImage returns an error.
+//
+// If rebasing is successful, base image annotations are set on the resulting
+// image to facilitate implicit rebasing next time.
+func rebaseImage(orig v1.Image, oldBase, newBase string, opt ...crane.Option) (v1.Image, error) {
+	m, err := orig.Manifest()
+	if err != nil {
+		return nil, err
+	}
+	if newBase == "" && m.Annotations != nil {
+		newBase = m.Annotations[specsv1.AnnotationBaseImageName]
+		if newBase != "" {
+			logs.Debug.Printf("Detected new base from %q annotation: %s", specsv1.AnnotationBaseImageName, newBase)
+		}
+	}
+	if newBase == "" {
+		return nil, fmt.Errorf("either new base or %q annotation is required", specsv1.AnnotationBaseImageName)
+	}
+	newBaseImg, err := crane.Pull(newBase, opt...)
+	if err != nil {
+		return nil, err
+	}
+
+	if oldBase == "" && m.Annotations != nil {
+		oldBase = m.Annotations[specsv1.AnnotationBaseImageDigest]
+		if oldBase != "" {
+			newBaseRef, err := name.ParseReference(newBase)
+			if err != nil {
+				return nil, err
+			}
+
+			oldBase = newBaseRef.Context().Digest(oldBase).String()
+			logs.Debug.Printf("Detected old base from %q annotation: %s", specsv1.AnnotationBaseImageDigest, oldBase)
+		}
+	}
+	if oldBase == "" {
+		return nil, fmt.Errorf("either old base or %q annotation is required", specsv1.AnnotationBaseImageDigest)
+	}
+
+	oldBaseImg, err := crane.Pull(oldBase, opt...)
+	if err != nil {
+		return nil, err
+	}
+
+	// NB: if newBase is an index, we need to grab the index's digest to
+	// annotate the resulting image, even though we pull the
+	// platform-specific image to rebase.
+	newBaseDigest, err := crane.Digest(newBase, opt...)
+	if err != nil {
+		return nil, err
+	}
+
+	rebased, err := mutate.Rebase(orig, oldBaseImg, newBaseImg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Update base image annotations for the new image manifest.
+	logs.Debug.Printf("Setting annotation %q: %q", specsv1.AnnotationBaseImageDigest, newBaseDigest)
+	logs.Debug.Printf("Setting annotation %q: %q", specsv1.AnnotationBaseImageName, newBase)
+	return mutate.Annotations(rebased, map[string]string{
+		specsv1.AnnotationBaseImageDigest: newBaseDigest,
+		specsv1.AnnotationBaseImageName:   newBase,
+	}).(v1.Image), nil
+}

cmd/crane/rebase_test.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -ex
+
+tmp=$(mktemp -d)
+
+go install ./cmd/registry
+go build -o ./crane ./cmd/crane
+
+# Start a local registry.
+registry &
+PID=$!
+function cleanup {
+  kill $PID
+  rm -r ${tmp}
+  rm ./crane
+}
+trap cleanup EXIT
+
+sleep 1  # Wait for registry to be up.
+
+# Create an image localhost:1338/base containing a.txt
+echo a > ${tmp}/a.txt
+./crane append -f <(tar -f - -c ${tmp}) -t localhost:1338/base
+rm ${tmp}/a.txt
+
+# Append to that image localhost:1338/rebaseme
+echo top > ${tmp}/top.txt
+./crane append -f <(tar -f - -c ${tmp}) -b localhost:1338/base -t localhost:1338/rebaseme
+rm ${tmp}/top.txt
+
+# Annotate that image as the base image (by ref and digest)
+# TODO: do this with a flag to --append
+./crane mutate localhost:1338/rebaseme \
+  --annotation org.opencontainers.image.base.name=localhost:1338/base \
+  --annotation org.opencontainers.image.base.digest=$(./crane digest localhost:1338/base) \
+  --tag localhost:1338/rebaseme
+
+# Update localhost:1338/base containing b.txt
+echo b > ${tmp}/b.txt
+./crane append -f <(tar -f - -c ${tmp}) -t localhost:1338/base
+rm ${tmp}/b.txt
+
+# Rebase using annotations
+rebased=$(./crane rebase localhost:1338/rebaseme)
+
+# List files in the rebased image.
+./crane export ${rebased} - | tar -tvf -
+
+# Extract b.txt out of the rebased image
+./crane export ${rebased} - | tar -Oxf - ${tmp:1}/b.txt
+
+# Extract top.txt out of the rebased image
+./crane export ${rebased} - | tar -Oxf - ${tmp:1}/top.txt
+
+# a.txt is _not_ in the rebased image.
+set +e
+./crane export ${rebased} - | tar -Oxf - ${tmp:1}/a.txt  # this should fail
+code=$?
+echo "finding a.txt exited ${code}"
+if [[ $code -ne 1 ]]; then
+  echo "a.txt was found in rebased image"
+  exit 1
+fi
+

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
-	github.com/opencontainers/image-spec v1.0.1
+	github.com/opencontainers/image-spec v1.0.2-0.20210730191737-8e42a01fb1b7
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/cobra v1.2.1

pkg/v1/mutate/rebase.go

@@ -46,6 +46,7 @@ func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) {
 			return nil, fmt.Errorf("failed to get digest of layer %d of %q: %v", i, orig, err)
 		}
 		if oldLayerDigest != origLayerDigest {
+			// TODO: this is a bad error message...
 			return nil, fmt.Errorf("image %q is not based on %q (layer %d mismatch)", orig, oldBase, i)
 		}
 	}