Annotate images with base image information (#354)

* WIP: annotate base images

* remove TODO

* .

* Annotate with base index digest, if based on an index

* use correct new proposed annotation

go.mod

@@ -3,31 +3,26 @@ module github.com/google/ko
 go 1.15
 
 require (
-	github.com/containerd/stargz-snapshotter/estargz v0.4.1
-	github.com/docker/cli v0.0.0-20200303162255-7d407207c304 // indirect
-	github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7
+	github.com/containerd/stargz-snapshotter/estargz v0.6.4
+	github.com/docker/docker v20.10.7+incompatible
 	github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-training/helloworld v0.0.0-20200225145412-ba5f4379d78b
-	github.com/google/go-cmp v0.5.4
-	github.com/google/go-containerregistry v0.5.0
+	github.com/google/go-cmp v0.5.6
+	github.com/google/go-containerregistry v0.5.2-0.20210622202051-acad0ede73ff
 	github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
 	github.com/mattmoor/dep-notify v0.0.0-20190205035814-a45dec370a17
 	github.com/onsi/gomega v1.10.3 // indirect
-	github.com/spf13/cobra v1.0.0
+	github.com/spf13/cobra v1.1.3
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.4.0
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
-	golang.org/x/mod v0.4.1 // indirect
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
-	golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65 // indirect
-	golang.org/x/tools v0.1.0
+	github.com/spf13/viper v1.7.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/tools v0.1.2
 	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
-	gotest.tools/v3 v3.0.2 // indirect
-	k8s.io/apimachinery v0.19.7
-	k8s.io/cli-runtime v0.19.7
+	k8s.io/apimachinery v0.20.6
+	k8s.io/cli-runtime v0.20.6
 	k8s.io/klog/v2 v2.5.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect
 	sigs.k8s.io/kind v0.8.1

pkg/build/gobuild.go

@@ -36,6 +36,7 @@ import (
 	"text/template"
 
 	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
@@ -65,8 +66,13 @@ For more information see:
 `
 )
 
-// GetBase takes an importpath and returns a base image.
-type GetBase func(context.Context, string) (Result, error)
+const (
+	baseDigestAnnotation = "org.opencontainers.image.base.digest"
+	baseRefAnnotation    = "org.opencontainers.image.base.name"
+)
+
+// GetBase takes an importpath and returns a base image reference and base image (or index).
+type GetBase func(context.Context, string) (name.Reference, Result, error)
 
 type builder func(context.Context, string, string, v1.Platform, Config) (string, error)
 
@@ -652,8 +658,8 @@ func (g *gobuild) configForImportPath(ip string) Config {
 	return config
 }
 
-func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platform *v1.Platform) (v1.Image, error) {
-	ref := newRef(s)
+func (g *gobuild) buildOne(ctx context.Context, refStr string, baseRef name.Reference, baseDigest v1.Hash, base v1.Image, platform *v1.Platform) (v1.Image, error) {
+	ref := newRef(refStr)
 
 	cf, err := base.ConfigFile()
 	if err != nil {
@@ -728,6 +734,11 @@ func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platfor
 		return nil, err
 	}
 
+	withApp = mutate.Annotations(withApp, map[string]string{
+		baseRefAnnotation:    baseRef.Name(),
+		baseDigestAnnotation: baseDigest.String(),
+	})
+
 	// Start from a copy of the base image's config file, and set
 	// the entrypoint to our app.
 	cfg, err := withApp.ConfigFile()
@@ -784,7 +795,7 @@ func updatePath(cf *v1.ConfigFile) {
 // Build implements build.Interface
 func (g *gobuild) Build(ctx context.Context, s string) (Result, error) {
 	// Determine the appropriate base image for this import path.
-	base, err := g.getBase(ctx, s)
+	baseRef, base, err := g.getBase(ctx, s)
 	if err != nil {
 		return nil, err
 	}
@@ -795,27 +806,33 @@ func (g *gobuild) Build(ctx context.Context, s string) (Result, error) {
 		return nil, err
 	}
 
+	// Take the digest of the base index or image, to annotate images we'll build later.
+	baseDigest, err := base.Digest()
+	if err != nil {
+		return nil, err
+	}
+
 	switch mt {
 	case types.OCIImageIndex, types.DockerManifestList:
-		base, ok := base.(v1.ImageIndex)
+		baseIndex, ok := base.(v1.ImageIndex)
 		if !ok {
 			return nil, fmt.Errorf("failed to interpret base as index: %v", base)
 		}
-		return g.buildAll(ctx, s, base)
+		return g.buildAll(ctx, s, baseRef, baseDigest, baseIndex)
 	case types.OCIManifestSchema1, types.DockerManifestSchema2:
-		base, ok := base.(v1.Image)
+		baseImage, ok := base.(v1.Image)
 		if !ok {
 			return nil, fmt.Errorf("failed to interpret base as image: %v", base)
 		}
-		return g.buildOne(ctx, s, base, nil)
+		return g.buildOne(ctx, s, baseRef, baseDigest, baseImage, nil)
 	default:
 		return nil, fmt.Errorf("base image media type: %s", mt)
 	}
 }
 
 // TODO(#192): Do these in parallel?
-func (g *gobuild) buildAll(ctx context.Context, s string, base v1.ImageIndex) (v1.ImageIndex, error) {
-	im, err := base.IndexManifest()
+func (g *gobuild) buildAll(ctx context.Context, ref string, baseRef name.Reference, baseDigest v1.Hash, baseIndex v1.ImageIndex) (v1.ImageIndex, error) {
+	im, err := baseIndex.IndexManifest()
 	if err != nil {
 		return nil, err
 	}
@@ -825,18 +842,18 @@ func (g *gobuild) buildAll(ctx context.Context, s string, base v1.ImageIndex) (v
 	for _, desc := range im.Manifests {
 		// Nested index is pretty rare. We could support this in theory, but return an error for now.
 		if desc.MediaType != types.OCIManifestSchema1 && desc.MediaType != types.DockerManifestSchema2 {
-			return nil, fmt.Errorf("%q has unexpected mediaType %q in base for %q", desc.Digest, desc.MediaType, s)
+			return nil, fmt.Errorf("%q has unexpected mediaType %q in base for %q", desc.Digest, desc.MediaType, ref)
 		}
 
 		if !g.platformMatcher.matches(desc.Platform) {
 			continue
 		}
 
-		base, err := base.Image(desc.Digest)
+		baseImage, err := baseIndex.Image(desc.Digest)
 		if err != nil {
 			return nil, err
 		}
-		img, err := g.buildOne(ctx, s, base, desc.Platform)
+		img, err := g.buildOne(ctx, ref, baseRef, baseDigest, baseImage, desc.Platform)
 		if err != nil {
 			return nil, err
 		}
@@ -851,7 +868,7 @@ func (g *gobuild) buildAll(ctx context.Context, s string, base v1.ImageIndex) (v
 		})
 	}
 
-	baseType, err := base.MediaType()
+	baseType, err := baseIndex.MediaType()
 	if err != nil {
 		return nil, err
 	}

pkg/build/gobuild_test.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
@@ -110,7 +111,7 @@ func TestGoBuildQualifyImport(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
-			ng, err := NewGo(context.Background(), test.dir, WithBaseImages(func(context.Context, string) (Result, error) { return base, nil }))
+			ng, err := NewGo(context.Background(), test.dir, WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return nil, base, nil }))
 			if err != nil {
 				t.Fatalf("NewGo() = %v", err)
 			}
@@ -131,13 +132,15 @@ func TestGoBuildQualifyImport(t *testing.T) {
 	}
 }
 
+var baseRef = name.MustParseReference("all.your/base")
+
 func TestGoBuildIsSupportedRef(t *testing.T) {
 	base, err := random.Image(1024, 3)
 	if err != nil {
 		t.Fatalf("random.Image() = %v", err)
 	}
 
-	ng, err := NewGo(context.Background(), "", WithBaseImages(func(context.Context, string) (Result, error) { return base, nil }))
+	ng, err := NewGo(context.Background(), "", WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return nil, base, nil }))
 	if err != nil {
 		t.Fatalf("NewGo() = %v", err)
 	}
@@ -186,7 +189,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
 	}
 
 	opts := []Option{
-		WithBaseImages(func(context.Context, string) (Result, error) { return base, nil }),
+		WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, base, nil }),
 		withModuleInfo(mods),
 		withBuildContext(stubBuildContext{
 			// make all referenced deps commands
@@ -259,7 +262,7 @@ func TestGoBuildNoKoData(t *testing.T) {
 		context.Background(),
 		"",
 		WithCreationTime(creationTime),
-		WithBaseImages(func(context.Context, string) (Result, error) { return base, nil }),
+		WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, base, nil }),
 		withBuilder(writeTempFile),
 	)
 	if err != nil {
@@ -477,6 +480,21 @@ func validateImage(t *testing.T, img v1.Image, baseLayers int64, creationTime v1
 			t.Errorf("created = %v, want %v", actual, creationTime)
 		}
 	})
+
+	t.Run("check annotations", func(t *testing.T) {
+		mf, err := img.Manifest()
+		if err != nil {
+			t.Fatalf("Manifest() = %v", err)
+		}
+		t.Logf("Got annotations: %v", mf.Annotations)
+		if _, found := mf.Annotations[baseDigestAnnotation]; !found {
+			t.Errorf("image annotations did not contain base image digest")
+		}
+		want := baseRef.Name()
+		if got := mf.Annotations[baseRefAnnotation]; got != want {
+			t.Errorf("base image ref; got %q, want %q", got, want)
+		}
+	})
 }
 
 type stubBuildContext map[string]*gb.Package
@@ -502,7 +520,7 @@ func TestGoBuild(t *testing.T) {
 		context.Background(),
 		"",
 		WithCreationTime(creationTime),
-		WithBaseImages(func(context.Context, string) (Result, error) { return base, nil }),
+		WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, base, nil }),
 		withBuilder(writeTempFile),
 		WithLabel("foo", "bar"),
 		WithLabel("hello", "world"),
@@ -575,7 +593,7 @@ func TestGoBuildIndex(t *testing.T) {
 		context.Background(),
 		"",
 		WithCreationTime(creationTime),
-		WithBaseImages(func(context.Context, string) (Result, error) { return base, nil }),
+		WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, base, nil }),
 		WithPlatforms("all"),
 		withBuilder(writeTempFile),
 	)
@@ -648,7 +666,7 @@ func TestNestedIndex(t *testing.T) {
 		context.Background(),
 		"",
 		WithCreationTime(creationTime),
-		WithBaseImages(func(context.Context, string) (Result, error) { return nestedBase, nil }),
+		WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, nestedBase, nil }),
 		withBuilder(writeTempFile),
 	)
 	if err != nil {

pkg/commands/config.go

@@ -50,7 +50,7 @@ var (
 // getBaseImage returns a function that determines the base image for a given import path.
 // If the `bo.BaseImage` parameter is non-empty, it overrides base image configuration from `.ko.yaml`.
 func getBaseImage(platform string, bo *options.BuildOptions) build.GetBase {
-	return func(ctx context.Context, s string) (build.Result, error) {
+	return func(ctx context.Context, s string) (name.Reference, build.Result, error) {
 		s = strings.TrimPrefix(s, build.StrictScheme)
 		// Viper configuration file keys are case insensitive, and are
 		// returned as all lowercase.  This means that import paths with
@@ -71,12 +71,13 @@ func getBaseImage(platform string, bo *options.BuildOptions) build.GetBase {
 		}
 		ref, err := name.ParseReference(baseImage, nameOpts...)
 		if err != nil {
-			return nil, fmt.Errorf("parsing base image (%q): %v", baseImage, err)
+			return nil, nil, fmt.Errorf("parsing base image (%q): %v", baseImage, err)
 		}
 
 		// For ko.local, look in the daemon.
 		if ref.Context().RegistryStr() == publish.LocalDomain {
-			return daemon.Image(ref)
+			img, err := daemon.Image(ref)
+			return ref, img, err
 		}
 
 		userAgent := ua()
@@ -108,24 +109,27 @@ func getBaseImage(platform string, bo *options.BuildOptions) build.GetBase {
 				p.Variant = parts[2]
 			}
 			if len(parts) > 3 {
-				return nil, fmt.Errorf("too many slashes in platform spec: %s", platform)
+				return nil, nil, fmt.Errorf("too many slashes in platform spec: %s", platform)
 			}
 			ropt = append(ropt, remote.WithPlatform(p))
 		}
 
 		log.Printf("Using base %s for %s", ref, s)
 		desc, err := remote.Get(ref, ropt...)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		switch desc.MediaType {
 		case types.OCIImageIndex, types.DockerManifestList:
 			if multiplatform {
-				return desc.ImageIndex()
+				idx, err := desc.ImageIndex()
+				return ref, idx, err
 			}
-			return desc.Image()
+			img, err := desc.Image()
+			return ref, img, err
 		default:
-			return desc.Image()
+			img, err := desc.Image()
+			return ref, img, err
 		}
 	}
 }

pkg/commands/config_test.go

@@ -32,7 +32,7 @@ func TestOverrideDefaultBaseImageUsingBuildOption(t *testing.T) {
 	}
 
 	baseFn := getBaseImage("all", bo)
-	res, err := baseFn(context.Background(), "ko://example.com/helloworld")
+	_, res, err := baseFn(context.Background(), "ko://example.com/helloworld")
 	if err != nil {
 		t.Fatalf("getBaseImage(): %v", err)
 	}