From 7e9709a63a6bedc92020d0c4335e81a07b10e0d2 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 4 Mar 2022 16:21:06 -0500 Subject: [PATCH] Produce OCI images by default (#623) * Produce OCI images by default This changes build logic to prefer to produce OCI images and indexes, even if original base images are Docker manifests or manifest lists. OCI indexes support annotations, while Docker manifest lists do not, and we'd like to inject base image information in annotations wherever possible. Since Quay.io recently added support for OCI manifests, this is no longer a serious breaking change -- and anyway, producing SBOMs by default already breaks Quay.io without --sbom=none. This behavior can be disabled with --preserve-docker-media-type=true, which will result in Docker-type manifests being produced if and only if the base image was a Docker-typed manifest. This partially reverts commit 42723d75e7076c4946351c9e3197ce65ff31b4ec. * drop e2e test * update generated docs * --preserve-media-type * docs --- doc/ko_apply.md | 1 + doc/ko_build.md | 1 + doc/ko_create.md | 1 + doc/ko_resolve.md | 1 + doc/ko_run.md | 1 + pkg/build/gobuild.go | 32 ++++++++-- pkg/build/gobuild_test.go | 109 ++++++++++++++++++++++++++++++++++ pkg/build/options.go | 10 ++++ pkg/commands/options/build.go | 6 ++ pkg/commands/resolver.go | 1 + 10 files changed, 158 insertions(+), 5 deletions(-) diff --git a/doc/ko_apply.md b/doc/ko_apply.md index feeb982268..73288128c2 100644 --- a/doc/ko_apply.md +++ b/doc/ko_apply.md @@ -70,6 +70,7 @@ ko apply -f FILENAME [flags] --password string Password for basic authentication to the API server (DEPRECATED) --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (DEPRECATED) diff --git a/doc/ko_build.md b/doc/ko_build.md index cee5b5f40c..9ed0a9e364 100644 --- a/doc/ko_build.md +++ b/doc/ko_build.md @@ -55,6 +55,7 @@ ko build IMPORTPATH... [flags] --oci-layout-path string Path to save the OCI image layout of the built images --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. diff --git a/doc/ko_create.md b/doc/ko_create.md index 06615e59b3..d6d6b2a529 100644 --- a/doc/ko_create.md +++ b/doc/ko_create.md @@ -70,6 +70,7 @@ ko create -f FILENAME [flags] --password string Password for basic authentication to the API server (DEPRECATED) --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (DEPRECATED) diff --git a/doc/ko_resolve.md b/doc/ko_resolve.md index 2355229005..d046e59d89 100644 --- a/doc/ko_resolve.md +++ b/doc/ko_resolve.md @@ -51,6 +51,7 @@ ko resolve -f FILENAME [flags] --oci-layout-path string Path to save the OCI image layout of the built images --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") diff --git a/doc/ko_run.md b/doc/ko_run.md index 02b08d5923..f7a2c8aa30 100644 --- a/doc/ko_run.md +++ b/doc/ko_run.md @@ -42,6 +42,7 @@ ko run IMPORTPATH [flags] --oci-layout-path string Path to save the OCI image layout of the built images --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 9a8af12a47..960d006abe 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -78,6 +78,7 @@ type gobuild struct { sbom sbomber disableOptimizations bool trimpath bool + preserveMediaType bool buildConfigs map[string]Config platformMatcher *platformMatcher dir string @@ -99,6 +100,7 @@ type gobuildOpener struct { sbom sbomber disableOptimizations bool trimpath bool + preserveMediaType bool buildConfigs map[string]Config platforms []string labels map[string]string @@ -126,6 +128,7 @@ func (gbo *gobuildOpener) Open() (Interface, error) { sbom: gbo.sbom, disableOptimizations: gbo.disableOptimizations, trimpath: gbo.trimpath, + preserveMediaType: gbo.preserveMediaType, buildConfigs: gbo.buildConfigs, labels: gbo.labels, dir: gbo.dir, @@ -694,6 +697,16 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl ref := newRef(refStr) + baseType := types.OCIManifestSchema1 + if g.preserveMediaType { + var err error + baseType, err = base.MediaType() + if err != nil { + return nil, err + } + } + base = mutate.MediaType(base, baseType) + cf, err := base.ConfigFile() if err != nil { return nil, err @@ -884,7 +897,7 @@ func (g *gobuild) Build(ctx context.Context, s string) (Result, error) { // Annotate the base image we pass to the build function with // annotations indicating the digest (and possibly tag) of the // base image. This will be inherited by the image produced. - if mt != types.DockerManifestList { + if mt != types.DockerManifestList && !g.preserveMediaType { anns := map[string]string{ specsv1.AnnotationBaseImageDigest: baseDigest.String(), } @@ -963,11 +976,17 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIn if err != nil { return err } + + mt := types.OCIManifestSchema1 + if g.preserveMediaType { + mt = desc.MediaType + } + adds[i] = ocimutate.IndexAddendum{ Add: img, Descriptor: v1.Descriptor{ URLs: desc.URLs, - MediaType: desc.MediaType, + MediaType: mt, Annotations: desc.Annotations, Platform: desc.Platform, }, @@ -979,9 +998,12 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIn return nil, err } - baseType, err := baseIndex.MediaType() - if err != nil { - return nil, err + baseType := types.OCIImageIndex + if g.preserveMediaType { + baseType, err = baseIndex.MediaType() + if err != nil { + return nil, err + } } idx := ocimutate.AppendManifests(mutate.IndexMediaType(empty.Index, baseType), adds...) diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index 890b81afb9..7c6449ff56 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -513,6 +513,17 @@ func TestGoBuildNoKoData(t *testing.T) { t.Errorf("created = %v, want %v", actual, creationTime) } }) + + t.Run("check OCI media type", func(t *testing.T) { + mt, err := img.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } + + if got, want := mt, types.OCIManifestSchema1; got != want { + t.Errorf("mediaType = %v, want %v", got, want) + } + }) } func validateImage(t *testing.T, img oci.SignedImage, baseLayers int64, creationTime v1.Time, checkAnnotations bool, expectSBOM bool) { @@ -919,6 +930,104 @@ func TestGoBuildIndex(t *testing.T) { t.Errorf("Digest mismatch: %s != %s", d1, d2) } }) + + t.Run("check OCI media type", func(t *testing.T) { + mt, err := idx.MediaType() + if err != nil { + t.Fatalf("MediaType() = %v", err) + } + + if got, want := mt, types.OCIImageIndex; got != want { + t.Errorf("mediaType = %v, want %v", got, want) + } + + for i, mf := range im.Manifests { + if got, want := mf.MediaType, types.OCIManifestSchema1; got != want { + t.Errorf("manifest[%d] mediaType = %s, want %s", i, got, want) + } + } + }) +} + +func TestPreserveMediaType(t *testing.T) { + mustRandomImage := func(t *testing.T) v1.Image { + img, err := random.Image(1, 1) + if err != nil { + t.Fatal(err) + } + return img + } + mustRandomIndex := func(t *testing.T) v1.ImageIndex { + idx, err := random.Index(1, 1, 3) + if err != nil { + t.Fatal(err) + } + return idx + } + + for _, c := range []struct { + desc string + preserve bool + base Result + want types.MediaType + }{{ + desc: "docker image -> oci image", + preserve: false, + base: mustRandomImage(t), + want: types.OCIManifestSchema1, + }, { + desc: "docker index -> oci index", + preserve: false, + base: mustRandomIndex(t), + want: types.OCIImageIndex, + }, { + desc: "docker image, preserved", + preserve: true, + base: mustRandomImage(t), + want: types.DockerManifestSchema2, + }, { + desc: "docker index, preserved", + preserve: true, + base: mutate.IndexMediaType(mustRandomIndex(t), types.DockerManifestList), + want: types.DockerManifestList, + }, { + desc: "oci image", + preserve: true, + base: mutate.MediaType(mustRandomImage(t), types.OCIManifestSchema1), + want: types.OCIManifestSchema1, + }, { + desc: "oci index", + preserve: true, + base: mutate.IndexMediaType(mustRandomIndex(t), types.OCIImageIndex), + want: types.OCIImageIndex, + }} { + t.Run(c.desc, func(t *testing.T) { + importpath := "github.com/google/ko" + ng, err := NewGo( + context.Background(), + "", + WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, c.base, nil }), + WithPlatforms("all"), + WithPreserveMediaType(c.preserve), + withBuilder(writeTempFile), + ) + if err != nil { + t.Fatalf("NewGo() = %v", err) + } + + result, err := ng.Build(context.Background(), StrictScheme+filepath.Join(importpath, "test")) + if err != nil { + t.Fatalf("Build() = %v", err) + } + + got, err := result.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } else if got != c.want { + t.Errorf("Got %q, want %q", got, c.want) + } + }) + } } func TestNestedIndex(t *testing.T) { diff --git a/pkg/build/options.go b/pkg/build/options.go index e29bca5ba1..ee25949bc3 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -73,6 +73,16 @@ func WithTrimpath(v bool) Option { } } +// WithPreserveMediaType is a functional option that controls whether to +// preserve media types from base images. If false, images that are produced +// will use OCI media types instead. +func WithPreserveMediaType(v bool) Option { + return func(gbo *gobuildOpener) error { + gbo.preserveMediaType = v + return nil + } +} + // WithConfig is a functional option for providing GoReleaser Build influenced // build settings for importpaths. // diff --git a/pkg/commands/options/build.go b/pkg/commands/options/build.go index 4f2fec976b..6db10667ef 100644 --- a/pkg/commands/options/build.go +++ b/pkg/commands/options/build.go @@ -67,6 +67,9 @@ type BuildOptions struct { // BuildConfigs stores the per-image build config from `.ko.yaml`. BuildConfigs map[string]build.Config + + // If true, don't convert Docker-typed base images to OCI when building. + PreserveMediaType bool } func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { @@ -80,6 +83,9 @@ func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { "Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]*") cmd.Flags().StringSliceVar(&bo.Labels, "image-label", []string{}, "Which labels (key=value) to add to the image.") + + cmd.Flags().BoolVar(&bo.PreserveMediaType, "preserve-media-type", false, "If false, push images in OCI format regardless of base image format") + bo.Trimpath = true } diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 91c09fba99..9526b6402a 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -108,6 +108,7 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { opts = append(opts, build.WithSPDX(version())) } opts = append(opts, build.WithTrimpath(bo.Trimpath)) + opts = append(opts, build.WithPreserveMediaType(bo.PreserveMediaType)) for _, lf := range bo.Labels { parts := strings.SplitN(lf, "=", 2) if len(parts) != 2 {