From a27762fc092df9276b05d966c4d79332f8b77356 Mon Sep 17 00:00:00 2001 From: Halvard Skogsrud Date: Tue, 27 Apr 2021 18:50:51 +1000 Subject: [PATCH] Enable embedding of ko publish - Export functions and a variable to enable embedding of ko's `publish` functionality to be embedded in other tools. See https://github.com/GoogleContainerTools/skaffold/pull/5611 - Remove DockerRepo PublishOption and flag. This removes the `DockerRepo` config option and `--docker-repo` flag from the PR. New PR with the extracted config option: https://github.com/google/ko/pull/351 - Fix copyright headers for boilerplate check. - Use DockerRepo PublishOption instead of env var. - Override defaultBaseImage using BuildOptions. Remove exported package global SetDefaultBaseImage and instead allow programmatic override of the default base image using the field `BaseImage` in `options.BuildOptions`. Also fix copyright header years. - Add BuildOptions parameter to getBaseImage This enables access to BaseImage for programmatically overriding the default base image from `.ko.yaml`. - Add UserAgent to BuildOptions and PublishOptions This enables programmatically overriding the `User-Agent` HTTP request header for both pulling the base image and pushing the built image. - Rename MakeBuilder to NewBuilder and MakePublisher to NewPublisher. For more idiomatic constructor function names. --- pkg/commands/config.go | 46 ++++++++++----- pkg/commands/config_test.go | 47 +++++++++++++++ pkg/commands/options/build.go | 6 ++ pkg/commands/options/publish.go | 6 +- pkg/commands/publisher.go | 33 ++++++----- pkg/commands/publisher_test.go | 100 ++++++++++++++++++++++++++++++++ pkg/commands/resolver.go | 24 ++++++-- pkg/commands/resolver_test.go | 76 +++++++++++++++++++----- 8 files changed, 292 insertions(+), 46 deletions(-) create mode 100644 pkg/commands/config_test.go create mode 100644 pkg/commands/publisher_test.go diff --git a/pkg/commands/config.go b/pkg/commands/config.go index 20f94f94be..9d79193b74 100644 --- a/pkg/commands/config.go +++ b/pkg/commands/config.go @@ -1,16 +1,18 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2018 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package commands @@ -31,6 +33,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/ko/pkg/build" + "github.com/google/ko/pkg/commands/options" "github.com/spf13/viper" ) @@ -39,7 +42,9 @@ var ( baseImageOverrides map[string]name.Reference ) -func getBaseImage(platform string) build.GetBase { +// getBaseImage returns a function that determines the base image for a given import path. +// If the `bo.BaseImage` parameter is non-empty, it overrides base image configuration from `.ko.yaml`. +func getBaseImage(platform string, bo *options.BuildOptions) build.GetBase { return func(ctx context.Context, s string) (build.Result, error) { s = strings.TrimPrefix(s, build.StrictScheme) // Viper configuration file keys are case insensitive, and are @@ -52,9 +57,20 @@ func getBaseImage(platform string) build.GetBase { if !ok { ref = defaultBaseImage } + if bo.BaseImage != "" { + var err error + ref, err = name.ParseReference(bo.BaseImage) + if err != nil { + return nil, fmt.Errorf("parsing bo.BaseImage (%q): %v", bo.BaseImage, err) + } + } + userAgent := ua() + if bo.UserAgent != "" { + userAgent = bo.UserAgent + } ropt := []remote.Option{ remote.WithAuthFromKeychain(authn.DefaultKeychain), - remote.WithUserAgent(ua()), + remote.WithUserAgent(userAgent), remote.WithContext(ctx), } diff --git a/pkg/commands/config_test.go b/pkg/commands/config_test.go new file mode 100644 index 0000000000..b034868bc0 --- /dev/null +++ b/pkg/commands/config_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2021 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "context" + "testing" + + "github.com/google/ko/pkg/commands/options" +) + +func TestOverrideDefaultBaseImageUsingBuildOption(t *testing.T) { + wantDigest := "sha256:76c39a6f76890f8f8b026f89e081084bc8c64167d74e6c93da7a053cb4ccb5dd" + wantImage := "gcr.io/distroless/static-debian9@" + wantDigest + bo := &options.BuildOptions{ + BaseImage: wantImage, + } + + baseFn := getBaseImage("all", bo) + res, err := baseFn(context.Background(), "ko://example.com/helloworld") + if err != nil { + t.Fatalf("getBaseImage(): %v", err) + } + + digest, err := res.Digest() + if err != nil { + t.Fatalf("res.Digest(): %v", err) + } + gotDigest := digest.String() + if gotDigest != wantDigest { + t.Errorf("got digest %s, wanted %s", gotDigest, wantDigest) + } +} diff --git a/pkg/commands/options/build.go b/pkg/commands/options/build.go index afd6201800..bed97c3383 100644 --- a/pkg/commands/options/build.go +++ b/pkg/commands/options/build.go @@ -22,10 +22,16 @@ import ( // BuildOptions represents options for the ko builder. type BuildOptions struct { + // BaseImage enables setting the default base image programmatically. + // If non-empty, this takes precedence over the value in `.ko.yaml`. + BaseImage string ConcurrentBuilds int DisableOptimizations bool Platform string Labels []string + // UserAgent enables overriding the default value of the `User-Agent` HTTP + // request header used when retrieving the base image. + UserAgent string } func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { diff --git a/pkg/commands/options/publish.go b/pkg/commands/options/publish.go index b15911da2c..ffbc82042c 100644 --- a/pkg/commands/options/publish.go +++ b/pkg/commands/options/publish.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Google LLC All Rights Reserved. +Copyright 2018 Google LLC All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -35,6 +35,10 @@ type PublishOptions struct { // LocalDomain overrides the default domain for images loaded into the local Docker daemon. Use with Local=true. LocalDomain string + // UserAgent enables overriding the default value of the `User-Agent` HTTP + // request header used when pushing the built image to an image registry. + UserAgent string + Tags []string // TagOnly resolves images into tag-only references. TagOnly bool diff --git a/pkg/commands/publisher.go b/pkg/commands/publisher.go index 300e1eed79..be39fa7b96 100644 --- a/pkg/commands/publisher.go +++ b/pkg/commands/publisher.go @@ -1,16 +1,18 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2018 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package commands @@ -41,6 +43,11 @@ func qualifyLocalImport(importpath string) (string, error) { return pkgs[0].PkgPath, nil } +// PublishImages publishes images +func PublishImages(ctx context.Context, importpaths []string, pub publish.Interface, b build.Interface) (map[string]name.Reference, error) { + return publishImages(ctx, importpaths, pub, b) +} + func publishImages(ctx context.Context, importpaths []string, pub publish.Interface, b build.Interface) (map[string]name.Reference, error) { imgs := make(map[string]name.Reference) for _, importpath := range importpaths { diff --git a/pkg/commands/publisher_test.go b/pkg/commands/publisher_test.go new file mode 100644 index 0000000000..e188f03ebc --- /dev/null +++ b/pkg/commands/publisher_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2021 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "context" + "fmt" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/google/ko/pkg/build" + "github.com/google/ko/pkg/commands/options" +) + +func TestPublishImages(t *testing.T) { + repo := "registry.example.com/repository" + sampleAppDir, err := sampleAppRelDir() + if err != nil { + t.Fatalf("sampleAppRelDir(): %v", err) + } + tests := []struct { + description string + publishArg string + importpath string + }{ + { + description: "import path with ko scheme", + publishArg: "ko://github.com/google/ko/test", + importpath: "github.com/google/ko/test", + }, + { + description: "import path without ko scheme", + publishArg: "github.com/google/ko/test", + importpath: "github.com/google/ko/test", + }, + { + description: "file path", + publishArg: sampleAppDir, + importpath: "github.com/google/ko/test", + }, + } + for _, test := range tests { + ctx := context.Background() + bo := &options.BuildOptions{ + ConcurrentBuilds: 1, + } + builder, err := NewBuilder(ctx, bo) + if err != nil { + t.Fatalf("%s: MakeBuilder(): %v", test.description, err) + } + po := &options.PublishOptions{ + DockerRepo: repo, + PreserveImportPaths: true, + } + publisher, err := NewPublisher(po) + if err != nil { + t.Fatalf("%s: MakePublisher(): %v", test.description, err) + } + importpathWithScheme := build.StrictScheme + test.importpath + refs, err := PublishImages(ctx, []string{test.publishArg}, publisher, builder) + if err != nil { + t.Fatalf("%s: PublishImages(): %v", test.description, err) + } + ref, exists := refs[importpathWithScheme] + if !exists { + t.Errorf("%s: could not find image for importpath %s", test.description, importpathWithScheme) + } + gotImageName := ref.Context().Name() + wantImageName := strings.ToLower(fmt.Sprintf("%s/%s", repo, test.importpath)) + if gotImageName != wantImageName { + t.Errorf("%s: got %s, wanted %s", test.description, gotImageName, wantImageName) + } + } +} + +func sampleAppRelDir() (string, error) { + _, filename, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("could not get current filename") + } + basepath := filepath.Dir(filename) + testAppDir := filepath.Join(basepath, "..", "..", "test") + return filepath.Rel(basepath, testAppDir) +} diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 2d813196ee..b6504b41cf 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -1,5 +1,5 @@ /* -Copyright 2021 Google LLC All Rights Reserved. +Copyright 2018 Google LLC All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/labels" ) +// ua returns the ko user agent. func ua() string { if v := version(); v != "" { return "ko/" + v @@ -79,7 +80,7 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { } opts := []build.Option{ - build.WithBaseImages(getBaseImage(platform)), + build.WithBaseImages(getBaseImage(platform, bo)), build.WithPlatforms(platform), } if creationTime != nil { @@ -98,6 +99,11 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { return opts, nil } +// NewBuilder creates a ko builder +func NewBuilder(ctx context.Context, bo *options.BuildOptions) (build.Interface, error) { + return makeBuilder(ctx, bo) +} + func makeBuilder(ctx context.Context, bo *options.BuildOptions) (*build.Caching, error) { opt, err := gobuildOptions(bo) if err != nil { @@ -129,6 +135,11 @@ func makeBuilder(ctx context.Context, bo *options.BuildOptions) (*build.Caching, return build.NewCaching(innerBuilder) } +// NewPublisher creates a ko publisher +func NewPublisher(po *options.PublishOptions) (publish.Interface, error) { + return makePublisher(po) +} + func makePublisher(po *options.PublishOptions) (publish.Interface, error) { // Create the publish.Interface that we will use to publish image references // to either a docker daemon or a container image registry. @@ -151,7 +162,7 @@ func makePublisher(po *options.PublishOptions) (publish.Interface, error) { } if _, err := name.NewRegistry(repoName); err != nil { if _, err := name.NewRepository(repoName); err != nil { - return nil, fmt.Errorf("failed to parse environment variable KO_DOCKER_REPO=%q as repository: %v", repoName, err) + return nil, fmt.Errorf("failed to parse %q as repository: %v", repoName, err) } } @@ -167,9 +178,13 @@ func makePublisher(po *options.PublishOptions) (publish.Interface, error) { tp := publish.NewTarball(po.TarballFile, repoName, namer, po.Tags) publishers = append(publishers, tp) } + userAgent := ua() + if po.UserAgent != "" { + userAgent = po.UserAgent + } if po.Push { dp, err := publish.NewDefault(repoName, - publish.WithUserAgent(ua()), + publish.WithUserAgent(userAgent), publish.WithAuthFromKeychain(authn.DefaultKeychain), publish.WithNamer(namer), publish.WithTags(po.Tags), @@ -208,6 +223,7 @@ type nopPublisher struct { } func (n nopPublisher) Publish(_ context.Context, br build.Result, s string) (name.Reference, error) { + s = strings.TrimPrefix(s, build.StrictScheme) h, err := br.Digest() if err != nil { return nil, err diff --git a/pkg/commands/resolver_test.go b/pkg/commands/resolver_test.go index 76ad033f8e..ff1fd2feeb 100644 --- a/pkg/commands/resolver_test.go +++ b/pkg/commands/resolver_test.go @@ -1,16 +1,18 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* +Copyright 2018 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package commands @@ -20,12 +22,14 @@ import ( "fmt" "io" "io/ioutil" + "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/random" "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/commands/options" @@ -134,6 +138,52 @@ kind: Bar } } +func TestMakeBuilder(t *testing.T) { + ctx := context.Background() + bo := &options.BuildOptions{ + ConcurrentBuilds: 1, + } + builder, err := NewBuilder(ctx, bo) + if err != nil { + t.Fatalf("MakeBuilder(): %v", err) + } + res, err := builder.Build(ctx, "ko://github.com/google/ko/test") + if err != nil { + t.Fatalf("builder.Build(): %v", err) + } + gotDigest, err := res.Digest() + if err != nil { + t.Fatalf("res.Digest(): %v", err) + } + fmt.Println(gotDigest.String()) +} + +func TestMakePublisher(t *testing.T) { + repo := "registry.example.com/repository" + po := &options.PublishOptions{ + DockerRepo: repo, + PreserveImportPaths: true, + } + publisher, err := NewPublisher(po) + if err != nil { + t.Fatalf("MakePublisher(): %v", err) + } + defer publisher.Close() + ctx := context.Background() + importpath := "github.com/google/ko/test" + importpathWithScheme := build.StrictScheme + importpath + buildResult := empty.Index + ref, err := publisher.Publish(ctx, buildResult, importpathWithScheme) + if err != nil { + t.Fatalf("publisher.Publish(): %v", err) + } + gotImageName := ref.Context().Name() + wantImageName := strings.ToLower(fmt.Sprintf("%s/%s", repo, importpath)) + if gotImageName != wantImageName { + t.Errorf("got %s, wanted %s", gotImageName, wantImageName) + } +} + func mustRepository(s string) name.Repository { n, err := name.NewRepository(s) if err != nil {