From 03e4b3e8479c07411200a7eb397b3656e7487f41 Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Fri, 30 Apr 2021 22:51:49 +0200 Subject: [PATCH] Add support for Go build flags There are use cases, where multiple Go build flags need to be set. However, the environment variable to pass flags to Go build has some limits for `ldFlags`. Add GoReleaser inspired configuration section to `.ko.yaml` to support setting specific Go build and ldFlags to be used by the build. Like GoReleaser the content of the configuration can use Go templates. Currently, only a section for environment variables is included. In order to reduce dependency overhead, only the respective config structs from https://github.com/goreleaser/goreleaser/blob/master/pkg/config/config.go are used internally to load from `.ko.yaml`. --- README.md | 45 ++++++++++++++++- pkg/build/config.go | 96 +++++++++++++++++++++++++++++++++++++ pkg/build/gobuild.go | 92 +++++++++++++++++++++++++++++++---- pkg/build/gobuild_test.go | 2 +- pkg/build/options.go | 12 +++++ pkg/commands/config.go | 53 ++++++++++++++++++++ pkg/commands/config_test.go | 32 +++++++++++++ pkg/commands/resolver.go | 5 ++ 8 files changed, 326 insertions(+), 11 deletions(-) create mode 100644 pkg/build/config.go diff --git a/README.md b/README.md index 97fddad325..ceebf92133 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,42 @@ baseImageOverrides: github.com/my-user/my-repo/cmd/foo: registry.example.com/base/for/foo ``` +### Overriding Go build settings + +By default, `ko` builds the binary with no additional build flags other than +`--trimpath` (depending on the Go version). You can replace the default build +arguments by providing build flags and ldflags using a +[GoReleaser](https://github.com/goreleaser/goreleaser) influenced `builds` +configuration section in your `.ko.yaml`. + +```yaml +builds: +- id: foo + dir: ./foobar/foo + flags: + - -tags + - netgo + ldflags: + - -s -w + - -extldflags "-static" + - -X main.version={{.Env.VERSION}} +- id: bar + main: ./foobar/bar/main.go + ldflags: + - -s + - -w +``` + +For the build, `ko` will pick the entry based on the respective import path +being used. It will be matched against the local path that is configured using +`dir` and `main`. In the context of `ko`, it is fine just to specify `dir` +with the intended import path. + +_Please note:_ Even though the configuration section is similar to the +[GoReleaser `builds` section](https://goreleaser.com/customization/build/), +only the `flags` and `ldflags` fields are currently supported. Also, the +templating support is currently limited to environment variables only. + ## Naming Images `ko` provides a few different strategies for naming the image it pushes, to @@ -335,10 +371,17 @@ is a common way to embed version info in go binaries (In fact, we do this for this flag directly; however, you can use the `GOFLAGS` environment variable instead: -``` +```sh GOFLAGS="-ldflags=-X=main.version=1.2.3" ko publish . ``` +## How can I set multiple `ldflags`? + +Currently, there is a limitation that does not allow to set multiple arguments +in `ldflags` using `GOFLAGS`. Using `-ldflags` multiple times also does not +work. In this use case, it works best to use the [`builds` section](#overriding-go-build-settings) +in the `.ko.yaml` file. + ## Why are my images all created in 1970? In order to support [reproducible builds](https://reproducible-builds.org), `ko` diff --git a/pkg/build/config.go b/pkg/build/config.go new file mode 100644 index 0000000000..cb19164845 --- /dev/null +++ b/pkg/build/config.go @@ -0,0 +1,96 @@ +/* +Copyright 2021 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package build + +import "strings" + +// Note: The structs, types, and functions are based upon GoReleaser build +// configuration to have a loosely compatible YAML configuration: +// https://github.com/goreleaser/goreleaser/blob/master/pkg/config/config.go + +// StringArray is a wrapper for an array of strings. +type StringArray []string + +// UnmarshalYAML is a custom unmarshaler that wraps strings in arrays. +func (a *StringArray) UnmarshalYAML(unmarshal func(interface{}) error) error { + var strings []string + if err := unmarshal(&strings); err != nil { + var str string + if err := unmarshal(&str); err != nil { + return err + } + *a = []string{str} + } else { + *a = strings + } + return nil +} + +// FlagArray is a wrapper for an array of strings. +type FlagArray []string + +// UnmarshalYAML is a custom unmarshaler that wraps strings in arrays. +func (a *FlagArray) UnmarshalYAML(unmarshal func(interface{}) error) error { + var flags []string + if err := unmarshal(&flags); err != nil { + var flagstr string + if err := unmarshal(&flagstr); err != nil { + return err + } + *a = strings.Fields(flagstr) + } else { + *a = flags + } + return nil +} + +// Config contains the build configuration section. The name was changed from +// the original GoReleaser name to match better with the ko naming. +// +// TODO: Introduce support for more fields where possible and where it makes +/// sense for `ko`, for example ModTimestamp, Env, or GoBinary. +// +type Config struct { + // ID only serves as an identifier internally + ID string `yaml:",omitempty"` + + // Dir is the directory out of which the build should be triggered + Dir string `yaml:",omitempty"` + + // Main points to the main package, or the source file with the main + // function, in which case only the package will be used for the importpath + Main string `yaml:",omitempty"` + + // Ldflags and Flags will be used for the Go build command line arguments + Ldflags StringArray `yaml:",omitempty"` + Flags FlagArray `yaml:",omitempty"` + + // Other GoReleaser fields that are not supported or do not make sense + // in the context of ko, for reference or for future use: + // Goos []string `yaml:",omitempty"` + // Goarch []string `yaml:",omitempty"` + // Goarm []string `yaml:",omitempty"` + // Gomips []string `yaml:",omitempty"` + // Targets []string `yaml:",omitempty"` + // Binary string `yaml:",omitempty"` + // Env []string `yaml:",omitempty"` + // Lang string `yaml:",omitempty"` + // Asmflags StringArray `yaml:",omitempty"` + // Gcflags StringArray `yaml:",omitempty"` + // ModTimestamp string `yaml:"mod_timestamp,omitempty"` + // GoBinary string `yaml:",omitempty"` +} diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 4d77de4f23..be74989436 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -33,6 +33,7 @@ import ( "path/filepath" "strconv" "strings" + "text/template" "github.com/containerd/stargz-snapshotter/estargz" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -67,7 +68,7 @@ For more information see: // GetBase takes an importpath and returns a base image. type GetBase func(context.Context, string) (Result, error) -type builder func(context.Context, string, string, v1.Platform, bool) (string, error) +type builder func(context.Context, string, string, v1.Platform, Config) (string, error) type buildContext interface { Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) @@ -84,6 +85,7 @@ type gobuild struct { kodataCreationTime v1.Time build builder disableOptimizations bool + buildConfigs map[string]Config mod *modules buildContext buildContext platformMatcher *platformMatcher @@ -100,6 +102,7 @@ type gobuildOpener struct { kodataCreationTime v1.Time build builder disableOptimizations bool + buildConfigs map[string]Config mod *modules buildContext buildContext platform string @@ -121,6 +124,7 @@ func (gbo *gobuildOpener) Open() (Interface, error) { kodataCreationTime: gbo.kodataCreationTime, build: gbo.build, disableOptimizations: gbo.disableOptimizations, + buildConfigs: gbo.buildConfigs, mod: gbo.mod, buildContext: gbo.buildContext, labels: gbo.labels, @@ -354,21 +358,22 @@ func platformToString(p v1.Platform) string { return fmt.Sprintf("%s/%s", p.OS, p.Architecture) } -func build(ctx context.Context, ip string, dir string, platform v1.Platform, disableOptimizations bool) (string, error) { +func build(ctx context.Context, ip string, dir string, platform v1.Platform, config Config) (string, error) { tmpDir, err := ioutil.TempDir("", "ko") if err != nil { return "", err } file := filepath.Join(tmpDir, "out") - args := make([]string, 0, 7) - args = append(args, "build") - if disableOptimizations { - // Disable optimizations (-N) and inlining (-l). - args = append(args, "-gcflags", "all=-N -l") + buildArgs, err := createBuildArgs(config) + if err != nil { + return "", err } + + args := make([]string, 0, 4+len(buildArgs)) + args = append(args, "build") + args = append(args, buildArgs...) args = append(args, "-o", file) - args = addGo113TrimPathFlag(args) args = append(args, ip) cmd := exec.CommandContext(ctx, "go", args...) cmd.Dir = dir @@ -578,6 +583,75 @@ func (g *gobuild) tarKoData(ref reference) (*bytes.Buffer, error) { return buf, walkRecursive(tw, root, kodataRoot, creationTime) } +func createTemplateData() map[string]interface{} { + envVars := map[string]string{} + for _, entry := range os.Environ() { + kv := strings.SplitN(entry, "=", 2) + envVars[kv[0]] = kv[1] + } + + return map[string]interface{}{ + "Env": envVars, + } +} + +func applyTemplating(list []string, data map[string]interface{}) error { + for i, entry := range list { + tmpl, err := template.New("argsTmpl").Option("missingkey=error").Parse(entry) + if err != nil { + return err + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return err + } + + list[i] = buf.String() + } + + return nil +} + +func createBuildArgs(buildCfg Config) ([]string, error) { + var args []string + + data := createTemplateData() + + if len(buildCfg.Flags) > 0 { + if err := applyTemplating(buildCfg.Flags, data); err != nil { + return nil, err + } + + args = append(args, buildCfg.Flags...) + } + + if len(buildCfg.Ldflags) > 0 { + if err := applyTemplating(buildCfg.Ldflags, data); err != nil { + return nil, err + } + + args = append(args, fmt.Sprintf("-ldflags=%s", strings.Join(buildCfg.Ldflags, " "))) + } + + return args, nil +} + +func (g *gobuild) configForImportPath(ip string) Config { + config, ok := g.buildConfigs[ip] + if !ok { + // Apply default build flags in case none were supplied + config.Flags = addGo113TrimPathFlag(config.Flags) + } + + if g.disableOptimizations { + // Disable optimizations (-N) and inlining (-l). + config.Flags = append(config.Flags, "-gcflags", "all=-N -l") + } + + return config +} + func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platform *v1.Platform) (v1.Image, error) { ref := newRef(s) @@ -594,7 +668,7 @@ func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platfor } // Do the build into a temporary file. - file, err := g.build(ctx, ref.Path(), g.dir, *platform, g.disableOptimizations) + file, err := g.build(ctx, ref.Path(), g.dir, *platform, g.configForImportPath(ref.Path())) if err != nil { return nil, err } diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index 6d145db8d1..4d9b1f8cd9 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -229,7 +229,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) { } // A helper method we use to substitute for the default "build" method. -func writeTempFile(_ context.Context, s string, _ string, _ v1.Platform, _ bool) (string, error) { +func writeTempFile(_ context.Context, s string, _ string, _ v1.Platform, _ Config) (string, error) { tmpDir, err := ioutil.TempDir("", "ko") if err != nil { return "", err diff --git a/pkg/build/options.go b/pkg/build/options.go index e79f9a1fe7..60c0fca7c8 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -54,6 +54,18 @@ func WithDisabledOptimizations() Option { } } +// WithConfig is a functional option for providing GoReleaser Build influenced +// build settings for importpaths. +// +// Set a fully qualified importpath (e.g. github.com/my-user/my-repo/cmd/app) +// as the mapping key for the respective Config. +func WithConfig(buildConfigs map[string]Config) Option { + return func(gbo *gobuildOpener) error { + gbo.buildConfigs = buildConfigs + return nil + } +} + // WithPlatforms is a functional option for building certain platforms for // multi-platform base images. To build everything from the base, use "all", // otherwise use a comma-separated list of platform specs, i.e.: diff --git a/pkg/commands/config.go b/pkg/commands/config.go index f90a0cbeec..533d2560c2 100644 --- a/pkg/commands/config.go +++ b/pkg/commands/config.go @@ -22,6 +22,7 @@ import ( "log" "os" "os/signal" + "path/filepath" "strconv" "strings" "syscall" @@ -37,11 +38,13 @@ import ( "github.com/google/ko/pkg/commands/options" "github.com/google/ko/pkg/publish" "github.com/spf13/viper" + "golang.org/x/tools/go/packages" ) var ( defaultBaseImage string baseImageOverrides map[string]string + buildConfigs map[string]build.Config ) // getBaseImage returns a function that determines the base image for a given import path. @@ -161,6 +164,50 @@ func createCancellableContext() context.Context { return ctx } +func createBuildConfigs(baseDir string, configs []build.Config) map[string]build.Config { + buildConfigs = make(map[string]build.Config) + for i, config := range configs { + // Make sure to behave like GoReleaser by defaulting to the current + // directory in case the build or main field is not set, check + // https://goreleaser.com/customization/build/ for details + if config.Dir == "" { + config.Dir = "." + } + if config.Main == "" { + config.Main = "." + } + + // To behave like GoReleaser, check whether the configured path points to a + // source file, and if so, just use the directory it is in + var path string + if fi, err := os.Stat(filepath.Join(baseDir, config.Dir, config.Main)); err == nil && fi.Mode().IsRegular() { + path = filepath.Dir(filepath.Join(config.Dir, config.Main)) + + } else { + path = filepath.Join(config.Dir, config.Main) + } + + // By default, paths configured in the builds section are considered + // local import paths, therefore add a "./" equivalent as a prefix to + // the constructured import path + importPath := fmt.Sprint(".", string(filepath.Separator), path) + + pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName, Dir: baseDir}, importPath) + if err != nil { + log.Fatalf("'builds': entry #%d does not contain a usuable path (%s): %v", i, importPath, err) + } + + if len(pkgs) != 1 { + log.Fatalf("'builds': entry #%d results in %d local packages, only 1 is expected", i, len(pkgs)) + } + + importPath = pkgs[0].PkgPath + buildConfigs[importPath] = config + } + + return buildConfigs +} + func init() { // If omitted, use this base image. viper.SetDefault("defaultBaseImage", "gcr.io/distroless/static:nonroot") @@ -194,4 +241,10 @@ func init() { } baseImageOverrides[k] = v } + + var builds []build.Config + if err := viper.UnmarshalKey("builds", &builds); err != nil { + log.Fatalf("configuration section 'builds' cannot be parsed") + } + buildConfigs = createBuildConfigs(".", builds) } diff --git a/pkg/commands/config_test.go b/pkg/commands/config_test.go index b034868bc0..f62b318282 100644 --- a/pkg/commands/config_test.go +++ b/pkg/commands/config_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/commands/options" ) @@ -45,3 +46,34 @@ func TestOverrideDefaultBaseImageUsingBuildOption(t *testing.T) { t.Errorf("got digest %s, wanted %s", gotDigest, wantDigest) } } + +func TestCreateBuildConfigs(t *testing.T) { + compare := func(expected string, actual string) { + if expected != actual { + t.Errorf("test case failed: expected '%#v', but actual value is '%#v'", expected, actual) + } + } + + buildConfigs := []build.Config{ + {ID: "defaults"}, + {ID: "OnlyMain", Main: "test"}, + {ID: "OnlyMainWithFile", Main: "test/main.go"}, + {ID: "OnlyDir", Dir: "test"}, + {ID: "DirAndMain", Dir: "test", Main: "main.go"}, + } + + for _, b := range buildConfigs { + for importPath, buildCfg := range createBuildConfigs("../..", []build.Config{b}) { + switch buildCfg.ID { + case "defaults": + compare("github.com/google/ko", importPath) + + case "OnlyMain", "OnlyMainWithFile", "OnlyDir", "DirAndMain": + compare("github.com/google/ko/test", importPath) + + default: + t.Fatalf("unknown test case: %s", buildCfg.ID) + } + } + } +} diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 1e0442fe42..a129c9782f 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -104,6 +104,11 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { } opts = append(opts, build.WithLabel(parts[0], parts[1])) } + + if len(buildConfigs) > 0 { + opts = append(opts, build.WithConfig(buildConfigs)) + } + return opts, nil }