From ead5e7fbd9ffef2ef02a11e2cf096a498b5322f5 Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Fri, 30 Apr 2021 22:51:49 +0200 Subject: [PATCH] Add support for Go build flags There are use cases, where multiple Go build flags need to be set. However, the environment variable to pass flags to Go build has some limits for `ldFlags`. Add GoReleaser inspired configuration section to `.ko.yaml` to support setting specific Go build and ldFlags to be used by the build. Like GoReleaser the content of the configuration can use Go templates. Currently, only a section for environment variables is included. In order to reduce dependency overhead, only the respective config structs from https://github.com/goreleaser/goreleaser/blob/master/pkg/config/config.go are used internally to load from `.ko.yaml`. --- README.md | 45 ++++++++++++++++++- pkg/build/config.go | 85 +++++++++++++++++++++++++++++++++++ pkg/build/gobuild.go | 95 +++++++++++++++++++++++++++++++++++---- pkg/build/gobuild_test.go | 2 +- pkg/build/options.go | 12 +++++ pkg/commands/config.go | 31 +++++++++++++ pkg/commands/resolver.go | 5 +++ 7 files changed, 264 insertions(+), 11 deletions(-) create mode 100644 pkg/build/config.go diff --git a/README.md b/README.md index 2fc5a22cc9..008890bf90 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,42 @@ baseImageOverrides: github.com/my-user/my-repo/cmd/foo: registry.example.com/base/for/foo ``` +### Overriding Go build settings + +By default, `ko` builds the binary with no additional build flags other than +`--trimpath` (depending on the Go version). You can replace the default build +arguments by providing build flags and ldflags using a +[GoReleaser](https://github.com/goreleaser/goreleaser) influenced `builds` +configuration section in your `.ko.yaml`. + +```yaml +builds: +- id: foo + dir: ./foobar/foo + flags: + - -tags + - netgo + ldflags: + - -s -w + - -extldflags "-static" + - -X main.version={{.Env.VERSION}} +- id: bar + main: ./foobar/bar/main.go + ldflags: + - -s + - -w +``` + +For the build, `ko` will pick the entry based on the respective import path +being used. It will be matched against the local path that is configured using +`dir` and `main`. In the context of `ko`, it is fine just to specify `dir` +with the intended import path. + +_Please note:_ Even though the configuration section is similar to the +[GoReleaser `builds` section](https://goreleaser.com/customization/build/), +only the `flags` and `ldflags` fields are currently supported. Also, the +templating support is currently limited to environment variables only. + ## Naming Images `ko` provides a few different strategies for naming the image it pushes, to @@ -320,10 +356,17 @@ is a common way to embed version info in go binaries (In fact, we do this for this flag directly; however, you can use the `GOFLAGS` environment variable instead: -``` +```sh GOFLAGS="-ldflags=-X=main.version=1.2.3" ko publish . ``` +## How can I set multiple `ldflags`? + +Currently, there is a limitation that does not allow to set multiple arguments +in `ldflags` using `GOFLAGS`. Using `-ldflags` multiple times also does not +work. In this use case, it works best to use the [`builds` section](#overriding-go-build-settings) +in the `.ko.yaml` file. + ## Why are my images all created in 1970? In order to support [reproducible builds](https://reproducible-builds.org), `ko` diff --git a/pkg/build/config.go b/pkg/build/config.go new file mode 100644 index 0000000000..b3142a0f2c --- /dev/null +++ b/pkg/build/config.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 Google LLC All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package build + +import "strings" + +// Note: The structs, types, and functions are based upon GoReleaser build +// configuration to have a loosely compatible YAML configuration: +// https://github.com/goreleaser/goreleaser/blob/master/pkg/config/config.go + +// StringArray is a wrapper for an array of strings. +type StringArray []string + +// UnmarshalYAML is a custom unmarshaler that wraps strings in arrays. +func (a *StringArray) UnmarshalYAML(unmarshal func(interface{}) error) error { + var strings []string + if err := unmarshal(&strings); err != nil { + var str string + if err := unmarshal(&str); err != nil { + return err + } + *a = []string{str} + } else { + *a = strings + } + return nil +} + +// FlagArray is a wrapper for an array of strings. +type FlagArray []string + +// UnmarshalYAML is a custom unmarshaler that wraps strings in arrays. +func (a *FlagArray) UnmarshalYAML(unmarshal func(interface{}) error) error { + var flags []string + if err := unmarshal(&flags); err != nil { + var flagstr string + if err := unmarshal(&flagstr); err != nil { + return err + } + *a = strings.Fields(flagstr) + } else { + *a = flags + } + return nil +} + +// Config contains the build configuration section. The name was changed from +// the original GoReleaser name to match better with the ko naming. +// +// TODO: Introduce support for more fields where possible and where it makes +/// sense for `ko`, for example ModTimestamp, Env, or GoBinary. +// +type Config struct { + // ID string `yaml:",omitempty"` + // Goos []string `yaml:",omitempty"` + // Goarch []string `yaml:",omitempty"` + // Goarm []string `yaml:",omitempty"` + // Gomips []string `yaml:",omitempty"` + // Targets []string `yaml:",omitempty"` + Dir string `yaml:",omitempty"` + Main string `yaml:",omitempty"` + Ldflags StringArray `yaml:",omitempty"` + Flags FlagArray `yaml:",omitempty"` + // Binary string `yaml:",omitempty"` + // Env []string `yaml:",omitempty"` + // Lang string `yaml:",omitempty"` + // Asmflags StringArray `yaml:",omitempty"` + // Gcflags StringArray `yaml:",omitempty"` + // ModTimestamp string `yaml:"mod_timestamp,omitempty"` + // GoBinary string `yaml:",omitempty"` +} diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 4f0565e896..3d464f59a0 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -31,6 +31,7 @@ import ( "path/filepath" "strconv" "strings" + "text/template" "github.com/containerd/stargz-snapshotter/estargz" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -64,7 +65,7 @@ For more information see: // GetBase takes an importpath and returns a base image. type GetBase func(context.Context, string) (Result, error) -type builder func(context.Context, string, v1.Platform, bool) (string, error) +type builder func(context.Context, string, v1.Platform, Config) (string, error) type buildContext interface { Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) @@ -80,6 +81,7 @@ type gobuild struct { creationTime v1.Time build builder disableOptimizations bool + buildConfigs map[string]Config mod *modules buildContext buildContext platformMatcher *platformMatcher @@ -94,6 +96,7 @@ type gobuildOpener struct { creationTime v1.Time build builder disableOptimizations bool + buildConfigs map[string]Config mod *modules buildContext buildContext platform string @@ -113,6 +116,7 @@ func (gbo *gobuildOpener) Open() (Interface, error) { creationTime: gbo.creationTime, build: gbo.build, disableOptimizations: gbo.disableOptimizations, + buildConfigs: gbo.buildConfigs, mod: gbo.mod, buildContext: gbo.buildContext, labels: gbo.labels, @@ -303,24 +307,28 @@ func platformToString(p v1.Platform) string { return fmt.Sprintf("%s/%s", p.OS, p.Architecture) } -func build(ctx context.Context, ip string, platform v1.Platform, disableOptimizations bool) (string, error) { +func build(ctx context.Context, ip string, platform v1.Platform, config Config) (string, error) { tmpDir, err := ioutil.TempDir("", "ko") if err != nil { return "", err } file := filepath.Join(tmpDir, "out") - args := make([]string, 0, 7) - args = append(args, "build") - if disableOptimizations { - // Disable optimizations (-N) and inlining (-l). - args = append(args, "-gcflags", "all=-N -l") + buildArgs, err := createBuildArgs(config) + if err != nil { + return "", err } + + args := make([]string, 0, 4+len(buildArgs)) + args = append(args, "build") + args = append(args, buildArgs...) args = append(args, "-o", file) - args = addGo113TrimPathFlag(args) args = append(args, ip) cmd := exec.CommandContext(ctx, "go", args...) + // Set working directory to match the one set in build config + cmd.Dir = config.Dir + // Last one wins defaultEnv := []string{ "CGO_ENABLED=0", @@ -520,6 +528,75 @@ func (g *gobuild) tarKoData(ref reference) (*bytes.Buffer, error) { return buf, walkRecursive(tw, root, kodataRoot) } +func createTemplateData() map[string]interface{} { + envVars := map[string]string{} + for _, entry := range os.Environ() { + kv := strings.SplitN(entry, "=", 2) + envVars[kv[0]] = kv[1] + } + + return map[string]interface{}{ + "Env": envVars, + } +} + +func applyTemplating(list []string, data map[string]interface{}) error { + for i, entry := range list { + tmpl, err := template.New("argsTmpl").Option("missingkey=error").Parse(entry) + if err != nil { + return err + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return err + } + + list[i] = buf.String() + } + + return nil +} + +func createBuildArgs(buildCfg Config) ([]string, error) { + var args []string + + data := createTemplateData() + + if len(buildCfg.Flags) > 0 { + if err := applyTemplating(buildCfg.Flags, data); err != nil { + return nil, err + } + + args = append(args, buildCfg.Flags...) + } + + if len(buildCfg.Ldflags) > 0 { + if err := applyTemplating(buildCfg.Ldflags, data); err != nil { + return nil, err + } + + args = append(args, fmt.Sprintf("-ldflags=%s", strings.Join(buildCfg.Ldflags, " "))) + } + + return args, nil +} + +func (g *gobuild) configForImportPath(ip string) Config { + config, ok := g.buildConfigs[ip] + if !ok { + // Apply default build flags in case none were supplied + config.Flags = addGo113TrimPathFlag(config.Flags) + } + + if g.disableOptimizations { + // Disable optimizations (-N) and inlining (-l). + config.Flags = append(config.Flags, "-gcflags", "all=-N -l") + } + + return config +} + func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platform *v1.Platform) (v1.Image, error) { ref := newRef(s) @@ -536,7 +613,7 @@ func (g *gobuild) buildOne(ctx context.Context, s string, base v1.Image, platfor } // Do the build into a temporary file. - file, err := g.build(ctx, ref.Path(), *platform, g.disableOptimizations) + file, err := g.build(ctx, ref.Path(), *platform, g.configForImportPath(ref.Path())) if err != nil { return nil, err } diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index 85dd4ba76c..aa864cc58a 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -132,7 +132,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) { } // A helper method we use to substitute for the default "build" method. -func writeTempFile(_ context.Context, s string, _ v1.Platform, _ bool) (string, error) { +func writeTempFile(_ context.Context, s string, _ v1.Platform, _ Config) (string, error) { tmpDir, err := ioutil.TempDir("", "ko") if err != nil { return "", err diff --git a/pkg/build/options.go b/pkg/build/options.go index a8a765811b..82ac63f2a7 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -45,6 +45,18 @@ func WithDisabledOptimizations() Option { } } +// WithConfig is a functional option for providing GoReleaser Build influenced +// build settings for importpaths. +// +// Set a fully qualified importpath (e.g. github.com/my-user/my-repo/cmd/app) +// as the mapping key for the respective Config. +func WithConfig(buildConfigs map[string]Config) Option { + return func(gbo *gobuildOpener) error { + gbo.buildConfigs = buildConfigs + return nil + } +} + // WithPlatforms is a functional option for building certain platforms for // multi-platform base images. To build everything from the base, use "all", // otherwise use a comma-separated list of platform specs, i.e.: diff --git a/pkg/commands/config.go b/pkg/commands/config.go index 20f94f94be..429c635029 100644 --- a/pkg/commands/config.go +++ b/pkg/commands/config.go @@ -17,9 +17,11 @@ package commands import ( "context" "fmt" + gb "go/build" "log" "os" "os/signal" + "path/filepath" "strconv" "strings" "syscall" @@ -37,6 +39,7 @@ import ( var ( defaultBaseImage name.Reference baseImageOverrides map[string]name.Reference + buildConfigs map[string]build.Config ) func getBaseImage(platform string) build.GetBase { @@ -160,4 +163,32 @@ func init() { } baseImageOverrides[k] = bi } + + var builds []build.Config + if err := viper.UnmarshalKey("builds", &builds); err != nil { + log.Fatalf("configuration section 'builds' cannot be parsed") + } + + buildConfigs = make(map[string]build.Config) + for _, build := range builds { + path := build.Dir + if len(path) == 0 { + path = "./" + } + + if len(build.Main) > 0 { + if mainDir := filepath.Dir(build.Main); mainDir != "." { + path = path + string(filepath.Separator) + mainDir + } + } + + if gb.IsLocalImport(path) { + path, err = qualifyLocalImport(path) + if err != nil { + log.Fatalf("failed to create qualified import path using path %s", path) + } + } + + buildConfigs[path] = build + } } diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 2d813196ee..e752498efc 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -95,6 +95,11 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { } opts = append(opts, build.WithLabel(parts[0], parts[1])) } + + if len(buildConfigs) > 0 { + opts = append(opts, build.WithConfig(buildConfigs)) + } + return opts, nil }