From c42ee5f0286bcd1f3d2cf85783206bcd8b83ba44 Mon Sep 17 00:00:00 2001 From: Nathan Mittler Date: Thu, 9 May 2024 16:01:42 -0700 Subject: [PATCH] feat: add template params for platform info This restructures the build logic in order to expand the buildArgs to include: - `Env`: the actual environment variables used to execute the build. This includes platform info (e.g. `GOOS`, `GOARCH`). - `GoEnv`: the map of variables from `go env`, but overridden with any platform-specific values defined in `Env`. Fixes #1301 Signed-off-by: Nathan Mittler --- docs/configuration.md | 29 ++++++------ pkg/build/gobuild.go | 95 ++++++++++++++++++++++++++++++++------- pkg/build/gobuild_test.go | 72 +++++++++++++++++++++++------ 3 files changed, 152 insertions(+), 44 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 323052d33..1ea0a29ae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -84,20 +84,21 @@ The `ko` builds supports templating of `flags` and `ldflags`, similar to the The table below lists the supported template parameters. -| Template param | Description | -|-----------------------|-------------------------------------------------------| -| `Env` | Map of system environment variables from `os.Environ` | -| `Date` | The UTC build date in RFC 3339 format | -| `Timestamp` | The UTC build date as Unix epoc seconds | -| `Git.Branch` | The current git branch | -| `Git.Tag` | The current git tag | -| `Git.ShortCommit` | The git commit short hash | -| `Git.FullCommit` | The git commit full hash | -| `Git.CommitDate` | The UTC commit date in RFC 3339 format | -| `Git.CommitTimestamp` | The UTC commit date in Unix format | -| `Git.IsDirty` | Whether or not current git state is dirty | -| `Git.IsClean` | Whether or not current git state is clean. | -| `Git.TreeState` | Either `clean` or `dirty` | +| Template param | Description | +|-----------------------|----------------------------------------------------------| +| `Env` | Map of environment variables used for the build | +| `GoEnv` | Map of `go env` environment variables used for the build | +| `Date` | The UTC build date in RFC 3339 format | +| `Timestamp` | The UTC build date as Unix epoc seconds | +| `Git.Branch` | The current git branch | +| `Git.Tag` | The current git tag | +| `Git.ShortCommit` | The git commit short hash | +| `Git.FullCommit` | The git commit full hash | +| `Git.CommitDate` | The UTC commit date in RFC 3339 format | +| `Git.CommitTimestamp` | The UTC commit date in Unix format | +| `Git.IsDirty` | Whether or not current git state is dirty | +| `Git.IsClean` | Whether or not current git state is clean. | +| `Git.TreeState` | Either `clean` or `dirty` | ### Setting default platforms diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 98cbf8536..8e51a782e 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -16,6 +16,7 @@ package build import ( "archive/tar" + "bufio" "bytes" "context" "errors" @@ -68,7 +69,7 @@ type buildContext struct { creationTime v1.Time ip string dir string - env []string + mergedEnv []string platform v1.Platform config Config } @@ -267,6 +268,8 @@ func getGoBinary() string { } func build(ctx context.Context, buildCtx buildContext) (string, error) { + // Create the set of build arguments from the config flags/ldflags with any + // template parameters applied. buildArgs, err := createBuildArgs(ctx, buildCtx) if err != nil { return "", err @@ -275,12 +278,6 @@ func build(ctx context.Context, buildCtx buildContext) (string, error) { args := make([]string, 0, 4+len(buildArgs)) args = append(args, "build") args = append(args, buildArgs...) - - env, err := buildEnv(buildCtx.platform, os.Environ(), buildCtx.env, buildCtx.config.Env) - if err != nil { - return "", fmt.Errorf("could not create env for %s: %w", buildCtx.ip, err) - } - tmpDir := "" if dir := os.Getenv("KOCACHE"); dir != "" { @@ -316,7 +313,7 @@ func build(ctx context.Context, buildCtx buildContext) (string, error) { gobin := getGoBinary() cmd := exec.CommandContext(ctx, gobin, args...) cmd.Dir = buildCtx.dir - cmd.Env = env + cmd.Env = buildCtx.mergedEnv var output bytes.Buffer cmd.Stderr = &output @@ -325,13 +322,49 @@ func build(ctx context.Context, buildCtx buildContext) (string, error) { log.Printf("Building %s for %s", buildCtx.ip, buildCtx.platform) if err := cmd.Run(); err != nil { if os.Getenv("KOCACHE") == "" { - os.RemoveAll(tmpDir) + _ = os.RemoveAll(tmpDir) } return "", fmt.Errorf("go build: %w: %s", err, output.String()) } return file, nil } +func goenv(ctx context.Context) (map[string]string, error) { + gobin := getGoBinary() + cmd := exec.CommandContext(ctx, gobin, "env") + var output bytes.Buffer + cmd.Stdout = &output + cmd.Stderr = &output + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("go env: %w: %s", err, output.String()) + } + + env := make(map[string]string) + scanner := bufio.NewScanner(bytes.NewReader(output.Bytes())) + + line := 0 + for scanner.Scan() { + line++ + kv := strings.SplitN(scanner.Text(), "=", 2) + if len(kv) != 2 { + return nil, fmt.Errorf("go env: failed parsing line: %d", line) + } + key := strings.TrimSpace(kv[0]) + value := strings.TrimSpace(kv[1]) + + // Unquote the value. Handle single or double quoted strings. + if len(value) > 1 && ((value[0] == '\'' && value[len(value)-1] == '\'') || + (value[0] == '"' && value[len(value)-1] == '"')) { + value = value[1 : len(value)-1] + } + env[key] = value + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("go env: failed parsing: %w", err) + } + return env, nil +} + func goversionm(ctx context.Context, file string, appPath string, appFileName string, se oci.SignedEntity, dir string) ([]byte, types.MediaType, error) { gobin := getGoBinary() @@ -724,15 +757,31 @@ func (g *gobuild) tarKoData(ref reference, platform *v1.Platform) (*bytes.Buffer return buf, walkRecursive(tw, root, chroot, creationTime, platform) } -func createTemplateData(ctx context.Context, buildCtx buildContext) map[string]interface{} { +func createTemplateData(ctx context.Context, buildCtx buildContext) (map[string]interface{}, error) { envVars := map[string]string{ "LDFLAGS": "", } - for _, entry := range os.Environ() { + for _, entry := range buildCtx.mergedEnv { kv := strings.SplitN(entry, "=", 2) + if len(kv) != 2 { + return nil, fmt.Errorf("invalid environment variable entry: %q", entry) + } envVars[kv[0]] = kv[1] } + // Get the go environment. + goEnv, err := goenv(ctx) + if err != nil { + return nil, err + } + + // Override go env with any matching values from the environment variables. + for k, v := range envVars { + if _, ok := goEnv[k]; ok { + goEnv[k] = v + } + } + // Get the git information, if available. info, err := git.GetInfo(ctx, buildCtx.dir) if err != nil { @@ -747,10 +796,11 @@ func createTemplateData(ctx context.Context, buildCtx buildContext) map[string]i return map[string]interface{}{ "Env": envVars, + "GoEnv": goEnv, "Git": info.TemplateValue(), "Date": date.Format(time.RFC3339), "Timestamp": date.UTC().Unix(), - } + }, nil } func applyTemplating(list []string, data map[string]interface{}) ([]string, error) { @@ -775,7 +825,10 @@ func applyTemplating(list []string, data map[string]interface{}) ([]string, erro func createBuildArgs(ctx context.Context, buildCtx buildContext) ([]string, error) { var args []string - data := createTemplateData(ctx, buildCtx) + data, err := createTemplateData(ctx, buildCtx) + if err != nil { + return nil, err + } if len(buildCtx.config.Flags) > 0 { flags, err := applyTemplating(buildCtx.config.Flags, data) @@ -865,13 +918,21 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl if !g.platformMatcher.matches(platform) { return nil, fmt.Errorf("base image platform %q does not match desired platforms %v", platform, g.platformMatcher.platforms) } - // Do the build into a temporary file. + config := g.configForImportPath(ref.Path()) + + // Merge the system, global, and build config environment variables. + mergedEnv, err := buildEnv(*platform, os.Environ(), g.env, config.Env) + if err != nil { + return nil, fmt.Errorf("could not create env for %s: %w", ref.Path(), err) + } + + // Do the build into a temporary file. file, err := g.build(ctx, buildContext{ creationTime: g.creationTime, ip: ref.Path(), dir: g.dir, - env: g.env, + mergedEnv: mergedEnv, platform: *platform, config: config, }) @@ -1101,7 +1162,7 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseRef name.Referen return nil, err } - matches := []v1.Descriptor{} + matches := make([]v1.Descriptor, 0) for _, desc := range im.Manifests { // Nested index is pretty rare. We could support this in theory, but return an error for now. if desc.MediaType != types.OCIManifestSchema1 && desc.MediaType != types.DockerManifestSchema2 { @@ -1226,7 +1287,7 @@ func parseSpec(spec []string) (*platformMatcher, error) { return &platformMatcher{spec: spec}, nil } - platforms := []v1.Platform{} + platforms := make([]v1.Platform, 0) for _, s := range spec { p, err := v1.ParsePlatform(s) if err != nil { diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index 32b1d2ec3..57b648720 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -315,16 +315,19 @@ func TestBuildEnv(t *testing.T) { } } -func TestCreateTemplateData(t *testing.T) { - t.Run("env", func(t *testing.T) { - t.Setenv("FOO", "bar") - params := createTemplateData(context.TODO(), buildContext{dir: t.TempDir()}) - vars := params["Env"].(map[string]string) - require.Equal(t, "bar", vars["FOO"]) - }) +func TestGoEnv(t *testing.T) { + goVars, err := goenv(context.TODO()) + require.NoError(t, err) + // Just check some basic values. + require.Equal(t, runtime.GOOS, goVars["GOOS"]) + require.Equal(t, runtime.GOARCH, goVars["GOARCH"]) +} + +func TestCreateTemplateData(t *testing.T) { t.Run("empty creation time", func(t *testing.T) { - params := createTemplateData(context.TODO(), buildContext{dir: t.TempDir()}) + params, err := createTemplateData(context.TODO(), buildContext{dir: t.TempDir()}) + require.NoError(t, err) // Make sure the date was set to time.Now(). actualDateStr := params["Date"].(string) @@ -346,10 +349,11 @@ func TestCreateTemplateData(t *testing.T) { expectedTime, err := time.Parse(time.RFC3339, "2012-11-01T22:08:00Z") require.NoError(t, err) - params := createTemplateData(context.TODO(), buildContext{ + params, err := createTemplateData(context.TODO(), buildContext{ creationTime: v1.Time{Time: expectedTime}, dir: t.TempDir(), }) + require.NoError(t, err) // Check the date. actualDateStr := params["Date"].(string) @@ -365,9 +369,10 @@ func TestCreateTemplateData(t *testing.T) { t.Run("no git available", func(t *testing.T) { dir := t.TempDir() - params := createTemplateData(context.TODO(), buildContext{dir: dir}) - gitParams := params["Git"].(map[string]interface{}) + params, err := createTemplateData(context.TODO(), buildContext{dir: dir}) + require.NoError(t, err) + gitParams := params["Git"].(map[string]interface{}) require.Equal(t, "", gitParams["Branch"]) require.Equal(t, "", gitParams["Tag"]) require.Equal(t, "", gitParams["ShortCommit"]) @@ -384,13 +389,54 @@ func TestCreateTemplateData(t *testing.T) { gittesting.GitCommit(t, dir, "commit1") gittesting.GitTag(t, dir, "v0.0.1") - params := createTemplateData(context.TODO(), buildContext{dir: dir}) - gitParams := params["Git"].(map[string]interface{}) + params, err := createTemplateData(context.TODO(), buildContext{dir: dir}) + require.NoError(t, err) + gitParams := params["Git"].(map[string]interface{}) require.Equal(t, "main", gitParams["Branch"]) require.Equal(t, "v0.0.1", gitParams["Tag"]) require.Equal(t, "clean", gitParams["TreeState"]) }) + + t.Run("env", func(t *testing.T) { + params, err := createTemplateData(context.TODO(), buildContext{ + dir: t.TempDir(), + mergedEnv: []string{"FOO=bar"}, + }) + require.NoError(t, err) + vars := params["Env"].(map[string]string) + require.Equal(t, "bar", vars["FOO"]) + }) + + t.Run("bad env", func(t *testing.T) { + _, err := createTemplateData(context.TODO(), buildContext{ + dir: t.TempDir(), + mergedEnv: []string{"bad var"}, + }) + require.Error(t, err) + }) + + t.Run("default go env", func(t *testing.T) { + params, err := createTemplateData(context.TODO(), buildContext{dir: t.TempDir()}) + require.NoError(t, err) + vars := params["GoEnv"].(map[string]string) + require.Equal(t, runtime.GOOS, vars["GOOS"]) + require.Equal(t, runtime.GOARCH, vars["GOARCH"]) + }) + + t.Run("env overrides go env", func(t *testing.T) { + params, err := createTemplateData(context.TODO(), buildContext{ + dir: t.TempDir(), + mergedEnv: []string{ + "GOOS=testgoos", + "GOARCH=testgoarch", + }, + }) + require.NoError(t, err) + vars := params["GoEnv"].(map[string]string) + require.Equal(t, "testgoos", vars["GOOS"]) + require.Equal(t, "testgoarch", vars["GOARCH"]) + }) } func TestBuildConfig(t *testing.T) {