Make IMAGE_REPO and IMAGE_TAG templated values in custom build and helm deploy

integration/examples/templated-fields/README.md

@@ -0,0 +1,83 @@
+### Example: use image values in templated fields for build and deploy 
+
+This example shows how `IMAGE_REPO` and `IMAGE_TAG` keywords are available in templated fields for custom build and helm deploy
+
+* **building** a single Go file app with ko
+* **tagging** using the default tagPolicy (`gitCommit`)
+* **deploying** two replica container pods using `helm`
+
+#### Before you begin
+
+For this tutorial to work, you will need to have Skaffold, Helm and a Kubernetes cluster set up.
+To learn more about how to set up, see the [getting started docs](https://skaffold.dev/docs/getting-started).
+
+#### Tutorial
+
+This tutorial will demonstrate how Skaffold can inject image repo and image tag values into the build and deploy stanzas.
+
+First, clone the Skaffold [repo](https://github.com/GoogleContainerTools/skaffold) and navigate to the [templated-fields example](https://github.com/GoogleContainerTools/skaffold/tree/master/examples/templated-fields) for sample code:
+
+```shell
+$ git clone https://github.com/GoogleContainerTools/skaffold
+$ cd skaffold/examples/templated-fields
+```
+
+`IMAGE_REPO` and `IMAGE_TAG` are available as templated fields in `build.sh` file
+
+```shell
+#from build.sh, line 13
+img="${IMAGE_REPO}:${IMAGE_TAG}"
+```
+
+and also in the `helm` deploy section of the skaffold config, which configures artifact `skaffold-templated` to build with `build.sh`:
+
+```yaml
+// from skaffold.yaml, line 22-24
+      setValueTemplates:
+          imageRepo: "{{.IMAGE_REPO}}"
+          imageTag: "{{.IMAGE_TAG}}"
+```
+
+These values are then being set as container environment variables `FOO_IMAGE_REPO` and `FOO_IMAGE_TAG` in the helm template `deployment.yaml` file, just as an example to show how they can be added to your helm templates.
+
+```yaml
+// from charts/templates/deployment.yaml, line 16-24
+      containers:
+      - name: {{ .Chart.Name }}
+        image: {{ .Values.image }}
+        env:
+          - name: FOO_IMAGE_REPO
+            value: {{ .Values.imageRepo }}
+          - name: FOO_IMAGE_TAG
+            value: {{ .Values.imageTag }}
+```
+
+For more information about how this works, see the Skaffold [custom builder](https://skaffold.dev/docs/how-tos/builders/#custom-build-script-run-locally) and [helm](https://skaffold.dev/docs/pipeline-stages/deployers/helm/) documentation.
+
+Now, use Skaffold to deploy this application to your Kubernetes cluster:
+
+```shell
+$ skaffold run --tail --default-repo <your repo>
+```
+
+With this command, Skaffold will build the `skaffold-templated` artifact with ko and deploy the application to Kubernetes using helm.
+You should be able to see something like:
+
+```shell
+Running image skaffold-templated:a866d5efd634062ea74662b20e172cd6e2d645f9f33f929bfaf8e856ec66bd94
+```
+
+ printed every second in the Skaffold logs, since the code being executed is `main.go`.
+
+```go
+// from main.go, line 12
+fmt.Printf("Running image %v:%v\n", os.Getenv("FOO_IMAGE_REPO"), os.Getenv("FOO_IMAGE_TAG"))
+```
+
+#### Cleanup
+
+To clean up your Kubernetes cluster, run:
+
+```shell
+$ skaffold delete
+```

integration/examples/templated-fields/build.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+if ! [ -x "$(command -v ko)" ]; then
+    pushd $(mktemp -d)
+    go mod init tmp; GOFLAGS= go get github.com/google/ko/cmd/[email protected]
+    popd
+fi
+
+output=$(ko publish --local --preserve-import-paths --tags= . | tee)
+ref=$(echo $output | tail -n1)
+
+img="${IMAGE_REPO}:${IMAGE_TAG}"
+docker tag $ref $img
+if $PUSH_IMAGE; then
+    docker push $img
+fi

integration/examples/templated-fields/charts/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Skaffold example with Helm
+name: skaffold-helm
+version: 0.1.0

integration/examples/templated-fields/charts/templates/deployment.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}
+  labels:
+    app: {{ .Chart.Name }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Chart.Name }}
+  replicas: {{ .Values.replicaCount }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Chart.Name }}
+    spec:
+      containers:
+      - name: {{ .Chart.Name }}
+        image: {{ .Values.image }}
+        env:
+          - name: FOO_IMAGE_REPO
+            value: {{ .Values.imageRepo }}
+          - name: FOO_IMAGE_TAG
+            value: {{ .Values.imageTag }}

integration/examples/templated-fields/charts/values.yaml

@@ -0,0 +1,4 @@
+# Default values for skaffold-helm.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+replicaCount: 2

integration/examples/templated-fields/go.mod

@@ -0,0 +1,3 @@
+module github.com/GoogleContainerTools/skaffold/examples/template-fields
+
+go 1.13

integration/examples/templated-fields/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+func main() {
+	for {
+		// FOO_IMAGE_REPO and FOO_IMAGE_TAG are defined in the Helm chart using Skaffold's templated IMAGE_REPO and IMAGE_TAG
+		fmt.Printf("Running image %v:%v\n", os.Getenv("FOO_IMAGE_REPO"), os.Getenv("FOO_IMAGE_TAG"))
+		time.Sleep(time.Second * 1)
+	}
+}

integration/examples/templated-fields/skaffold.yaml

@@ -0,0 +1,24 @@
+apiVersion: skaffold/v2beta5
+kind: Config
+metadata:
+  name: my-app
+
+build:
+  artifacts:
+    - image: skaffold-templated
+      custom:
+        buildCommand: "./build.sh"
+        dependencies:
+          paths:
+          - "go.mod"
+          - "**.go"
+deploy:
+  helm:
+    releases:
+      - name: skaffold-templated
+        chartPath: charts
+        artifactOverrides:
+          image: skaffold-templated
+        setValueTemplates:
+          imageRepo: "{{.IMAGE_REPO}}"
+          imageTag: "{{.IMAGE_TAG}}"

pkg/skaffold/build/custom/script.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/misc"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 )
@@ -95,6 +96,16 @@ func (b *Builder) retrieveEnv(a *latest.Artifact, tag string) ([]string, error)
 		fmt.Sprintf("%s=%t", constants.PushImage, b.pushImages),
 		fmt.Sprintf("%s=%s", constants.BuildContext, buildContext),
 	}
+
+	ref, err := docker.ParseReference(tag)
+	if err != nil {
+		return nil, fmt.Errorf("parsing image %v: %w", tag, err)
+	}
+
+	// Standardize access to Image reference fields in templates
+	envs = append(envs, fmt.Sprintf("%s=%s", constants.ImageRef.Repo, ref.BaseName))
+	envs = append(envs, fmt.Sprintf("%s=%s", constants.ImageRef.Tag, ref.Tag))
+
 	envs = append(envs, b.additionalEnv...)
 	envs = append(envs, util.OSEnviron()...)
 	return envs, nil

pkg/skaffold/build/custom/script_test.go

@@ -44,24 +44,24 @@ func TestRetrieveEnv(t *testing.T) {
 			tag:          "gcr.io/image/tag:mytag",
 			environ:      nil,
 			buildContext: "/some/path",
-			expected:     []string{"IMAGE=gcr.io/image/tag:mytag", "IMAGES=gcr.io/image/tag:mytag", "PUSH_IMAGE=false", "BUILD_CONTEXT=/some/path"},
+			expected:     []string{"IMAGE=gcr.io/image/tag:mytag", "IMAGES=gcr.io/image/tag:mytag", "PUSH_IMAGE=false", "BUILD_CONTEXT=/some/path", "IMAGE_REPO=gcr.io/image/tag", "IMAGE_TAG=mytag"},
 		}, {
 			description:  "make sure environ is correctly applied",
 			tag:          "gcr.io/image/tag:anothertag",
 			environ:      []string{"PATH=/path", "HOME=/root"},
 			buildContext: "/some/path",
-			expected:     []string{"IMAGE=gcr.io/image/tag:anothertag", "IMAGES=gcr.io/image/tag:anothertag", "PUSH_IMAGE=false", "BUILD_CONTEXT=/some/path", "PATH=/path", "HOME=/root"},
+			expected:     []string{"IMAGE=gcr.io/image/tag:anothertag", "IMAGES=gcr.io/image/tag:anothertag", "PUSH_IMAGE=false", "BUILD_CONTEXT=/some/path", "IMAGE_REPO=gcr.io/image/tag", "IMAGE_TAG=anothertag", "PATH=/path", "HOME=/root"},
 		}, {
 			description: "push image is true",
 			tag:         "gcr.io/image/push:tag",
 			pushImages:  true,
-			expected:    []string{"IMAGE=gcr.io/image/push:tag", "IMAGES=gcr.io/image/push:tag", "PUSH_IMAGE=true", "BUILD_CONTEXT="},
+			expected:    []string{"IMAGE=gcr.io/image/push:tag", "IMAGES=gcr.io/image/push:tag", "PUSH_IMAGE=true", "BUILD_CONTEXT=", "IMAGE_REPO=gcr.io/image/push", "IMAGE_TAG=tag"},
 		}, {
 			description:   "add additional env",
 			tag:           "gcr.io/image/push:tag",
 			pushImages:    true,
 			additionalEnv: []string{"KUBECONTEXT=mycluster"},
-			expected:      []string{"IMAGE=gcr.io/image/push:tag", "IMAGES=gcr.io/image/push:tag", "PUSH_IMAGE=true", "BUILD_CONTEXT=", "KUBECONTEXT=mycluster"},
+			expected:      []string{"IMAGE=gcr.io/image/push:tag", "IMAGES=gcr.io/image/push:tag", "PUSH_IMAGE=true", "BUILD_CONTEXT=", "IMAGE_REPO=gcr.io/image/push", "IMAGE_TAG=tag", "KUBECONTEXT=mycluster"},
 		},
 	}
 	for _, test := range tests {
@@ -98,8 +98,8 @@ func TestRetrieveCmd(t *testing.T) {
 				},
 			},
 			tag:               "image:tag",
-			expected:          expectedCmd("workspace", "sh", []string{"-c", "./build.sh"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=workspace"}),
-			expectedOnWindows: expectedCmd("workspace", "cmd.exe", []string{"/C", "./build.sh"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=workspace"}),
+			expected:          expectedCmd("workspace", "sh", []string{"-c", "./build.sh"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=workspace", "IMAGE_REPO=image", "IMAGE_TAG=tag"}),
+			expectedOnWindows: expectedCmd("workspace", "cmd.exe", []string{"/C", "./build.sh"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=workspace", "IMAGE_REPO=image", "IMAGE_TAG=tag"}),
 		},
 		{
 			description: "buildcommand with multiple args",
@@ -111,8 +111,8 @@ func TestRetrieveCmd(t *testing.T) {
 				},
 			},
 			tag:               "image:tag",
-			expected:          expectedCmd("", "sh", []string{"-c", "./build.sh --flag=$IMAGES --anotherflag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT="}),
-			expectedOnWindows: expectedCmd("", "cmd.exe", []string{"/C", "./build.sh --flag=$IMAGES --anotherflag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT="}),
+			expected:          expectedCmd("", "sh", []string{"-c", "./build.sh --flag=$IMAGES --anotherflag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=", "IMAGE_REPO=image", "IMAGE_TAG=tag"}),
+			expectedOnWindows: expectedCmd("", "cmd.exe", []string{"/C", "./build.sh --flag=$IMAGES --anotherflag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=", "IMAGE_REPO=image", "IMAGE_TAG=tag"}),
 		},
 		{
 			description: "buildcommand with go template",
@@ -125,8 +125,8 @@ func TestRetrieveCmd(t *testing.T) {
 			},
 			tag:               "image:tag",
 			env:               []string{"FLAG=some-flag"},
-			expected:          expectedCmd("", "sh", []string{"-c", "./build.sh --flag=some-flag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=", "FLAG=some-flag"}),
-			expectedOnWindows: expectedCmd("", "cmd.exe", []string{"/C", "./build.sh --flag=some-flag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=", "FLAG=some-flag"}),
+			expected:          expectedCmd("", "sh", []string{"-c", "./build.sh --flag=some-flag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=", "IMAGE_REPO=image", "IMAGE_TAG=tag", "FLAG=some-flag"}),
+			expectedOnWindows: expectedCmd("", "cmd.exe", []string{"/C", "./build.sh --flag=some-flag"}, []string{"IMAGE=image:tag", "IMAGES=image:tag", "PUSH_IMAGE=false", "BUILD_CONTEXT=", "IMAGE_REPO=image", "IMAGE_TAG=tag", "FLAG=some-flag"}),
 		},
 	}
 	for _, test := range tests {

pkg/skaffold/constants/constants.go

@@ -105,6 +105,15 @@ var (
 	Timeout = "TIMEOUT"
 )
 
+var ImageRef = struct {
+	Repo   string
+	Tag    string
+	Digest string
+}{
+	Repo:   "IMAGE_REPO",
+	Tag:    "IMAGE_TAG",
+	Digest: "IMAGE_DIGEST",
+}
 var DefaultKubectlManifests = []string{"k8s/*.yaml"}
 
 var Labels = struct {

pkg/skaffold/deploy/helm.go

@@ -287,27 +287,9 @@ func (h *HelmDeployer) Render(ctx context.Context, out io.Writer, builds []build
 			args = append(args, "--set-string", value)
 		}
 
-		sortedKeys := make([]string, 0, len(r.SetValues))
-		for k := range r.SetValues {
-			sortedKeys = append(sortedKeys, k)
-		}
-		sort.Strings(sortedKeys)
-		for _, k := range sortedKeys {
-			args = append(args, "--set", fmt.Sprintf("%s=%s", k, r.SetValues[k]))
-		}
-
-		sortedKeys = make([]string, 0, len(r.SetValueTemplates))
-		for k := range r.SetValueTemplates {
-			sortedKeys = append(sortedKeys, k)
-		}
-		sort.Strings(sortedKeys)
-
-		for _, key := range sortedKeys {
-			v, err := util.ExpandEnvTemplate(r.SetValueTemplates[key], nil)
-			if err != nil {
-				return err
-			}
-			args = append(args, "--set", fmt.Sprintf("%s=%s", key, v))
+		args, err = constructOverrideArgs(&r, builds, args, func(string) {})
+		if err != nil {
+			return err
 		}
 
 		if r.Namespace != "" {
@@ -554,18 +536,34 @@ func installArgs(r latest.HelmRelease, builds []build.Artifact, valuesSet map[st
 		args = append(args, "--set-string", value)
 	}
 
+	args, err = constructOverrideArgs(&r, builds, args, func(k string) {
+		valuesSet[k] = true
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if r.Wait {
+		args = append(args, "--wait")
+	}
+
+	return args, nil
+}
+
+// constructOverrideArgs creates the command line arguments for overrides
+func constructOverrideArgs(r *latest.HelmRelease, builds []build.Artifact, args []string, record func(string)) ([]string, error) {
 	sortedKeys := make([]string, 0, len(r.SetValues))
 	for k := range r.SetValues {
 		sortedKeys = append(sortedKeys, k)
 	}
 	sort.Strings(sortedKeys)
 	for _, k := range sortedKeys {
-		valuesSet[r.SetValues[k]] = true
+		record(r.SetValues[k])
 		args = append(args, "--set", fmt.Sprintf("%s=%s", k, r.SetValues[k]))
 	}
 
 	for k, v := range r.SetFiles {
-		valuesSet[v] = true
+		record(v)
 		args = append(args, "--set-file", fmt.Sprintf("%s=%s", k, v))
 	}
 
@@ -593,7 +591,7 @@ func installArgs(r latest.HelmRelease, builds []build.Artifact, valuesSet map[st
 			return nil, err
 		}
 
-		valuesSet[v] = true
+		record(v)
 		args = append(args, "--set", fmt.Sprintf("%s=%s", k, v))
 	}
 
@@ -610,11 +608,6 @@ func installArgs(r latest.HelmRelease, builds []build.Artifact, valuesSet map[st
 
 		args = append(args, "-f", exp)
 	}
-
-	if r.Wait {
-		args = append(args, "--wait")
-	}
-
 	return args, nil
 }
 
@@ -637,6 +630,16 @@ func envVarForImage(imageName string, digest string) map[string]string {
 		"DIGEST":     digest, // The `DIGEST` name is kept for compatibility reasons
 	}
 
+	// Standardize access to Image reference fields in templates
+	ref, err := docker.ParseReference(digest)
+	if err == nil {
+		customMap[constants.ImageRef.Repo] = ref.BaseName
+		customMap[constants.ImageRef.Tag] = ref.Tag
+		customMap[constants.ImageRef.Digest] = ref.Digest
+	} else {
+		logrus.Warnf("unable to extract values for %v, %v and %v from image %v due to error:\n%v", constants.ImageRef.Repo, constants.ImageRef.Tag, constants.ImageRef.Digest, digest, err)
+	}
+
 	if digest == "" {
 		return customMap
 	}