Add dynamic environment variable substitution to subpaths

pkg/features/kube_features.go

@@ -239,6 +239,13 @@ const (
 	//
 	// Implement support for limiting pids in pods
 	SupportPodPidsLimit utilfeature.Feature = "SupportPodPidsLimit"
+
+	// owner: @kevtaylor
+	// alpha: v1.11
+	//
+	// Allow subpath environment variable substitution
+	// Only applicable if the VolumeSubpath feature is also enabled
+	VolumeSubpathEnvExpansion utilfeature.Feature = "VolumeSubpathEnvExpansion"
 )
 
 func init() {
@@ -281,6 +288,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	SupportIPVSProxyMode:                        {Default: false, PreRelease: utilfeature.Beta},
 	VolumeSubpath:                               {Default: true, PreRelease: utilfeature.GA},
 	SupportPodPidsLimit:                         {Default: false, PreRelease: utilfeature.Alpha},
+	VolumeSubpathEnvExpansion:                   {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/kubelet/container/helpers.go

@@ -145,6 +145,11 @@ func ExpandContainerCommandOnlyStatic(containerCommand []string, envs []v1.EnvVa
 	return command
 }
 
+func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (expandedSubpath string) {
+	mapping := expansion.MappingFuncFor(EnvVarsToMap(envs))
+	return expansion.Expand(mount.SubPath, mapping)
+}
+
 func ExpandContainerCommandAndArgs(container *v1.Container, envs []EnvVar) (command []string, args []string) {
 	mapping := expansion.MappingFuncFor(EnvVarsToMap(envs))
 

pkg/kubelet/container/helpers_test.go

@@ -138,6 +138,114 @@ func TestExpandCommandAndArgs(t *testing.T) {
 	}
 }
 
+func TestExpandVolumeMountsWithSubpath(t *testing.T) {
+	cases := []struct {
+		name              string
+		container         *v1.Container
+		envs              []EnvVar
+		expectedSubPath   string
+		expectedMountPath string
+	}{
+		{
+			name: "subpath with no expansion",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPath: "foo"}},
+			},
+			expectedSubPath:   "foo",
+			expectedMountPath: "",
+		},
+		{
+			name: "volumes with expanded subpath",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPath: "foo/$(POD_NAME)"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "POD_NAME",
+					Value: "bar",
+				},
+			},
+			expectedSubPath:   "foo/bar",
+			expectedMountPath: "",
+		},
+		{
+			name: "volumes expanded with empty subpath",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPath: ""}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "POD_NAME",
+					Value: "bar",
+				},
+			},
+			expectedSubPath:   "",
+			expectedMountPath: "",
+		},
+		{
+			name: "volumes expanded with no envs subpath",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPath: "/foo/$(POD_NAME)"}},
+			},
+			expectedSubPath:   "/foo/$(POD_NAME)",
+			expectedMountPath: "",
+		},
+		{
+			name: "volumes expanded with leading environment variable",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPath: "$(POD_NAME)/bar"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "POD_NAME",
+					Value: "foo",
+				},
+			},
+			expectedSubPath:   "foo/bar",
+			expectedMountPath: "",
+		},
+		{
+			name: "volumes with volume and subpath",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{MountPath: "/foo", SubPath: "$(POD_NAME)/bar"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "POD_NAME",
+					Value: "foo",
+				},
+			},
+			expectedSubPath:   "foo/bar",
+			expectedMountPath: "/foo",
+		},
+		{
+			name: "volumes with volume and no subpath",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{MountPath: "/foo"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "POD_NAME",
+					Value: "foo",
+				},
+			},
+			expectedSubPath:   "",
+			expectedMountPath: "/foo",
+		},
+	}
+
+	for _, tc := range cases {
+		actualSubPath := ExpandContainerVolumeMounts(tc.container.VolumeMounts[0], tc.envs)
+		if e, a := tc.expectedSubPath, actualSubPath; !reflect.DeepEqual(e, a) {
+			t.Errorf("%v: unexpected subpath; expected %v, got %v", tc.name, e, a)
+		}
+		if e, a := tc.expectedMountPath, tc.container.VolumeMounts[0].MountPath; !reflect.DeepEqual(e, a) {
+			t.Errorf("%v: unexpected mountpath; expected %v, got %v", tc.name, e, a)
+		}
+	}
+
+}
+
 func TestShouldContainerBeRestarted(t *testing.T) {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{

pkg/kubelet/kubelet_pods.go

@@ -164,7 +164,8 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
 }
 
 // makeMounts determines the mount points for the given container.
-func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface) ([]kubecontainer.Mount, func(), error) {
+func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface, expandEnvs []kubecontainer.EnvVar) ([]kubecontainer.Mount, func(), error) {
+
 	// Kubernetes only mounts on /etc/hosts if:
 	// - container is not an infrastructure (pause) container
 	// - container is not already mounting on /etc/hosts
@@ -201,6 +202,11 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 				return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
 			}
 
+			// Expand subpath variables
+			if utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
+				mount.SubPath = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
+			}
+
 			if filepath.IsAbs(mount.SubPath) {
 				return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
 			}
@@ -486,17 +492,17 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
 		opts.Devices = append(opts.Devices, blkVolumes...)
 	}
 
-	mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter)
+	envs, err := kl.makeEnvironmentVariables(pod, container, podIP)
 	if err != nil {
-		return nil, cleanupAction, err
+		return nil, nil, err
 	}
-	opts.Mounts = append(opts.Mounts, mounts...)
+	opts.Envs = append(opts.Envs, envs...)
 
-	envs, err := kl.makeEnvironmentVariables(pod, container, podIP)
+	mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter, opts.Envs)
 	if err != nil {
 		return nil, cleanupAction, err
 	}
-	opts.Envs = append(opts.Envs, envs...)
+	opts.Mounts = append(opts.Mounts, mounts...)
 
 	// Disabling adding TerminationMessagePath on Windows as these files would be mounted as docker volume and
 	// Docker for Windows has a bug where only directories can be mounted

pkg/kubelet/kubelet_pods_test.go

@@ -317,7 +317,7 @@ func TestMakeMounts(t *testing.T) {
 				return
 			}
 
-			mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm)
+			mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, nil)
 
 			// validate only the error if we expect an error
 			if tc.expectErr {
@@ -340,7 +340,7 @@ func TestMakeMounts(t *testing.T) {
 				t.Errorf("Failed to enable feature gate for MountPropagation: %v", err)
 				return
 			}
-			mounts, _, err = makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm)
+			mounts, _, err = makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm, nil)
 			if !tc.expectErr {
 				expectedPrivateMounts := []kubecontainer.Mount{}
 				for _, mount := range tc.expectedMounts {
@@ -401,7 +401,7 @@ func TestDisabledSubpath(t *testing.T) {
 	defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true")
 
 	for name, test := range cases {
-		_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm)
+		_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm, nil)
 		if err != nil && !test.expectError {
 			t.Errorf("test %v failed: %v", name, err)
 		}

pkg/kubelet/kubelet_pods_windows_test.go

@@ -64,7 +64,9 @@ func TestMakeMountsWindows(t *testing.T) {
 		},
 	}
 
-	mounts, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes)
+	//mounts, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes)
+	fm := &mount.FakeMounter{}
+	mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm, nil)
 
 	expectedMounts := []kubecontainer.Mount{
 		{