Add keyvault-flexvolume addon (Azure#3498)

kkmsft · Jul 20, 2018 · 76d16a7 · 76d16a7
1 parent 05895b9
commit 76d16a7
Show file tree

Hide file tree

Showing 18 changed files with 340 additions and 0 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -73,6 +73,7 @@ Here are the valid values for the orchestrator types:
 | [cluster-autoscaler](../examples/addons/cluster-autoscaler/README.md) | false               | 1                   | Delivers the Kubernetes cluster autoscaler component. See https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/azure for more info |
 | [nvidia-device-plugin](../examples/addons/nvidia-device-plugin/README.md) | true if using a Kubernetes cluster (v1.10+) with an N-series agent pool               | 1                   | Delivers the Kubernetes NVIDIA device plugin component. See https://github.com/NVIDIA/k8s-device-plugin for more info |
 | container-monitoring                       | false               | 1                   | Delivers the Kubernetes container monitoring component |
+| [keyvault-flexvolume](../examples/addons/keyvault-flexvolume/README.md)                        | false               | as many as linux agent nodes                   | Access secrets, keys, and certs in Azure Key Vault from pods |
 
 To give a bit more info on the `addons` property: We've tried to expose the basic bits of data that allow useful configuration of these cluster features. Here are some example usage patterns that will unpack what `addons` provide:
 

diff --git a/examples/addons/keyvault-flexvolume/README.md b/examples/addons/keyvault-flexvolume/README.md
@@ -0,0 +1,96 @@
+# Azure Key Vault FlexVolume Add-on
+
+[The Azure Key Vault FlexVolume](https://github.com/Azure/kubernetes-keyvault-flexvol) integrates Azure Key Vault with Kubernetes via a FlexVolume.  
+
+With the Azure Key Vault FlexVolume, developers can access application-specific secrets, keys, and certs stored in Azure Key Vault directly from their pods.
+
+Add this add-on to your apimodel as shown below to automatically enable Key Vault FlexVolume in your new Kubernetes cluster.
+
+```json
+{
+    "apiVersion": "vlabs",
+    "properties": {
+      "orchestratorProfile": {
+        "orchestratorType": "Kubernetes",
+        "kubernetesConfig": {
+          "addons": [
+            {
+              "name": "keyvault-flexvolume",
+              "enabled" : true
+            }
+          ]
+        }
+      },
+      "masterProfile": {
+        "count": 1,
+        "dnsPrefix": "",
+        "vmSize": "Standard_DS2_v2",
+      },
+      "agentPoolProfiles": [
+        {
+          "name": "agentpool",
+          "count": 3,
+          "vmSize": "Standard_DS2_v2",
+          "availabilityProfile": "VirtualMachineScaleSets"
+        }
+      ],
+      "linuxProfile": {
+        "adminUsername": "azureuser",
+        "ssh": {
+          "publicKeys": [
+            {
+              "keyData": ""
+            }
+          ]
+        }
+      },
+      "servicePrincipalProfile": {
+        "clientId": "",
+        "secret": ""
+      }
+    }
+  }
+
+```
+
+To validate the add-on is running as expected, run the following commands:
+
+You should see the keyvault flexvolume installer pods running on each agent node:
+
+```bash
+kubectl get pods -n kv
+
+keyvault-flexvolume-f7bx8   1/1       Running   0          3m
+keyvault-flexvolume-rcxbl   1/1       Running   0          3m
+keyvault-flexvolume-z6jm6   1/1       Running   0          3m
+```
+
+Follow the README at https://github.com/Azure/kubernetes-keyvault-flexvol for get started steps.
+
+## 
+To update resources:
+
+```json
+"kubernetesConfig": {
+        "addons": [
+          {
+            "name": "keyvault-flexvolume",
+            "enabled": true,
+            "containers": [
+                {
+                    "name": "keyvault-flexvolume",
+                    "image": "ritazh/kv-flexvol-installer:v0.0.3",
+                    "cpuRequests": "100m",
+                    "memoryRequests": "300Mi",
+                    "cpuLimits": "100m",
+                    "memoryLimits": "300Mi"
+                }
+            ]
+          }
+        ]
+      }
+```
+
+## Supported Orchestrators
+
+Kubernetes
diff --git a/examples/addons/keyvault-flexvolume/kubernetes-keyvault-flexvolume.json b/examples/addons/keyvault-flexvolume/kubernetes-keyvault-flexvolume.json
@@ -0,0 +1,43 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "kubernetesConfig": {
+        "addons": [
+          {
+            "name": "keyvault-flexvolume",
+            "enabled": true
+          }
+        ]
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_DS2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool",
+        "count": 3,
+        "vmSize": "Standard_DS2_v2",
+        "availabilityProfile": "VirtualMachineScaleSets"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/examples/e2e-tests/kubernetes/kubernetes-config/addons-disabled.json b/examples/e2e-tests/kubernetes/kubernetes-config/addons-disabled.json
@@ -13,6 +13,10 @@
             "name": "aci-connector",
             "enabled": false
           },
+          {
+            "name": "keyvault-flexvolume",
+            "enabled": false
+          },
           {
             "name": "kubernetes-dashboard",
             "enabled": false

diff --git a/examples/e2e-tests/kubernetes/kubernetes-config/addons-enabled.json b/examples/e2e-tests/kubernetes/kubernetes-config/addons-enabled.json
@@ -14,6 +14,10 @@
             "name": "aci-connector",
             "enabled": true
           },
+          {
+            "name": "keyvault-flexvolume",
+            "enabled": true
+          },
           {
             "name": "kubernetes-dashboard",
             "enabled": true

diff --git a/parts/k8s/addons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml b/parts/k8s/addons/kubernetesmasteraddons-keyvault-flexvolume-installer.yaml
@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: "EnsureExists"
+  name: kv
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    app: keyvault-flexvolume
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: "EnsureExists"
+  name: keyvault-flexvolume
+  namespace: kv
+spec:
+  template:
+    metadata:
+      labels:
+        app: keyvault-flexvolume
+        kubernetes.io/cluster-service: "true"
+        addonmanager.kubernetes.io/mode: "EnsureExists"
+    spec:
+      tolerations:
+      containers:
+      - name: keyvault-flexvolume
+        image: "ritazh/kv-flexvol-installer:v0.0.3"
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: <kubernetesKeyVaultFlexVolumeInstallerCPURequests>
+            memory: <kubernetesKeyVaultFlexVolumeInstallerMemoryRequests>
+          limits:
+            cpu: <kubernetesKeyVaultFlexVolumeInstallerCPULimit>
+            memory: <kubernetesKeyVaultFlexVolumeInstallerMemoryLimit>
+        env:
+        - name: TARGET_DIR
+          value: "/etc/kubernetes/volumeplugins"
+        volumeMounts:
+        - mountPath: "/etc/kubernetes/volumeplugins"
+          name: volplugins
+      volumes:
+      - hostPath:
+          path: "/etc/kubernetes/volumeplugins"
+        name: volplugins
+      nodeSelector:
+        beta.kubernetes.io/os: linux
diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -240,6 +240,14 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     sed -i "s|<kubernetesClusterAutoscalerUseManagedIdentity>|{{WrapAsVariable "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
 {{end}}
 
+{{if .OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled}}
+    sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerCPURequests>|{{WrapAsVariable "kubernetesKeyVaultFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
+    sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerMemoryRequests>|{{WrapAsVariable "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
+    sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerCPULimit>|{{WrapAsVariable "kubernetesKeyVaultFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
+    sed -i "s|<kubernetesKeyVaultFlexVolumeInstallerMemoryLimit>|{{WrapAsVariable "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml"
+
+
+{{end}}
 {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}}
     sed -i "s|<kubernetesReschedulerSpec>|{{WrapAsVariable "kubernetesReschedulerSpec"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"
     sed -i "s|<kubernetesReschedulerCPURequests>|{{WrapAsVariable "kubernetesReschedulerCPURequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml"

diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t
@@ -127,6 +127,10 @@
     "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]",
     "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]",
     "kubernetesClusterAutoscalerUseManagedIdentity": "[parameters('kubernetesClusterAutoscalerUseManagedIdentity')]",
+    "kubernetesKeyVaultFlexVolumeInstallerCPURequests": "[parameters('kubernetesKeyVaultFlexVolumeInstallerCPURequests')]",
+    "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests": "[parameters('kubernetesKeyVaultFlexVolumeInstallerMemoryRequests')]",
+    "kubernetesKeyVaultFlexVolumeInstallerCPULimit": "[parameters('kubernetesKeyVaultFlexVolumeInstallerCPULimit')]",
+    "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit": "[parameters('kubernetesKeyVaultFlexVolumeInstallerMemoryLimit')]",
     "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]",
     "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]",
     "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]",

diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t
@@ -521,6 +521,34 @@
       },
       "type": "string"
     },
+    "kubernetesKeyVaultFlexVolumeInstallerCPURequests": {
+      {{PopulateClassicModeDefaultValue "kubernetesKeyVaultFlexVolumeInstallerCPURequests"}}
+      "metadata": {
+        "description": "Key Vault FlexVolume Installer CPU Requests"
+      },
+      "type": "string"
+    },
+    "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests": {
+      {{PopulateClassicModeDefaultValue "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests"}}
+      "metadata": {
+        "description": "Key Vault FlexVolume Installer Memory Requests"
+      },
+      "type": "string"
+    },
+    "kubernetesKeyVaultFlexVolumeInstallerCPULimit": {
+      {{PopulateClassicModeDefaultValue "kubernetesKeyVaultFlexVolumeInstallerCPULimit"}}
+      "metadata": {
+        "description": "Key Vault FlexVolume Installer CPU Limit"
+      },
+      "type": "string"
+    },
+    "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit": {
+      {{PopulateClassicModeDefaultValue "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit"}}
+      "metadata": {
+        "description": "Key Vault FlexVolume Installer Memory Limit"
+      },
+      "type": "string"
+    },
     "kubernetesReschedulerSpec": {
       {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}}
       "metadata": {

diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go
@@ -122,6 +122,11 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet
 			"audit-policy.yaml",
 			common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.8.0"),
 		},
+		{
+			"kubernetesmasteraddons-keyvault-flexvolume-installer.yaml",
+			"keyvault-flexvolume-installer.yaml",
+			profile.OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled(),
+		},
 	}
 }
 

diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go
@@ -108,6 +108,8 @@ const (
 	DefaultDashboardAddonName = "kubernetes-dashboard"
 	// DefaultClusterAutoscalerAddonName is the name of the autoscaler addon deployment
 	DefaultClusterAutoscalerAddonName = "cluster-autoscaler"
+	// DefaultKeyVaultFlexVolumeAddonName is the name of the keyvault flexvolume addon deployment
+	DefaultKeyVaultFlexVolumeAddonName = "keyvault-flexvolume"
 	// DefaultKubernetesDNSServiceIP specifies the IP address that kube-dns
 	// listens on by default. must by in the default Service CIDR range.
 	DefaultKubernetesDNSServiceIP = "10.0.0.10"

diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go
@@ -269,6 +269,21 @@ var (
 		},
 	}
 
+	// DefaultKeyVaultFlexVolumeAddonsConfig is the default KeyVault FlexVolume Kubernetes addon Config
+	DefaultKeyVaultFlexVolumeAddonsConfig = api.KubernetesAddon{
+		Name:    DefaultKeyVaultFlexVolumeAddonName,
+		Enabled: helpers.PointerToBool(api.DefaultKeyVaultFlexVolumeAddonEnabled),
+		Containers: []api.KubernetesContainerSpec{
+			{
+				Name:           DefaultKeyVaultFlexVolumeAddonName,
+				CPURequests:    "50m",
+				MemoryRequests: "10Mi",
+				CPULimits:      "50m",
+				MemoryLimits:   "10Mi",
+			},
+		},
+	}
+
 	// DefaultDashboardAddonsConfig is the default kubernetes-dashboard addon Config
 	DefaultDashboardAddonsConfig = api.KubernetesAddon{
 		Name:    DefaultDashboardAddonName,
@@ -433,6 +448,7 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 				DefaultTillerAddonsConfig,
 				DefaultACIConnectorAddonsConfig,
 				DefaultClusterAutoscalerAddonsConfig,
+				DefaultKeyVaultFlexVolumeAddonsConfig,
 				DefaultDashboardAddonsConfig,
 				DefaultReschedulerAddonsConfig,
 				DefaultMetricsServerAddonsConfig,
@@ -496,6 +512,11 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 				// Provide default acs-engine config for Azure NetworkPolicy addon
 				o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultAzureNetworkPolicyAddonsConfig)
 			}
+			kv := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultKeyVaultFlexVolumeAddonName)
+			if kv < 0 {
+				// Provide default acs-engine config for KeyVault FlexVolume
+				o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultKeyVaultFlexVolumeAddonsConfig)
+			}
 		}
 		if o.KubernetesConfig.KubernetesImageBase == "" {
 			o.KubernetesConfig.KubernetesImageBase = cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase
@@ -607,6 +628,10 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 		if a.OrchestratorProfile.KubernetesConfig.Addons[aNP].IsEnabled(a.OrchestratorProfile.KubernetesConfig.NetworkPlugin == NetworkPluginAzure && a.OrchestratorProfile.KubernetesConfig.NetworkPolicy == NetworkPolicyAzure) {
 			a.OrchestratorProfile.KubernetesConfig.Addons[aNP] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[aNP], DefaultAzureNetworkPolicyAddonsConfig)
 		}
+		kv := getAddonsIndexByName(a.OrchestratorProfile.KubernetesConfig.Addons, DefaultKeyVaultFlexVolumeAddonName)
+		if a.OrchestratorProfile.KubernetesConfig.Addons[kv].IsEnabled(api.DefaultKeyVaultFlexVolumeAddonEnabled) {
+			a.OrchestratorProfile.KubernetesConfig.Addons[kv] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[kv], DefaultKeyVaultFlexVolumeAddonsConfig)
+		}
 
 		if o.KubernetesConfig.PrivateCluster == nil {
 			o.KubernetesConfig.PrivateCluster = &api.PrivateCluster{}

diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go
@@ -90,6 +90,14 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 					addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName])
 				}
 			}
+			kvFlexVolumeInstallerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultKeyVaultFlexVolumeAddonName)
+			c = getAddonContainersIndexByName(kvFlexVolumeInstallerAddon.Containers, DefaultKeyVaultFlexVolumeAddonName)
+			if c > -1 {
+				addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerCPURequests", kvFlexVolumeInstallerAddon.Containers[c].CPURequests)
+				addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerCPULimit", kvFlexVolumeInstallerAddon.Containers[c].CPULimits)
+				addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests", kvFlexVolumeInstallerAddon.Containers[c].MemoryRequests)
+				addValue(parametersMap, "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit", kvFlexVolumeInstallerAddon.Containers[c].MemoryLimits)
+			}
 			dashboardAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName)
 			c = getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName)
 			if c > -1 {