Generic cloud-controller-manager config interface (#2017)

* etcd client port is static 2397 * humans are the best * using vars for etcd master ports * generic cloud-controller-manager * we need this if after all * added ccm example and using helpers * debug * err.Error() for string * bye bye debug * we do need that sleep, though!
Azure · Jan 10, 2018 · 5e7549d · 5e7549d
1 parent 24c19e0
commit 5e7549d
Show file tree

Hide file tree

Showing 11 changed files with 181 additions and 16 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -228,6 +228,40 @@ Below is a list of controller-manager options that are *not* currently user-conf
 |"--profiling"|"false"|
 |"--use-service-account-credentials"|"false" ("true" if kubernetesConfig.enableRbac is true)|
 
+#### cloudControllerManagerConfig
+
+`cloudControllerManagerConfig` declares runtime configuration for the cloud-controller-manager daemon running on all master nodes in a Cloud Controller Manager configuration. Like `kubeletConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom cloud-controller-manager config:
+
+```
+"kubernetesConfig": {
+    "cloudControllerManagerConfig": {
+        "--route-reconciliation-period": "1m"
+    }
+}
+```
+
+See [here](https://kubernetes.io/docs/reference/generated/cloud-controller-manager/) for a reference of supported controller-manager options.
+
+Below is a list of cloud-controller-manager options that acs-engine will configure by default:
+
+|controller-manager option|default value|
+|---|---|
+|"--route-reconciliation-period"|"10s"|
+
+
+Below is a list of cloud-controller-manager options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces controller-manager configuration, or because a static configuration is required to build a functional cluster:
+
+|controller-manager option|default value|
+|---|---|
+|"--kubeconfig"|"/var/lib/kubelet/kubeconfig"|
+|"--allocate-node-cidrs"|"false"|
+|"--cluster-cidr"|"10.240.0.0/12"|
+|"--cluster-name"|<auto-generated using api model properties>|
+|"--cloud-provider"|"azure"|
+|"--cloud-config"|"/etc/kubernetes/azure.json"|
+|"--leader-elect"|"true"|
+|"--v"|"2"|
+
 #### apiServerConfig
 
 `apiServerConfig` declares runtime configuration for the kube-apiserver daemon running on all master nodes. Like `kubeletConfig` and `controllerManagerConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom apiserver config:

diff --git a/examples/kubernetes-config/kubernetes-cloud-controller-manager.json b/examples/kubernetes-config/kubernetes-cloud-controller-manager.json
@@ -0,0 +1,39 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.8",
+      "kubernetesConfig": {
+        "useCloudControllerManager": true
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v2",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureUser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -237,10 +237,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     # If Calico Policy enabled then update Cluster Cidr
     sed -i "s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml"
 {{end}}
-
 {{if UseCloudControllerManager }}
-    sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s|<masterFqdnPrefix>|{{WrapAsVariable "masterFqdnPrefix"}}|g; s|<allocateNodeCidrs>|{{WrapAsVariable "allocateNodeCidrs"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g; s|<kubernetesCtrlMgrRouteReconciliationPeriod>|{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \
-      /etc/kubernetes/manifests/cloud-controller-manager.yaml
+    sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
+    sed -i "s|<kubernetesCloudControllerManagerConfig>|{{GetCloudControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml"
 {{end}}
     sed -i "s|<kubernetesControllerManagerConfig>|{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
     sed -i "s|<kubernetesAPIServerConfig>|{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"

diff --git a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml
@@ -11,18 +11,8 @@ spec:
   containers:
     - name: "cloud-controller-manager"
       image: "<kubernetesCcmImageSpec>"
-      command: 
-        - "cloud-controller-manager"
-        - "--kubeconfig=/var/lib/kubelet/kubeconfig"
-        - "--allocate-node-cidrs=<allocateNodeCidrs>"
-        - "--cluster-cidr=<kubeClusterCidr>"
-        - "--cluster-name=<masterFqdnPrefix>"
-        - "--cloud-provider=azure"
-        - "--cloud-config=/etc/kubernetes/azure.json"
-        - "--leader-elect=true"
-# TODO: RBAC support
-        - "--route-reconciliation-period=<kubernetesCtrlMgrRouteReconciliationPeriod>"
-        - "--v=2"
+      command: ["cloud-controller-manager"]
+      args: [<kubernetesCloudControllerManagerConfig>]
       volumeMounts: 
         - name: "etc-kubernetes"
           mountPath: "/etc/kubernetes"

diff --git a/pkg/acsengine/defaults-cloud-controller-manager.go b/pkg/acsengine/defaults-cloud-controller-manager.go
@@ -0,0 +1,69 @@
+package acsengine
+
+import (
+	"strconv"
+
+	"github.com/Azure/acs-engine/pkg/api"
+)
+
+func setCloudControllerManagerConfig(cs *api.ContainerService) {
+	o := cs.Properties.OrchestratorProfile
+	staticLinuxCloudControllerManagerConfig := map[string]string{
+		"--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()),
+		"--cloud-provider":      "azure",
+		"--cloud-config":        "/etc/kubernetes/azure.json",
+		"--cluster-cidr":        o.KubernetesConfig.ClusterSubnet,
+		"--kubeconfig":          "/var/lib/kubelet/kubeconfig",
+		"--leader-elect":        "true",
+		"--v":                   "2",
+	}
+
+	// Set --cluster-name based on appropriate DNS prefix
+	if cs.Properties.MasterProfile != nil {
+		staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix
+	} else if cs.Properties.HostedMasterProfile != nil {
+		staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix
+	}
+
+	staticWindowsCloudControllerManagerConfig := make(map[string]string)
+	for key, val := range staticLinuxCloudControllerManagerConfig {
+		staticWindowsCloudControllerManagerConfig[key] = val
+	}
+	// Windows cloud-controller-manager config overrides
+	// TODO placeholder for specific config overrides for Windows clusters
+
+	// Default cloud-controller-manager config
+	defaultCloudControllerManagerConfig := map[string]string{
+		"--node-monitor-grace-period": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
+	}
+
+	// If no user-configurable cloud-controller-manager config values exists, use the defaults
+	if o.KubernetesConfig.CloudControllerManagerConfig == nil {
+		o.KubernetesConfig.CloudControllerManagerConfig = defaultCloudControllerManagerConfig
+	} else {
+		for key, val := range defaultCloudControllerManagerConfig {
+			// If we don't have a user-configurable cloud-controller-manager config for each option
+			if _, ok := o.KubernetesConfig.CloudControllerManagerConfig[key]; !ok {
+				// then assign the default value
+				o.KubernetesConfig.CloudControllerManagerConfig[key] = val
+			}
+		}
+	}
+
+	// We don't support user-configurable values for the following,
+	// so any of the value assignments below will override user-provided values
+	var overrideCloudControllerManagerConfig map[string]string
+	if cs.Properties.HasWindows() {
+		overrideCloudControllerManagerConfig = staticWindowsCloudControllerManagerConfig
+	} else {
+		overrideCloudControllerManagerConfig = staticLinuxCloudControllerManagerConfig
+	}
+	for key, val := range overrideCloudControllerManagerConfig {
+		o.KubernetesConfig.CloudControllerManagerConfig[key] = val
+	}
+
+	// TODO add RBAC support
+	/*if *o.KubernetesConfig.EnableRbac {
+		o.KubernetesConfig.CloudControllerManagerConfig["--use-service-account-credentials"] = "true"
+	}*/
+}
diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go
@@ -403,6 +403,8 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 		setKubeletConfig(cs)
 		// Configure controller-manager
 		setControllerManagerConfig(cs)
+		// Configure cloud-controller-manager
+		setCloudControllerManagerConfig(cs)
 		// Configure apiserver
 		setAPIServerConfig(cs)
 

diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go
@@ -512,7 +512,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s
 			addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN)
 		}
 
-		if properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager {
+		if helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager) {
 			kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"]
 			if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" {
 				kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage
@@ -875,6 +875,20 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			}
 			return strings.TrimSuffix(buf.String(), ", ")
 		},
+		"GetCloudControllerManagerConfigKeyVals": func(kc *api.KubernetesConfig) string {
+			cloudControllerManagerConfig := kc.CloudControllerManagerConfig
+			// Order by key for consistency
+			keys := []string{}
+			for key := range cloudControllerManagerConfig {
+				keys = append(keys, key)
+			}
+			sort.Strings(keys)
+			var buf bytes.Buffer
+			for _, key := range keys {
+				buf.WriteString(fmt.Sprintf("\\\"%s=%s\\\", ", key, cloudControllerManagerConfig[key]))
+			}
+			return strings.TrimSuffix(buf.String(), ", ")
+		},
 		"GetAPIServerConfigKeyVals": func(kc *api.KubernetesConfig) string {
 			apiServerConfig := kc.APIServerConfig
 			// Order by key for consistency

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -675,6 +675,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	convertAddonsToVlabs(api, vlabs)
 	convertKubeletConfigToVlabs(api, vlabs)
 	convertControllerManagerConfigToVlabs(api, vlabs)
+	convertCloudControllerManagerConfigToVlabs(api, vlabs)
 	convertAPIServerConfigToVlabs(api, vlabs)
 }
 
@@ -692,6 +693,13 @@ func convertControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.Kuberne
 	}
 }
 
+func convertCloudControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) {
+	v.CloudControllerManagerConfig = map[string]string{}
+	for key, val := range a.CloudControllerManagerConfig {
+		v.CloudControllerManagerConfig[key] = val
+	}
+}
+
 func convertAPIServerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) {
 	v.APIServerConfig = map[string]string{}
 	for key, val := range a.APIServerConfig {

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -619,6 +619,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	convertAddonsToAPI(vlabs, api)
 	convertKubeletConfigToAPI(vlabs, api)
 	convertControllerManagerConfigToAPI(vlabs, api)
+	convertCloudControllerManagerConfigToAPI(vlabs, api)
 	convertAPIServerConfigToAPI(vlabs, api)
 }
 
@@ -676,6 +677,13 @@ func convertControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *Kubernete
 	}
 }
 
+func convertCloudControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) {
+	a.CloudControllerManagerConfig = map[string]string{}
+	for key, val := range v.CloudControllerManagerConfig {
+		a.CloudControllerManagerConfig[key] = val
+	}
+}
+
 func convertAPIServerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) {
 	a.APIServerConfig = map[string]string{}
 	for key, val := range v.APIServerConfig {

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -229,6 +229,7 @@ type KubernetesConfig struct {
 	Addons                       []KubernetesAddon `json:"addons,omitempty"`
 	KubeletConfig                map[string]string `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig      map[string]string `json:"controllerManagerConfig,omitempty"`
+	CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"`
 	APIServerConfig              map[string]string `json:"apiServerConfig,omitempty"`
 }
 

diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go
@@ -247,6 +247,7 @@ type KubernetesConfig struct {
 	Addons                       []KubernetesAddon `json:"addons,omitempty"`
 	KubeletConfig                map[string]string `json:"kubeletConfig,omitempty"`
 	ControllerManagerConfig      map[string]string `json:"controllerManagerConfig,omitempty"`
+	CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"`
 	APIServerConfig              map[string]string `json:"apiServerConfig,omitempty"`
 }