From 5e7549dce38dd886ead84315bdda82fd72f11320 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 10 Jan 2018 11:44:51 -0800 Subject: [PATCH] Generic cloud-controller-manager config interface (#2017) * etcd client port is static 2397 * humans are the best * using vars for etcd master ports * generic cloud-controller-manager * we need this if after all * added ccm example and using helpers * debug * err.Error() for string * bye bye debug * we do need that sleep, though! --- docs/clusterdefinition.md | 34 +++++++++ .../kubernetes-cloud-controller-manager.json | 39 +++++++++++ parts/k8s/kubernetesmastercustomdata.yml | 5 +- ...rnetesmaster-cloud-controller-manager.yaml | 14 +--- .../defaults-cloud-controller-manager.go | 69 +++++++++++++++++++ pkg/acsengine/defaults.go | 2 + pkg/acsengine/engine.go | 16 ++++- pkg/api/converterfromapi.go | 8 +++ pkg/api/convertertoapi.go | 8 +++ pkg/api/types.go | 1 + pkg/api/vlabs/types.go | 1 + 11 files changed, 181 insertions(+), 16 deletions(-) create mode 100644 examples/kubernetes-config/kubernetes-cloud-controller-manager.json create mode 100644 pkg/acsengine/defaults-cloud-controller-manager.go diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 08b6cb9793..d27d1c9789 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -228,6 +228,40 @@ Below is a list of controller-manager options that are *not* currently user-conf |"--profiling"|"false"| |"--use-service-account-credentials"|"false" ("true" if kubernetesConfig.enableRbac is true)| +#### cloudControllerManagerConfig + +`cloudControllerManagerConfig` declares runtime configuration for the cloud-controller-manager daemon running on all master nodes in a Cloud Controller Manager configuration. Like `kubeletConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom cloud-controller-manager config: + +``` +"kubernetesConfig": { + "cloudControllerManagerConfig": { + "--route-reconciliation-period": "1m" + } +} +``` + +See [here](https://kubernetes.io/docs/reference/generated/cloud-controller-manager/) for a reference of supported controller-manager options. + +Below is a list of cloud-controller-manager options that acs-engine will configure by default: + +|controller-manager option|default value| +|---|---| +|"--route-reconciliation-period"|"10s"| + + +Below is a list of cloud-controller-manager options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces controller-manager configuration, or because a static configuration is required to build a functional cluster: + +|controller-manager option|default value| +|---|---| +|"--kubeconfig"|"/var/lib/kubelet/kubeconfig"| +|"--allocate-node-cidrs"|"false"| +|"--cluster-cidr"|"10.240.0.0/12"| +|"--cluster-name"|| +|"--cloud-provider"|"azure"| +|"--cloud-config"|"/etc/kubernetes/azure.json"| +|"--leader-elect"|"true"| +|"--v"|"2"| + #### apiServerConfig `apiServerConfig` declares runtime configuration for the kube-apiserver daemon running on all master nodes. Like `kubeletConfig` and `controllerManagerConfig` it is a generic key/value object, and a child property of `kubernetesConfig`. An example custom apiserver config: diff --git a/examples/kubernetes-config/kubernetes-cloud-controller-manager.json b/examples/kubernetes-config/kubernetes-cloud-controller-manager.json new file mode 100644 index 0000000000..99a32a6a80 --- /dev/null +++ b/examples/kubernetes-config/kubernetes-cloud-controller-manager.json @@ -0,0 +1,39 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.8", + "kubernetesConfig": { + "useCloudControllerManager": true + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool1", + "count": 3, + "vmSize": "Standard_D2_v2", + "availabilityProfile": "AvailabilitySet" + } + ], + "linuxProfile": { + "adminUsername": "azureUser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 76917a7922..0cfeac38ed 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -237,10 +237,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER # If Calico Policy enabled then update Cluster Cidr sed -i "s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml" {{end}} - {{if UseCloudControllerManager }} - sed -i "s||{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s||{{WrapAsVariable "masterFqdnPrefix"}}|g; s||{{WrapAsVariable "allocateNodeCidrs"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g; s||{{GetCloudControllerManagerRouteReconciliationPeriod .OrchestratorProfile.KubernetesConfig}}|g" \ - /etc/kubernetes/manifests/cloud-controller-manager.yaml + sed -i "s||{{WrapAsVariable "kubernetesCcmImageSpec"}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml" + sed -i "s||{{GetCloudControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml" {{end}} sed -i "s||{{GetControllerManagerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" sed -i "s||{{GetAPIServerConfigKeyVals .OrchestratorProfile.KubernetesConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" diff --git a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml index bef8e55df9..008db5f350 100644 --- a/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml +++ b/parts/k8s/manifests/kubernetesmaster-cloud-controller-manager.yaml @@ -11,18 +11,8 @@ spec: containers: - name: "cloud-controller-manager" image: "" - command: - - "cloud-controller-manager" - - "--kubeconfig=/var/lib/kubelet/kubeconfig" - - "--allocate-node-cidrs=" - - "--cluster-cidr=" - - "--cluster-name=" - - "--cloud-provider=azure" - - "--cloud-config=/etc/kubernetes/azure.json" - - "--leader-elect=true" -# TODO: RBAC support - - "--route-reconciliation-period=" - - "--v=2" + command: ["cloud-controller-manager"] + args: [] volumeMounts: - name: "etc-kubernetes" mountPath: "/etc/kubernetes" diff --git a/pkg/acsengine/defaults-cloud-controller-manager.go b/pkg/acsengine/defaults-cloud-controller-manager.go new file mode 100644 index 0000000000..b9d00002ec --- /dev/null +++ b/pkg/acsengine/defaults-cloud-controller-manager.go @@ -0,0 +1,69 @@ +package acsengine + +import ( + "strconv" + + "github.com/Azure/acs-engine/pkg/api" +) + +func setCloudControllerManagerConfig(cs *api.ContainerService) { + o := cs.Properties.OrchestratorProfile + staticLinuxCloudControllerManagerConfig := map[string]string{ + "--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()), + "--cloud-provider": "azure", + "--cloud-config": "/etc/kubernetes/azure.json", + "--cluster-cidr": o.KubernetesConfig.ClusterSubnet, + "--kubeconfig": "/var/lib/kubelet/kubeconfig", + "--leader-elect": "true", + "--v": "2", + } + + // Set --cluster-name based on appropriate DNS prefix + if cs.Properties.MasterProfile != nil { + staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix + } else if cs.Properties.HostedMasterProfile != nil { + staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix + } + + staticWindowsCloudControllerManagerConfig := make(map[string]string) + for key, val := range staticLinuxCloudControllerManagerConfig { + staticWindowsCloudControllerManagerConfig[key] = val + } + // Windows cloud-controller-manager config overrides + // TODO placeholder for specific config overrides for Windows clusters + + // Default cloud-controller-manager config + defaultCloudControllerManagerConfig := map[string]string{ + "--node-monitor-grace-period": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + } + + // If no user-configurable cloud-controller-manager config values exists, use the defaults + if o.KubernetesConfig.CloudControllerManagerConfig == nil { + o.KubernetesConfig.CloudControllerManagerConfig = defaultCloudControllerManagerConfig + } else { + for key, val := range defaultCloudControllerManagerConfig { + // If we don't have a user-configurable cloud-controller-manager config for each option + if _, ok := o.KubernetesConfig.CloudControllerManagerConfig[key]; !ok { + // then assign the default value + o.KubernetesConfig.CloudControllerManagerConfig[key] = val + } + } + } + + // We don't support user-configurable values for the following, + // so any of the value assignments below will override user-provided values + var overrideCloudControllerManagerConfig map[string]string + if cs.Properties.HasWindows() { + overrideCloudControllerManagerConfig = staticWindowsCloudControllerManagerConfig + } else { + overrideCloudControllerManagerConfig = staticLinuxCloudControllerManagerConfig + } + for key, val := range overrideCloudControllerManagerConfig { + o.KubernetesConfig.CloudControllerManagerConfig[key] = val + } + + // TODO add RBAC support + /*if *o.KubernetesConfig.EnableRbac { + o.KubernetesConfig.CloudControllerManagerConfig["--use-service-account-credentials"] = "true" + }*/ +} diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index c06b7ea13b..4bc16b98d5 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -403,6 +403,8 @@ func setOrchestratorDefaults(cs *api.ContainerService) { setKubeletConfig(cs) // Configure controller-manager setControllerManagerConfig(cs) + // Configure cloud-controller-manager + setCloudControllerManagerConfig(cs) // Configure apiserver setAPIServerConfig(cs) diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 3a61f7d6e0..19c2b826b6 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -512,7 +512,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN) } - if properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager { + if helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager) { kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"] if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" { kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage @@ -875,6 +875,20 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } return strings.TrimSuffix(buf.String(), ", ") }, + "GetCloudControllerManagerConfigKeyVals": func(kc *api.KubernetesConfig) string { + cloudControllerManagerConfig := kc.CloudControllerManagerConfig + // Order by key for consistency + keys := []string{} + for key := range cloudControllerManagerConfig { + keys = append(keys, key) + } + sort.Strings(keys) + var buf bytes.Buffer + for _, key := range keys { + buf.WriteString(fmt.Sprintf("\\\"%s=%s\\\", ", key, cloudControllerManagerConfig[key])) + } + return strings.TrimSuffix(buf.String(), ", ") + }, "GetAPIServerConfigKeyVals": func(kc *api.KubernetesConfig) string { apiServerConfig := kc.APIServerConfig // Order by key for consistency diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index b72e6c5939..1fb3c6000a 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -675,6 +675,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet convertAddonsToVlabs(api, vlabs) convertKubeletConfigToVlabs(api, vlabs) convertControllerManagerConfigToVlabs(api, vlabs) + convertCloudControllerManagerConfigToVlabs(api, vlabs) convertAPIServerConfigToVlabs(api, vlabs) } @@ -692,6 +693,13 @@ func convertControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.Kuberne } } +func convertCloudControllerManagerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { + v.CloudControllerManagerConfig = map[string]string{} + for key, val := range a.CloudControllerManagerConfig { + v.CloudControllerManagerConfig[key] = val + } +} + func convertAPIServerConfigToVlabs(a *KubernetesConfig, v *vlabs.KubernetesConfig) { v.APIServerConfig = map[string]string{} for key, val := range a.APIServerConfig { diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index cc36fddaa6..92ba223b42 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -619,6 +619,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes convertAddonsToAPI(vlabs, api) convertKubeletConfigToAPI(vlabs, api) convertControllerManagerConfigToAPI(vlabs, api) + convertCloudControllerManagerConfigToAPI(vlabs, api) convertAPIServerConfigToAPI(vlabs, api) } @@ -676,6 +677,13 @@ func convertControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *Kubernete } } +func convertCloudControllerManagerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) { + a.CloudControllerManagerConfig = map[string]string{} + for key, val := range v.CloudControllerManagerConfig { + a.CloudControllerManagerConfig[key] = val + } +} + func convertAPIServerConfigToAPI(v *vlabs.KubernetesConfig, a *KubernetesConfig) { a.APIServerConfig = map[string]string{} for key, val := range v.APIServerConfig { diff --git a/pkg/api/types.go b/pkg/api/types.go index 77326266ee..ad5261a534 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -229,6 +229,7 @@ type KubernetesConfig struct { Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` } diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index de58196cec..fa3bfc3307 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -247,6 +247,7 @@ type KubernetesConfig struct { Addons []KubernetesAddon `json:"addons,omitempty"` KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` }