From 05ebc78b57ce2ef92a83bcf6b81b3a9b46ce7f0b Mon Sep 17 00:00:00 2001 From: chshou Date: Fri, 30 Mar 2018 14:55:45 -0700 Subject: [PATCH] Make --azure-container-registry-config optional (#2546) * move --azure-container-registry-config to default list * add docs and unit tests * added test cases --- docs/clusterdefinition.md | 2 +- pkg/acsengine/defaults-apiserver_test.go | 1 + pkg/acsengine/defaults-kubelet.go | 50 ++++++++--------- pkg/acsengine/defaults-kubelet_test.go | 68 ++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index f6c8b73b23..5d3ac2bb68 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -171,6 +171,7 @@ Below is a list of kubelet options that acs-engine will configure by default: |"--image-gc-high-threshold"|"85"| |"--image-gc-low-threshold"|"850"| |"--non-masquerade-cidr"|"10.0.0.0/8"| +|"--azure-container-registry-config"|"/etc/kubernetes/azure.json"| |"--feature-gates"|No default (can be a comma-separated list). On agent nodes `Accelerators=true` will be applied in the `--feature-gates` option.| Below is a list of kubelet options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster: @@ -178,7 +179,6 @@ Below is a list of kubelet options that are *not* currently user-configurable, e |kubelet option|default value| |---|---| |"--address"|"0.0.0.0"| -|"--azure-container-registry-config"|"/etc/kubernetes/azure.json"| |"--allow-privileged"|"true"| |"--pod-manifest-path"|"/etc/kubernetes/manifests"| |"--network-plugin"|"cni"| diff --git a/pkg/acsengine/defaults-apiserver_test.go b/pkg/acsengine/defaults-apiserver_test.go index e65d215973..8b39c510ae 100644 --- a/pkg/acsengine/defaults-apiserver_test.go +++ b/pkg/acsengine/defaults-apiserver_test.go @@ -313,6 +313,7 @@ func createContainerService(containerServiceName string, orchestratorVersion str ContainerRuntime: DefaultContainerRuntime, NetworkPolicy: DefaultNetworkPolicy, EtcdVersion: DefaultEtcdVersion, + KubeletConfig: make(map[string]string), } cs.Properties.CertificateProfile = &api.CertificateProfile{} diff --git a/pkg/acsengine/defaults-kubelet.go b/pkg/acsengine/defaults-kubelet.go index 09e412a500..463e797a3c 100644 --- a/pkg/acsengine/defaults-kubelet.go +++ b/pkg/acsengine/defaults-kubelet.go @@ -12,18 +12,17 @@ func setKubeletConfig(cs *api.ContainerService) { o := cs.Properties.OrchestratorProfile cloudSpecConfig := GetCloudSpecConfig(cs.Location) staticLinuxKubeletConfig := map[string]string{ - "--address": "0.0.0.0", - "--allow-privileged": "true", - "--anonymous-auth": "false", - "--authorization-mode": "Webhook", - "--client-ca-file": "/etc/kubernetes/certs/ca.crt", - "--pod-manifest-path": "/etc/kubernetes/manifests", - "--cluster-dns": o.KubernetesConfig.DNSServiceIP, - "--cgroups-per-qos": "true", - "--enforce-node-allocatable": "pods", - "--kubeconfig": "/var/lib/kubelet/kubeconfig", - "--azure-container-registry-config": "/etc/kubernetes/azure.json", - "--keep-terminated-pod-volumes": "false", + "--address": "0.0.0.0", + "--allow-privileged": "true", + "--anonymous-auth": "false", + "--authorization-mode": "Webhook", + "--client-ca-file": "/etc/kubernetes/certs/ca.crt", + "--pod-manifest-path": "/etc/kubernetes/manifests", + "--cluster-dns": o.KubernetesConfig.DNSServiceIP, + "--cgroups-per-qos": "true", + "--enforce-node-allocatable": "pods", + "--kubeconfig": "/var/lib/kubelet/kubeconfig", + "--keep-terminated-pod-volumes": "false", } staticWindowsKubeletConfig := make(map[string]string) @@ -33,19 +32,20 @@ func setKubeletConfig(cs *api.ContainerService) { // Default Kubelet config defaultKubeletConfig := map[string]string{ - "--cluster-domain": "cluster.local", - "--network-plugin": "cni", - "--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"], - "--max-pods": strconv.Itoa(DefaultKubernetesKubeletMaxPods), - "--eviction-hard": DefaultKubernetesHardEvictionThreshold, - "--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"], - "--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), - "--non-masquerade-cidr": o.KubernetesConfig.ClusterSubnet, - "--cloud-provider": "azure", - "--cloud-config": "/etc/kubernetes/azure.json", - "--event-qps": DefaultKubeletEventQPS, - "--cadvisor-port": DefaultKubeletCadvisorPort, + "--cluster-domain": "cluster.local", + "--network-plugin": "cni", + "--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"], + "--max-pods": strconv.Itoa(DefaultKubernetesKubeletMaxPods), + "--eviction-hard": DefaultKubernetesHardEvictionThreshold, + "--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"], + "--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "--non-masquerade-cidr": o.KubernetesConfig.ClusterSubnet, + "--cloud-provider": "azure", + "--cloud-config": "/etc/kubernetes/azure.json", + "--azure-container-registry-config": "/etc/kubernetes/azure.json", + "--event-qps": DefaultKubeletEventQPS, + "--cadvisor-port": DefaultKubeletCadvisorPort, } // If no user-configurable kubelet config values exists, use the defaults diff --git a/pkg/acsengine/defaults-kubelet_test.go b/pkg/acsengine/defaults-kubelet_test.go index fd518ddd26..ae7fcf2c0b 100644 --- a/pkg/acsengine/defaults-kubelet_test.go +++ b/pkg/acsengine/defaults-kubelet_test.go @@ -6,6 +6,34 @@ import ( "github.com/Azure/acs-engine/pkg/helpers" ) +func TestKubeletConfigDefaults(t *testing.T) { + cs := createContainerService("testcluster", "1.8.6", 3, 2) + setKubeletConfig(cs) + k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + // TODO test all default config values + for key, val := range map[string]string{"--azure-container-registry-config": "/etc/kubernetes/azure.json"} { + if k[key] != val { + t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s", + key, k[key], val) + } + } + + cs = createContainerService("testcluster", "1.8.6", 3, 2) + // TODO test all default overrides + overrideVal := "/etc/override" + cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{ + "--azure-container-registry-config": overrideVal, + } + setKubeletConfig(cs) + k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + for key, val := range map[string]string{"--azure-container-registry-config": overrideVal} { + if k[key] != val { + t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s", + key, k[key], val) + } + } +} + func TestKubeletConfigUseCloudControllerManager(t *testing.T) { // Test UseCloudControllerManager = true cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2) @@ -29,6 +57,46 @@ func TestKubeletConfigUseCloudControllerManager(t *testing.T) { } +func TestKubeletConfigCloudConfig(t *testing.T) { + // Test default value and custom value for --cloud-config + cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2) + setKubeletConfig(cs) + k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + if k["--cloud-config"] != "/etc/kubernetes/azure.json" { + t.Fatalf("got unexpected '--cloud-config' kubelet config default value: %s", + k["--cloud-config"]) + } + + cs = createContainerService("testcluster", defaultTestClusterVer, 3, 2) + cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--cloud-config"] = "custom.json" + setKubeletConfig(cs) + k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + if k["--cloud-config"] != "custom.json" { + t.Fatalf("got unexpected '--cloud-config' kubelet config default value: %s", + k["--cloud-config"]) + } +} + +func TestKubeletConfigAzureContainerRegistryCofig(t *testing.T) { + // Test default value and custom value for --azure-container-registry-config + cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2) + setKubeletConfig(cs) + k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + if k["--azure-container-registry-config"] != "/etc/kubernetes/azure.json" { + t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s", + k["--azure-container-registry-config"]) + } + + cs = createContainerService("testcluster", defaultTestClusterVer, 3, 2) + cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--azure-container-registry-config"] = "custom.json" + setKubeletConfig(cs) + k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + if k["--azure-container-registry-config"] != "custom.json" { + t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s", + k["--azure-container-registry-config"]) + } +} + func TestKubeletConfigNetworkPolicy(t *testing.T) { // Test NetworkPolicy = none cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)