Azure · jackfrancis · Mar 30, 2018 · Mar 28, 2018 · Mar 28, 2018 · Mar 29, 2018
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -171,14 +171,14 @@ Below is a list of kubelet options that acs-engine will configure by default:
 |"--image-gc-high-threshold"|"85"|
 |"--image-gc-low-threshold"|"850"|
 |"--non-masquerade-cidr"|"10.0.0.0/8"|
+|"--azure-container-registry-config"|"/etc/kubernetes/azure.json"|
 |"--feature-gates"|No default (can be a comma-separated list). On agent nodes `Accelerators=true` will be applied in the `--feature-gates` option.|
 
 Below is a list of kubelet options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster:
 
 |kubelet option|default value|
 |---|---|
 |"--address"|"0.0.0.0"|
-|"--azure-container-registry-config"|"/etc/kubernetes/azure.json"|
 |"--allow-privileged"|"true"|
 |"--pod-manifest-path"|"/etc/kubernetes/manifests"|
 |"--network-plugin"|"cni"|

diff --git a/pkg/acsengine/defaults-apiserver_test.go b/pkg/acsengine/defaults-apiserver_test.go
@@ -313,6 +313,7 @@ func createContainerService(containerServiceName string, orchestratorVersion str
 		ContainerRuntime:    DefaultContainerRuntime,
 		NetworkPolicy:       DefaultNetworkPolicy,
 		EtcdVersion:         DefaultEtcdVersion,
+		KubeletConfig:       make(map[string]string),
 	}
 
 	cs.Properties.CertificateProfile = &api.CertificateProfile{}

diff --git a/pkg/acsengine/defaults-kubelet.go b/pkg/acsengine/defaults-kubelet.go
@@ -12,18 +12,17 @@ func setKubeletConfig(cs *api.ContainerService) {
 	o := cs.Properties.OrchestratorProfile
 	cloudSpecConfig := GetCloudSpecConfig(cs.Location)
 	staticLinuxKubeletConfig := map[string]string{
-		"--address":                         "0.0.0.0",
-		"--allow-privileged":                "true",
-		"--anonymous-auth":                  "false",
-		"--authorization-mode":              "Webhook",
-		"--client-ca-file":                  "/etc/kubernetes/certs/ca.crt",
-		"--pod-manifest-path":               "/etc/kubernetes/manifests",
-		"--cluster-dns":                     o.KubernetesConfig.DNSServiceIP,
-		"--cgroups-per-qos":                 "true",
-		"--enforce-node-allocatable":        "pods",
-		"--kubeconfig":                      "/var/lib/kubelet/kubeconfig",
-		"--azure-container-registry-config": "/etc/kubernetes/azure.json",
-		"--keep-terminated-pod-volumes":     "false",
+		"--address":                     "0.0.0.0",
+		"--allow-privileged":            "true",
+		"--anonymous-auth":              "false",
+		"--authorization-mode":          "Webhook",
+		"--client-ca-file":              "/etc/kubernetes/certs/ca.crt",
+		"--pod-manifest-path":           "/etc/kubernetes/manifests",
+		"--cluster-dns":                 o.KubernetesConfig.DNSServiceIP,
+		"--cgroups-per-qos":             "true",
+		"--enforce-node-allocatable":    "pods",
+		"--kubeconfig":                  "/var/lib/kubelet/kubeconfig",
+		"--keep-terminated-pod-volumes": "false",
 	}
 
 	staticWindowsKubeletConfig := make(map[string]string)
@@ -33,19 +32,20 @@ func setKubeletConfig(cs *api.ContainerService) {
 
 	// Default Kubelet config
 	defaultKubeletConfig := map[string]string{
-		"--cluster-domain":               "cluster.local",
-		"--network-plugin":               "cni",
-		"--pod-infra-container-image":    cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"],
-		"--max-pods":                     strconv.Itoa(DefaultKubernetesKubeletMaxPods),
-		"--eviction-hard":                DefaultKubernetesHardEvictionThreshold,
-		"--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"],
-		"--image-gc-high-threshold":      strconv.Itoa(DefaultKubernetesGCHighThreshold),
-		"--image-gc-low-threshold":       strconv.Itoa(DefaultKubernetesGCLowThreshold),
-		"--non-masquerade-cidr":          o.KubernetesConfig.ClusterSubnet,
-		"--cloud-provider":               "azure",
-		"--cloud-config":                 "/etc/kubernetes/azure.json",
-		"--event-qps":                    DefaultKubeletEventQPS,
-		"--cadvisor-port":                DefaultKubeletCadvisorPort,
+		"--cluster-domain":                  "cluster.local",
+		"--network-plugin":                  "cni",
+		"--pod-infra-container-image":       cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"],
+		"--max-pods":                        strconv.Itoa(DefaultKubernetesKubeletMaxPods),
+		"--eviction-hard":                   DefaultKubernetesHardEvictionThreshold,
+		"--node-status-update-frequency":    KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"],
+		"--image-gc-high-threshold":         strconv.Itoa(DefaultKubernetesGCHighThreshold),
+		"--image-gc-low-threshold":          strconv.Itoa(DefaultKubernetesGCLowThreshold),
+		"--non-masquerade-cidr":             o.KubernetesConfig.ClusterSubnet,
+		"--cloud-provider":                  "azure",
+		"--cloud-config":                    "/etc/kubernetes/azure.json",
+		"--azure-container-registry-config": "/etc/kubernetes/azure.json",
+		"--event-qps":                       DefaultKubeletEventQPS,
+		"--cadvisor-port":                   DefaultKubeletCadvisorPort,
 	}
 
 	// If no user-configurable kubelet config values exists, use the defaults

diff --git a/pkg/acsengine/defaults-kubelet_test.go b/pkg/acsengine/defaults-kubelet_test.go
@@ -6,6 +6,34 @@ import (
 	"github.com/Azure/acs-engine/pkg/helpers"
 )
 
+func TestKubeletConfigDefaults(t *testing.T) {
+	cs := createContainerService("testcluster", "1.8.6", 3, 2)
+	setKubeletConfig(cs)
+	k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	// TODO test all default config values
+	for key, val := range map[string]string{"--azure-container-registry-config": "/etc/kubernetes/azure.json"} {
+		if k[key] != val {
+			t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
+				key, k[key], val)
+		}
+	}
+
+	cs = createContainerService("testcluster", "1.8.6", 3, 2)
+	// TODO test all default overrides
+	overrideVal := "/etc/override"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{
+		"--azure-container-registry-config": overrideVal,
+	}
+	setKubeletConfig(cs)
+	k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	for key, val := range map[string]string{"--azure-container-registry-config": overrideVal} {
+		if k[key] != val {
+			t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
+				key, k[key], val)
+		}
+	}
+}
+
 func TestKubeletConfigUseCloudControllerManager(t *testing.T) {
 	// Test UseCloudControllerManager = true
 	cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
@@ -29,6 +57,46 @@ func TestKubeletConfigUseCloudControllerManager(t *testing.T) {
 
 }
 
+func TestKubeletConfigCloudConfig(t *testing.T) {
+	// Test default value and custom value for --cloud-config
+	cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
+	setKubeletConfig(cs)
+	k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	if k["--cloud-config"] != "/etc/kubernetes/azure.json" {
+		t.Fatalf("got unexpected '--cloud-config' kubelet config default value: %s",
+			k["--cloud-config"])
+	}
+
+	cs = createContainerService("testcluster", defaultTestClusterVer, 3, 2)
+	cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--cloud-config"] = "custom.json"
+	setKubeletConfig(cs)
+	k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	if k["--cloud-config"] != "custom.json" {
+		t.Fatalf("got unexpected '--cloud-config' kubelet config default value: %s",
+			k["--cloud-config"])
+	}
+}
+
+func TestKubeletConfigAzureContainerRegistryCofig(t *testing.T) {
+	// Test default value and custom value for --azure-container-registry-config
+	cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
+	setKubeletConfig(cs)
+	k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	if k["--azure-container-registry-config"] != "/etc/kubernetes/azure.json" {
+		t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s",
+			k["--azure-container-registry-config"])
+	}
+
+	cs = createContainerService("testcluster", defaultTestClusterVer, 3, 2)
+	cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--azure-container-registry-config"] = "custom.json"
+	setKubeletConfig(cs)
+	k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+	if k["--azure-container-registry-config"] != "custom.json" {
+		t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s",
+			k["--azure-container-registry-config"])
+	}
+}
+
 func TestKubeletConfigNetworkPolicy(t *testing.T) {
 	// Test NetworkPolicy = none
 	cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)