Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

Commit

Permalink
Make --azure-container-registry-config optional (#2546)
Browse files Browse the repository at this point in the history 
* move --azure-container-registry-config to default list

* add docs and unit tests

* added test cases
  • Loading branch information
@yolocs @jackfrancis
yolocs authored and jackfrancis committed Mar 30, 2018
1 parent 3f6b25a commit 05ebc78
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 26 deletions.
2 changes: 1 addition & 1 deletion docs/clusterdefinition.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,14 +171,14 @@ Below is a list of kubelet options that acs-engine will configure by default:
|"--image-gc-high-threshold"|"85"|
|"--image-gc-low-threshold"|"850"|
|"--non-masquerade-cidr"|"10.0.0.0/8"|
|"--azure-container-registry-config"|"/etc/kubernetes/azure.json"|
|"--feature-gates"|No default (can be a comma-separated list). On agent nodes `Accelerators=true` will be applied in the `--feature-gates` option.|

Below is a list of kubelet options that are *not* currently user-configurable, either because a higher order configuration vector is available that enforces kubelet configuration, or because a static configuration is required to build a functional cluster:

|kubelet option|default value|
|---|---|
|"--address"|"0.0.0.0"|
|"--azure-container-registry-config"|"/etc/kubernetes/azure.json"|
|"--allow-privileged"|"true"|
|"--pod-manifest-path"|"/etc/kubernetes/manifests"|
|"--network-plugin"|"cni"|
Expand Down
1 change: 1 addition & 0 deletions pkg/acsengine/defaults-apiserver_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,7 @@ func createContainerService(containerServiceName string, orchestratorVersion str
ContainerRuntime: DefaultContainerRuntime,
NetworkPolicy: DefaultNetworkPolicy,
EtcdVersion: DefaultEtcdVersion,
KubeletConfig: make(map[string]string),
}

cs.Properties.CertificateProfile = &api.CertificateProfile{}
Expand Down
50 changes: 25 additions & 25 deletions pkg/acsengine/defaults-kubelet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,17 @@ func setKubeletConfig(cs *api.ContainerService) {
o := cs.Properties.OrchestratorProfile
cloudSpecConfig := GetCloudSpecConfig(cs.Location)
staticLinuxKubeletConfig := map[string]string{
"--address": "0.0.0.0",
"--allow-privileged": "true",
"--anonymous-auth": "false",
"--authorization-mode": "Webhook",
"--client-ca-file": "/etc/kubernetes/certs/ca.crt",
"--pod-manifest-path": "/etc/kubernetes/manifests",
"--cluster-dns": o.KubernetesConfig.DNSServiceIP,
"--cgroups-per-qos": "true",
"--enforce-node-allocatable": "pods",
"--kubeconfig": "/var/lib/kubelet/kubeconfig",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--keep-terminated-pod-volumes": "false",
"--address": "0.0.0.0",
"--allow-privileged": "true",
"--anonymous-auth": "false",
"--authorization-mode": "Webhook",
"--client-ca-file": "/etc/kubernetes/certs/ca.crt",
"--pod-manifest-path": "/etc/kubernetes/manifests",
"--cluster-dns": o.KubernetesConfig.DNSServiceIP,
"--cgroups-per-qos": "true",
"--enforce-node-allocatable": "pods",
"--kubeconfig": "/var/lib/kubelet/kubeconfig",
"--keep-terminated-pod-volumes": "false",
}

staticWindowsKubeletConfig := make(map[string]string)
Expand All @@ -33,19 +32,20 @@ func setKubeletConfig(cs *api.ContainerService) {

// Default Kubelet config
defaultKubeletConfig := map[string]string{
"--cluster-domain": "cluster.local",
"--network-plugin": "cni",
"--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"],
"--max-pods": strconv.Itoa(DefaultKubernetesKubeletMaxPods),
"--eviction-hard": DefaultKubernetesHardEvictionThreshold,
"--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"],
"--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
"--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
"--non-masquerade-cidr": o.KubernetesConfig.ClusterSubnet,
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
"--event-qps": DefaultKubeletEventQPS,
"--cadvisor-port": DefaultKubeletCadvisorPort,
"--cluster-domain": "cluster.local",
"--network-plugin": "cni",
"--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"],
"--max-pods": strconv.Itoa(DefaultKubernetesKubeletMaxPods),
"--eviction-hard": DefaultKubernetesHardEvictionThreshold,
"--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"],
"--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
"--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
"--non-masquerade-cidr": o.KubernetesConfig.ClusterSubnet,
"--cloud-provider": "azure",
"--cloud-config": "/etc/kubernetes/azure.json",
"--azure-container-registry-config": "/etc/kubernetes/azure.json",
"--event-qps": DefaultKubeletEventQPS,
"--cadvisor-port": DefaultKubeletCadvisorPort,
}

// If no user-configurable kubelet config values exists, use the defaults
Expand Down
68 changes: 68 additions & 0 deletions pkg/acsengine/defaults-kubelet_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,34 @@ import (
"github.com/Azure/acs-engine/pkg/helpers"
)

func TestKubeletConfigDefaults(t *testing.T) {
cs := createContainerService("testcluster", "1.8.6", 3, 2)
setKubeletConfig(cs)
k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
// TODO test all default config values
for key, val := range map[string]string{"--azure-container-registry-config": "/etc/kubernetes/azure.json"} {
if k[key] != val {
t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
key, k[key], val)
}
}

cs = createContainerService("testcluster", "1.8.6", 3, 2)
// TODO test all default overrides
overrideVal := "/etc/override"
cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = map[string]string{
"--azure-container-registry-config": overrideVal,
}
setKubeletConfig(cs)
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
for key, val := range map[string]string{"--azure-container-registry-config": overrideVal} {
if k[key] != val {
t.Fatalf("got unexpected kubelet config value for %s: %s, expected %s",
key, k[key], val)
}
}
}

func TestKubeletConfigUseCloudControllerManager(t *testing.T) {
// Test UseCloudControllerManager = true
cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
Expand All @@ -29,6 +57,46 @@ func TestKubeletConfigUseCloudControllerManager(t *testing.T) {

}

func TestKubeletConfigCloudConfig(t *testing.T) {
// Test default value and custom value for --cloud-config
cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
setKubeletConfig(cs)
k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--cloud-config"] != "/etc/kubernetes/azure.json" {
t.Fatalf("got unexpected '--cloud-config' kubelet config default value: %s",
k["--cloud-config"])
}

cs = createContainerService("testcluster", defaultTestClusterVer, 3, 2)
cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--cloud-config"] = "custom.json"
setKubeletConfig(cs)
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--cloud-config"] != "custom.json" {
t.Fatalf("got unexpected '--cloud-config' kubelet config default value: %s",
k["--cloud-config"])
}
}

func TestKubeletConfigAzureContainerRegistryCofig(t *testing.T) {
// Test default value and custom value for --azure-container-registry-config
cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
setKubeletConfig(cs)
k := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--azure-container-registry-config"] != "/etc/kubernetes/azure.json" {
t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s",
k["--azure-container-registry-config"])
}

cs = createContainerService("testcluster", defaultTestClusterVer, 3, 2)
cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--azure-container-registry-config"] = "custom.json"
setKubeletConfig(cs)
k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
if k["--azure-container-registry-config"] != "custom.json" {
t.Fatalf("got unexpected '--azure-container-registry-config' kubelet config default value: %s",
k["--azure-container-registry-config"])
}
}

func TestKubeletConfigNetworkPolicy(t *testing.T) {
// Test NetworkPolicy = none
cs := createContainerService("testcluster", defaultTestClusterVer, 3, 2)
Expand Down

0 comments on commit 05ebc78

Please sign in to comment.