From 26e0f7e299c5587e64b7498a7ac9c89eca09338a Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Mon, 12 Mar 2018 16:56:52 +0100 Subject: [PATCH 1/6] Specifying 256GB instead of 128 for etcd disk 128 is the limit of the P10 pricing tier which allocates 500 io/s per disk. With disk like this the master nodes have a constant 30% IOWait load. Etcd being overly needy in iops we need to change tier and 256 is the limit of the next tier (P15) which allocates 1000 io/s. See: https://azure.microsoft.com/en-us/pricing/details/managed-disks/ Signed-off-by: Sylvain Rabot --- pkg/acsengine/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 0f40f07dc0..18cb8c7868 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -105,7 +105,7 @@ const ( // DefaultEtcdVersion specifies the default etcd version to install DefaultEtcdVersion = "3.2.16" // DefaultEtcdDiskSize specifies the default size for Kubernetes master etcd disk volumes in GB - DefaultEtcdDiskSize = "128" + DefaultEtcdDiskSize = "256" // DefaultReschedulerAddonName is the name of the rescheduler addon deployment DefaultReschedulerAddonName = "rescheduler" // DefaultMetricsServerAddonName is the name of the kubernetes Metrics server addon deployment From 820058449249a448b4851732dd3208c1da13b0ba Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 21 Mar 2018 09:07:49 -0700 Subject: [PATCH 2/6] scale up default etcd disk size --- pkg/acsengine/defaults.go | 11 ++++++++++- pkg/acsengine/engine.go | 9 +-------- pkg/api/types.go | 12 ++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 5e49bfefd4..815ad3f48f 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -446,7 +446,16 @@ func setOrchestratorDefaults(cs *api.ContainerService) { } if "" == a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB { - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSize + switch { + case a.TotalNodes() > 20: + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = "2048" + case a.TotalNodes() > 10: + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = "1024" + case a.TotalNodes() > 3: + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = "512" + default: + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSize + } } if a.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() && a.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.OSDiskSizeGB == 0 { diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index a141db3b1f..f33c0e9882 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -676,14 +676,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "jumpboxStorageProfile", cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile) } if cs.Properties.HostedMasterProfile == nil { - var totalNodes int - if cs.Properties.MasterProfile != nil { - totalNodes = cs.Properties.MasterProfile.Count - } - for _, pool := range cs.Properties.AgentPoolProfiles { - totalNodes = totalNodes + pool.Count - } - addValue(parametersMap, "totalNodes", totalNodes) + addValue(parametersMap, "totalNodes", cs.Properties.TotalNodes()) } if properties.OrchestratorProfile.KubernetesConfig == nil || diff --git a/pkg/api/types.go b/pkg/api/types.go index dab27f4cb2..e91082563c 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -564,6 +564,18 @@ func (p *Properties) HasStorageAccountDisks() bool { return false } +// TotalNodes returns the total number of nodes in the cluster configuration +func (p *Properties) TotalNodes() int { + var totalNodes int + if p.MasterProfile != nil { + totalNodes = p.MasterProfile.Count + } + for _, pool := range p.AgentPoolProfiles { + totalNodes = totalNodes + pool.Count + } + return totalNodes +} + // IsCustomVNET returns true if the customer brought their own VNET func (m *MasterProfile) IsCustomVNET() bool { return len(m.VnetSubnetID) > 0 From cd01bda30a4112cbc6fe92c8f0ca2a23f022bb39 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 28 Mar 2018 16:33:37 -0700 Subject: [PATCH 3/6] bump ci --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e03746e77c..f13fccdf9e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,7 +177,7 @@ jobs: echo 'export TIMEOUT=20m' >> $BASH_ENV echo 'export ORCHESTRATOR_RELEASE=1.9' >> $BASH_ENV echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV - echo 'export CREATE_VNET=true' >> $BASH_ENV + echo 'export CREATE_VNET=true' >> $BASH_ENV echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV echo 'export RETAIN_SSH=false' >> $BASH_ENV echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV From 24e0033d0da38569b36bf10fd4878e132bba50e4 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 28 Mar 2018 16:33:45 -0700 Subject: [PATCH 4/6] bump ci --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f13fccdf9e..e03746e77c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -177,7 +177,7 @@ jobs: echo 'export TIMEOUT=20m' >> $BASH_ENV echo 'export ORCHESTRATOR_RELEASE=1.9' >> $BASH_ENV echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV - echo 'export CREATE_VNET=true' >> $BASH_ENV + echo 'export CREATE_VNET=true' >> $BASH_ENV echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV echo 'export RETAIN_SSH=false' >> $BASH_ENV echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV From 67d5c06184bf90620f5e35a88e4e9fca2cf4b227 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 2 Apr 2018 13:53:39 -0700 Subject: [PATCH 5/6] const and tests --- pkg/acsengine/const.go | 6 ++++ pkg/acsengine/defaults.go | 6 ++-- pkg/acsengine/defaults_test.go | 57 ++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 18cb8c7868..e49a238e5d 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -106,6 +106,12 @@ const ( DefaultEtcdVersion = "3.2.16" // DefaultEtcdDiskSize specifies the default size for Kubernetes master etcd disk volumes in GB DefaultEtcdDiskSize = "256" + // EtcdDiskSizeGT3Nodes = size for Kubernetes master etcd disk volumes in GB if > 3 nodes + EtcdDiskSizeGT3Nodes = "512" + // EtcdDiskSizeGT10Nodes = size for Kubernetes master etcd disk volumes in GB if > 10 nodes + EtcdDiskSizeGT10Nodes = "1024" + // EtcdDiskSizeGT20Nodes = size for Kubernetes master etcd disk volumes in GB if > 20 nodes + EtcdDiskSizeGT20Nodes = "2048" // DefaultReschedulerAddonName is the name of the rescheduler addon deployment DefaultReschedulerAddonName = "rescheduler" // DefaultMetricsServerAddonName is the name of the kubernetes Metrics server addon deployment diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 815ad3f48f..80fcec9072 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -448,11 +448,11 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if "" == a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB { switch { case a.TotalNodes() > 20: - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = "2048" + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = EtcdDiskSizeGT20Nodes case a.TotalNodes() > 10: - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = "1024" + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = EtcdDiskSizeGT10Nodes case a.TotalNodes() > 3: - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = "512" + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = EtcdDiskSizeGT3Nodes default: a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSize } diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go index 951f1e5d73..bd73d31323 100644 --- a/pkg/acsengine/defaults_test.go +++ b/pkg/acsengine/defaults_test.go @@ -326,6 +326,63 @@ func TestKubeletFeatureGatesEnsureMasterAndAgentConfigUsedFor1_6_0(t *testing.T) } } +func TestEtcdDiskSize(t *testing.T) { + mockCS := getMockBaseContainerService("1.8.10") + properties := mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 1 + setOrchestratorDefaults(&mockCS) + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != DefaultEtcdDiskSize { + t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, DefaultEtcdDiskSize) + } + + mockCS = getMockBaseContainerService("1.8.10") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 5 + setOrchestratorDefaults(&mockCS) + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != EtcdDiskSizeGT3Nodes { + t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, EtcdDiskSizeGT3Nodes) + } + + mockCS = getMockBaseContainerService("1.8.10") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 5 + properties.AgentPoolProfiles[0].Count = 6 + setOrchestratorDefaults(&mockCS) + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != EtcdDiskSizeGT10Nodes { + t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, EtcdDiskSizeGT10Nodes) + } + + mockCS = getMockBaseContainerService("1.8.10") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 5 + properties.AgentPoolProfiles[0].Count = 16 + setOrchestratorDefaults(&mockCS) + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != EtcdDiskSizeGT20Nodes { + t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, EtcdDiskSizeGT20Nodes) + } + + mockCS = getMockBaseContainerService("1.8.10") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 5 + properties.AgentPoolProfiles[0].Count = 50 + customEtcdDiskSize := "512" + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = customEtcdDiskSize + setOrchestratorDefaults(&mockCS) + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != customEtcdDiskSize { + t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, customEtcdDiskSize) + } +} + func getMockAddon(name string) api.KubernetesAddon { return api.KubernetesAddon{ Name: name, From 3a345012cf92c6e1782cdece15d42be9d9793a0e Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 2 Apr 2018 14:27:26 -0700 Subject: [PATCH 6/6] docs and const name change --- docs/clusterdefinition.md | 1 + pkg/acsengine/const.go | 12 ++++++------ pkg/acsengine/defaults.go | 6 +++--- pkg/acsengine/defaults_test.go | 12 ++++++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 6467f944a2..d00492a137 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -41,6 +41,7 @@ Here are the valid values for the orchestrator types: |enableRbac|no|Enable [Kubernetes RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) (boolean - default == true) | |enableAggregatedAPIs|no|Enable [Kubernetes Aggregated APIs](https://kubernetes.io/docs/concepts/api-extension/apiserver-aggregation/).This is required by [Service Catalog](https://github.com/kubernetes-incubator/service-catalog/blob/master/README.md). (boolean - default == false) | |enableDataEncryptionAtRest|no|Enable [kuberetes data encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/).This is currently an alpha feature. (boolean - default == false) | +|etcdDiskSizeGB|no|Size in GB to assign to etcd data volume. Defaults (if no user value provided) are: 256 GB for clusters up to 3 nodes; 512 GB for clusters with between 4 and 10 nodes; 1024 GB for clusters with between 11 and 20 nodes; and 2048 GB for clusters with more than 20 nodes| |privateCluster|no|Build a cluster without public addresses assigned. See `privateClusters` [below](#feat-private-cluster).| |maxPods|no|The maximum number of pods per node. The minimum valid value, necessary for running kube-system pods, is 5. Default value is 30 when networkPolicy equals azure, 110 otherwise.| |gcHighThreshold|no|Sets the --image-gc-high-threshold value on the kublet configuration. Default is 85. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/) | diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index e49a238e5d..3849f5deb6 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -106,12 +106,12 @@ const ( DefaultEtcdVersion = "3.2.16" // DefaultEtcdDiskSize specifies the default size for Kubernetes master etcd disk volumes in GB DefaultEtcdDiskSize = "256" - // EtcdDiskSizeGT3Nodes = size for Kubernetes master etcd disk volumes in GB if > 3 nodes - EtcdDiskSizeGT3Nodes = "512" - // EtcdDiskSizeGT10Nodes = size for Kubernetes master etcd disk volumes in GB if > 10 nodes - EtcdDiskSizeGT10Nodes = "1024" - // EtcdDiskSizeGT20Nodes = size for Kubernetes master etcd disk volumes in GB if > 20 nodes - EtcdDiskSizeGT20Nodes = "2048" + // DefaultEtcdDiskSizeGT3Nodes = size for Kubernetes master etcd disk volumes in GB if > 3 nodes + DefaultEtcdDiskSizeGT3Nodes = "512" + // DefaultEtcdDiskSizeGT10Nodes = size for Kubernetes master etcd disk volumes in GB if > 10 nodes + DefaultEtcdDiskSizeGT10Nodes = "1024" + // DefaultEtcdDiskSizeGT20Nodes = size for Kubernetes master etcd disk volumes in GB if > 20 nodes + DefaultEtcdDiskSizeGT20Nodes = "2048" // DefaultReschedulerAddonName is the name of the rescheduler addon deployment DefaultReschedulerAddonName = "rescheduler" // DefaultMetricsServerAddonName is the name of the kubernetes Metrics server addon deployment diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 80fcec9072..43a5ca4722 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -448,11 +448,11 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if "" == a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB { switch { case a.TotalNodes() > 20: - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = EtcdDiskSizeGT20Nodes + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSizeGT20Nodes case a.TotalNodes() > 10: - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = EtcdDiskSizeGT10Nodes + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSizeGT10Nodes case a.TotalNodes() > 3: - a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = EtcdDiskSizeGT3Nodes + a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSizeGT3Nodes default: a.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB = DefaultEtcdDiskSize } diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go index bd73d31323..69d0c310c2 100644 --- a/pkg/acsengine/defaults_test.go +++ b/pkg/acsengine/defaults_test.go @@ -342,9 +342,9 @@ func TestEtcdDiskSize(t *testing.T) { properties.OrchestratorProfile.OrchestratorType = "Kubernetes" properties.MasterProfile.Count = 5 setOrchestratorDefaults(&mockCS) - if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != EtcdDiskSizeGT3Nodes { + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != DefaultEtcdDiskSizeGT3Nodes { t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", - properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, EtcdDiskSizeGT3Nodes) + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, DefaultEtcdDiskSizeGT3Nodes) } mockCS = getMockBaseContainerService("1.8.10") @@ -353,9 +353,9 @@ func TestEtcdDiskSize(t *testing.T) { properties.MasterProfile.Count = 5 properties.AgentPoolProfiles[0].Count = 6 setOrchestratorDefaults(&mockCS) - if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != EtcdDiskSizeGT10Nodes { + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != DefaultEtcdDiskSizeGT10Nodes { t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", - properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, EtcdDiskSizeGT10Nodes) + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, DefaultEtcdDiskSizeGT10Nodes) } mockCS = getMockBaseContainerService("1.8.10") @@ -364,9 +364,9 @@ func TestEtcdDiskSize(t *testing.T) { properties.MasterProfile.Count = 5 properties.AgentPoolProfiles[0].Count = 16 setOrchestratorDefaults(&mockCS) - if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != EtcdDiskSizeGT20Nodes { + if properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB != DefaultEtcdDiskSizeGT20Nodes { t.Fatalf("EtcdDiskSizeGB did not have the expected size, got %s, expected %s", - properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, EtcdDiskSizeGT20Nodes) + properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB, DefaultEtcdDiskSizeGT20Nodes) } mockCS = getMockBaseContainerService("1.8.10")