From 7c00f032533ad3daa9db78f8aff3331b550a50cb Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 15:42:04 -0700 Subject: [PATCH 01/10] default to Avail Set for k8s < 1.10 --- pkg/acsengine/defaults.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index a9486a541e..0073d4386e 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -756,6 +756,9 @@ func setStorageDefaults(a *api.Properties) { } if len(profile.AvailabilityProfile) == 0 { profile.AvailabilityProfile = api.VirtualMachineScaleSets + if a.OrchestratorProfile.OrchestratorType == api.Kubernetes && common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.10.0") { + profile.AvailabilityProfile = api.AvailabilitySet + } } if len(profile.ScaleSetEvictionPolicy) == 0 && profile.ScaleSetPriority == api.ScaleSetPriorityLow { profile.ScaleSetEvictionPolicy = api.ScaleSetEvictionPolicyDelete From 7c05995905f67f6ce16fd7da5a1b78c7a0f91fbb Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 15:42:42 -0700 Subject: [PATCH 02/10] forgot not --- pkg/acsengine/defaults.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 0073d4386e..486d28fb0d 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -756,7 +756,7 @@ func setStorageDefaults(a *api.Properties) { } if len(profile.AvailabilityProfile) == 0 { profile.AvailabilityProfile = api.VirtualMachineScaleSets - if a.OrchestratorProfile.OrchestratorType == api.Kubernetes && common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.10.0") { + if a.OrchestratorProfile.OrchestratorType == api.Kubernetes && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.10.0") { profile.AvailabilityProfile = api.AvailabilitySet } } From 95a70f216b0831f426299599465b6a5028c9dbf3 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 15:49:05 -0700 Subject: [PATCH 03/10] unit test --- pkg/acsengine/defaults_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go index cad5d8edc0..c043d4aeb4 100644 --- a/pkg/acsengine/defaults_test.go +++ b/pkg/acsengine/defaults_test.go @@ -479,6 +479,19 @@ func TestStorageProfile(t *testing.T) { t.Fatalf("MasterProfile.StorageProfile did not have the expected configuration, got %s, expected %s", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile, api.ManagedDisks) } + if !properties.AgentPoolProfiles[0].IsAvailabilitySets() { + t.Fatalf("AgentPoolProfile[0].AvailabilityProfile did not have the expected configuration, got %s, expected %s", + properties.AgentPoolProfiles[0].AvailabilityProfile, api.AvailabilitySet) + } + + mockCS = getMockBaseContainerService("1.10.0") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + if !properties.AgentPoolProfiles[0].IsVirtualMachineScaleSets() { + t.Fatalf("AgentPoolProfile[0].AvailabilityProfile did not have the expected configuration, got %s, expected %s", + properties.AgentPoolProfiles[0].AvailabilityProfile, api.VirtualMachineScaleSets) + } + } func TestAgentPoolProfile(t *testing.T) { From cfc002775ed39e3d5a9ef94f6b52d4a778c0aefe Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 15:52:48 -0700 Subject: [PATCH 04/10] Docs --- docs/clusterdefinition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 1be235f1fe..219ce957f0 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -440,7 +440,7 @@ A cluster can have 0 to 12 agent pool profiles. Agent Pool Profiles are used for |Name|Required|Description| |---|---|---| -|availabilityProfile|no|Supported values are `VirtualMachineScaleSets` (default) and `AvailabilitySet`. For Kubernetes clusters before version 1.10, use `AvailabilitySet`. Otherwise, you should use `VirtualMachineScaleSets`| +|availabilityProfile|no|Supported values are `VirtualMachineScaleSets` (default, except for Kubernetes cluster before version 1.10) and `AvailabilitySet`.| |count|yes|Describes the node count| |scaleSetPriority|no|Supported values are `Regular` (default) and `Low`. Only applies to clusters with availabilityProfile `VirtualMachineScaleSets`. Enables the usage of [Low-priority VMs on Scale Sets](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-use-low-priority).| |scaleSetEvictionPolicy|no|Supported values are `Delete` (default) and `Deallocate`. Only applies to clusters with availabilityProfile of `VirtualMachineScaleSets` and scaleSetPriority of `Low`.| From 7323115654fcffbbf2390c1918fc8cdd5509fca4 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 15:55:44 -0700 Subject: [PATCH 05/10] missing s --- docs/clusterdefinition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 219ce957f0..8a274bb1ad 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -440,7 +440,7 @@ A cluster can have 0 to 12 agent pool profiles. Agent Pool Profiles are used for |Name|Required|Description| |---|---|---| -|availabilityProfile|no|Supported values are `VirtualMachineScaleSets` (default, except for Kubernetes cluster before version 1.10) and `AvailabilitySet`.| +|availabilityProfile|no|Supported values are `VirtualMachineScaleSets` (default, except for Kubernetes clusters before version 1.10) and `AvailabilitySet`.| |count|yes|Describes the node count| |scaleSetPriority|no|Supported values are `Regular` (default) and `Low`. Only applies to clusters with availabilityProfile `VirtualMachineScaleSets`. Enables the usage of [Low-priority VMs on Scale Sets](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-use-low-priority).| |scaleSetEvictionPolicy|no|Supported values are `Delete` (default) and `Deallocate`. Only applies to clusters with availabilityProfile of `VirtualMachineScaleSets` and scaleSetPriority of `Low`.| From 60cfd1749c6ec4aa22e91abed7d8028c76cc330f Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 15:56:42 -0700 Subject: [PATCH 06/10] remove override in example --- examples/kubernetes.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/kubernetes.json b/examples/kubernetes.json index aad12b0b8a..2923d007eb 100644 --- a/examples/kubernetes.json +++ b/examples/kubernetes.json @@ -13,8 +13,7 @@ { "name": "agentpool1", "count": 2, - "vmSize": "Standard_D2_v2", - "availabilityProfile": "AvailabilitySet" + "vmSize": "Standard_D2_v2" } ], "linuxProfile": { From 2d61ea829b3bfc2e3d5b36f718865dde938d1831 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 16:08:53 -0700 Subject: [PATCH 07/10] fix unit test --- pkg/acsengine/defaults_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go index c043d4aeb4..e098778a7a 100644 --- a/pkg/acsengine/defaults_test.go +++ b/pkg/acsengine/defaults_test.go @@ -487,6 +487,7 @@ func TestStorageProfile(t *testing.T) { mockCS = getMockBaseContainerService("1.10.0") properties = mockCS.Properties properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + SetPropertiesDefaults(&mockCS, false) if !properties.AgentPoolProfiles[0].IsVirtualMachineScaleSets() { t.Fatalf("AgentPoolProfile[0].AvailabilityProfile did not have the expected configuration, got %s, expected %s", properties.AgentPoolProfiles[0].AvailabilityProfile, api.VirtualMachineScaleSets) From ff64148df5c9415dad92366bd5b173ceb4563fe4 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 17:13:21 -0700 Subject: [PATCH 08/10] fix validation --- pkg/api/vlabs/validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 703de5f4e4..af61688538 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -588,7 +588,7 @@ func (a *Properties) Validate(isUpdate bool) error { } // validation for VMSS for Kubernetes - if a.OrchestratorProfile.OrchestratorType == Kubernetes && (agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets || len(agentPoolProfile.AvailabilityProfile) == 0) { + if a.OrchestratorProfile.OrchestratorType == Kubernetes && agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets { version := common.RationalizeReleaseAndVersion( a.OrchestratorProfile.OrchestratorType, a.OrchestratorProfile.OrchestratorRelease, From 07f3ba565a19c78d07fd1b0306bbe36e7f0cc6de Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 23 May 2018 17:33:21 -0700 Subject: [PATCH 09/10] move instance metadata for k8s --- pkg/api/vlabs/validate.go | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index af61688538..617fcdf9c9 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -587,7 +587,7 @@ func (a *Properties) Validate(isUpdate bool) error { } } - // validation for VMSS for Kubernetes + // validation for VMSS with Kubernetes if a.OrchestratorProfile.OrchestratorType == Kubernetes && agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets { version := common.RationalizeReleaseAndVersion( a.OrchestratorProfile.OrchestratorType, @@ -611,25 +611,9 @@ func (a *Properties) Validate(isUpdate bool) error { return fmt.Errorf("VirtualMachineScaleSets are only available in Kubernetes version %s or greater; unable to validate for Kubernetes version %s", minVersion, version) } - } - - // validation for instanceMetadata using VMSS on Kubernetes - if a.OrchestratorProfile.OrchestratorType == Kubernetes && (agentPoolProfile.AvailabilityProfile == VirtualMachineScaleSets || len(agentPoolProfile.AvailabilityProfile) == 0) { - version := common.RationalizeReleaseAndVersion( - a.OrchestratorProfile.OrchestratorType, - a.OrchestratorProfile.OrchestratorRelease, - a.OrchestratorProfile.OrchestratorVersion, - false) - if version == "" { - return fmt.Errorf("the following user supplied OrchestratorProfile configuration is not supported: OrchestratorType: %s, OrchestratorRelease: %s, OrchestratorVersion: %s. Please check supported Release or Version for this build of acs-engine", a.OrchestratorProfile.OrchestratorType, a.OrchestratorProfile.OrchestratorRelease, a.OrchestratorProfile.OrchestratorVersion) - } - - sv, err := semver.NewVersion(version) - if err != nil { - return fmt.Errorf("could not validate version %s", version) - } - minVersion := "1.10.2" - cons, err := semver.NewConstraint("<" + minVersion) + // validation for instanceMetadata using VMSS with Kubernetes + minVersion = "1.10.2" + cons, err = semver.NewConstraint("<" + minVersion) if err != nil { return fmt.Errorf("could not apply semver constraint < %s against version %s", minVersion, version) } From 37b728aead4680462203731cd95e50e284fc3d7e Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Thu, 24 May 2018 18:03:02 -0700 Subject: [PATCH 10/10] improve error messages --- pkg/api/vlabs/validate.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 617fcdf9c9..776fcef60e 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -608,8 +608,7 @@ func (a *Properties) Validate(isUpdate bool) error { return fmt.Errorf("could not apply semver constraint < %s against version %s", minVersion, version) } if cons.Check(sv) { - return fmt.Errorf("VirtualMachineScaleSets are only available in Kubernetes version %s or greater; unable to validate for Kubernetes version %s", - minVersion, version) + return fmt.Errorf("VirtualMachineScaleSets are only available in Kubernetes version %s or greater. Please set \"orchestratorVersion\" to %s or above", minVersion, minVersion) } // validation for instanceMetadata using VMSS with Kubernetes minVersion = "1.10.2" @@ -619,11 +618,11 @@ func (a *Properties) Validate(isUpdate bool) error { } if a.OrchestratorProfile.KubernetesConfig != nil && a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata != nil { if *a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata && cons.Check(sv) { - return fmt.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\"", minVersion) + return fmt.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\" or set \"orchestratorVersion\" to %s or above", minVersion, minVersion) } } else { if cons.Check(sv) { - return fmt.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\"", minVersion) + return fmt.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\" or set \"orchestratorVersion\" to %s or above", minVersion, minVersion) } } }