From f88396f7f09ab3281f31cdb23ee84dca124eea4d Mon Sep 17 00:00:00 2001 From: Gerardo Saca Date: Thu, 5 Jul 2018 00:32:54 -0700 Subject: [PATCH 1/4] Add UseAksExtension condition --- parts/k8s/kubernetesagentresourcesvmas.t | 6 ++++-- parts/k8s/kubernetesagentresourcesvmss.t | 6 ++++-- parts/k8s/kubernetesmasterresources.t | 9 ++++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index ee939fbf1d..4350b8847e 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -324,8 +324,9 @@ {{end}} } } - }, - { + } + {{if UseAksExtension}} + ,{ "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/computeAksLinuxBilling')]", "apiVersion": "[variables('apiVersionDefault')]", @@ -346,4 +347,5 @@ } } } + {{end}} diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index d9264d2437..4d72dbb94b 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -145,8 +145,9 @@ "commandToExecute": "[concat(variables('provisionScriptParametersCommon'),' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" } } - }, - { + } + {{if UseAksExtension}} + ,{ "name": "[concat(variables('{{.Name}}VMNamePrefix'), '-computeAksLinuxBilling')]", "location": "[variables('location')]", "properties": { @@ -157,6 +158,7 @@ "settings": {} } } + {{end}} {{if UseManagedIdentity}} ,{ "name": "managedIdentityExtension", diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index 9c8aedb63c..69bc8a0060 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -904,8 +904,9 @@ {{end}} } } - }, - { + } + {{if UseAksExtension}} + ,{ "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('masterVMNamePrefix'), copyIndex(variables('masterOffset')), '/computeAksLinuxBilling')]", "apiVersion": "[variables('apiVersionDefault')]", @@ -925,4 +926,6 @@ "settings": { } } - }{{WriteLinkedTemplatesForExtensions}} + } + {{end}} + {{WriteLinkedTemplatesForExtensions}} From 2ae0d4f2ee22b05d87c2cebbd7879b166629eda5 Mon Sep 17 00:00:00 2001 From: Gerardo Saca Date: Sat, 7 Jul 2018 07:09:28 -0700 Subject: [PATCH 2/4] Add UseAksExtension config --- docs/clusterdefinition.md | 2 ++ docs/kubernetes/features.md | 4 ++++ parts/k8s/kubernetesmastercustomdata.yml | 1 + parts/k8s/kubernetesmastervars.t | 2 ++ parts/k8s/kuberneteswinagentresourcesvmas.t | 8 +++++--- parts/k8s/kuberneteswinagentresourcesvmss.t | 6 ++++-- parts/k8s/kuberneteswindowssetup.ps1 | 2 ++ pkg/acsengine/defaults.go | 4 ++++ pkg/acsengine/params_k8s.go | 1 + pkg/acsengine/template_generator.go | 3 +++ pkg/api/const.go | 2 ++ pkg/api/converterfromapi.go | 1 + pkg/api/convertertoapi.go | 1 + pkg/api/types.go | 1 + pkg/api/vlabs/types.go | 1 + 15 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 6b4e339eaf..f90ed85dd4 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -61,6 +61,8 @@ Here are the valid values for the orchestrator types: | useInstanceMetadata | no | Use the Azure cloudprovider instance metadata service for appropriate resource discovery operations. Default is `true` | | useManagedIdentity | no | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false) | +| useAksExtension | no | TBD. Default is `true` | + #### addons `addons` describes various addons configuration. It is a child property of `kubernetesConfig`. Below is a list of currently available addons: diff --git a/docs/kubernetes/features.md b/docs/kubernetes/features.md index 554fd28297..84d29f57ff 100644 --- a/docs/kubernetes/features.md +++ b/docs/kubernetes/features.md @@ -26,6 +26,10 @@ Enable Managed Identity by adding `useManagedIdentity` in `kubernetesConfig`. } ``` + + +## AKS Billing Extension + ## Optional: Disable Kubernetes Role-Based Access Control (RBAC) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 4cf667f5b6..b45ea08c93 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -238,6 +238,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerUseAksExtension"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" {{end}} {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index e0c18fded6..0a26b4b0dd 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -123,6 +123,7 @@ "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]", "kubernetesClusterAutoscalerUseManagedIdentity": "[parameters('kubernetesClusterAutoscalerUseManagedIdentity')]", + "kubernetesClusterAutoscalerUseAksExtension": "[parameters('kubernetesClusterAutoscalerUseAksExtension')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", @@ -138,6 +139,7 @@ "cloudProviderRatelimitQPS": "[parameters('cloudProviderRatelimitQPS')]", "cloudProviderRatelimitBucket": "[parameters('cloudProviderRatelimitBucket')]", "useManagedIdentityExtension": "{{ UseManagedIdentity }}", + "UseAksExtension": "{{ UseAksExtension }}", "useInstanceMetadata": "{{ UseInstanceMetadata }}", "kubernetesKubeDNSSpec": "[parameters('kubernetesKubeDNSSpec')]", "kubernetesDNSMasqSpec": "[parameters('kubernetesDNSMasqSpec')]", diff --git a/parts/k8s/kuberneteswinagentresourcesvmas.t b/parts/k8s/kuberneteswinagentresourcesvmas.t index 4ae8bbeb97..2003c90a13 100644 --- a/parts/k8s/kuberneteswinagentresourcesvmas.t +++ b/parts/k8s/kuberneteswinagentresourcesvmas.t @@ -274,8 +274,9 @@ "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',variables('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',variables('clientPrivateKey'),' -AADClientId ',variables('servicePrincipalClientId'),' -AADClientSecret ',variables('servicePrincipalClientSecret'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]" } } - }, - { + } + {{if UseAksExtension}} + ,{ "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/computeAksLinuxBilling')]", "apiVersion": "[variables('apiVersionDefault')]", @@ -295,4 +296,5 @@ "settings": { } } - } \ No newline at end of file + } + {{end}} \ No newline at end of file diff --git a/parts/k8s/kuberneteswinagentresourcesvmss.t b/parts/k8s/kuberneteswinagentresourcesvmss.t index 6be6d9e0c1..336ddc717a 100644 --- a/parts/k8s/kuberneteswinagentresourcesvmss.t +++ b/parts/k8s/kuberneteswinagentresourcesvmss.t @@ -113,8 +113,9 @@ "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',variables('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',variables('clientPrivateKey'),' -AADClientId ',variables('servicePrincipalClientId'),' -AADClientSecret ',variables('servicePrincipalClientSecret'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]" } } - }, - { + } + {{if UseAksExtension}} + ,{ "name": "[concat(variables('{{.Name}}VMNamePrefix'), '-computeAksLinuxBilling')]", "location": "[variables('location')]", "properties": { @@ -125,6 +126,7 @@ "settings": {} } } + {{end}} {{if UseManagedIdentity}} ,{ "name": "managedIdentityExtension", diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index 6acf4c4e6e..da34878a1f 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -64,6 +64,7 @@ $global:KubeNetwork = "l2bridge" $global:KubeDnsSearchPath = "svc.cluster.local" $global:UseManagedIdentityExtension = "{{WrapAsVariable "useManagedIdentityExtension"}}" +$global:UseAksExtension = "{{WrapAsVariable "UseAksExtension"}}" $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}" $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") @@ -183,6 +184,7 @@ Write-AzureConfig() "primaryAvailabilitySetName": "$global:PrimaryAvailabilitySetName", "primaryScaleSetName": "$global:PrimaryScaleSetName", "useManagedIdentityExtension": $global:UseManagedIdentityExtension, + "UseAksExtension": $global:UseAksExtension, "useInstanceMetadata": $global:UseInstanceMetadata } "@ diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 052e15643d..6231a5007d 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -654,6 +654,10 @@ func setOrchestratorDefaults(cs *api.ContainerService) { a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = helpers.PointerToBool(api.DefaultUseInstanceMetadata) } + if a.OrchestratorProfile.KubernetesConfig.UseAksExtension == nil { + a.OrchestratorProfile.KubernetesConfig.UseAksExtension = helpers.PointerToBool(api.DefaultUseAksExtension) + } + // Configure kubelet setKubeletConfig(cs) // Configure controller-manager diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index e98d2a44fd..846e59e528 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -84,6 +84,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", helpers.IsTrueBoolPointer(clusterAutoscalerAddon.Enabled)) addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity))) + addValue(parametersMap, "kubernetesClusterAutoscalerUseAksExtension", helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseAksExtension)) if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index bc5104d0d0..6064b244e8 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -284,6 +284,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "UseManagedIdentity": func() bool { return cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity }, + "UseAksExtension": func() bool { + return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseAksExtension) + }, "UseInstanceMetadata": func() bool { return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata) }, diff --git a/pkg/api/const.go b/pkg/api/const.go index cb4f3fe697..fee53e14d8 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -97,6 +97,8 @@ const ( DefaultReschedulerAddonEnabled = false // DefaultRBACEnabled determines the acs-engine provided default for enabling kubernetes RBAC DefaultRBACEnabled = true + // DefaultUseAksExtension determines the acs-engine provided default for enabling the AKS billing extension + DefaultUseAksExtension = true // DefaultUseInstanceMetadata determines the acs-engine provided default for enabling Azure cloudprovider instance metadata service DefaultUseInstanceMetadata = true // DefaultSecureKubeletEnabled determines the acs-engine provided default for securing kubelet communications diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 9b62a2d78a..6716e0e4ea 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -728,6 +728,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.CloudProviderRateLimitBucket = api.CloudProviderRateLimitBucket vlabs.CloudProviderRateLimitQPS = api.CloudProviderRateLimitQPS vlabs.UseManagedIdentity = api.UseManagedIdentity + vlabs.UseAksExtension = api.UseAksExtension vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage vlabs.DockerEngineVersion = api.DockerEngineVersion vlabs.CustomCcmImage = api.CustomCcmImage diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 66c1894308..dbe75d9d88 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -685,6 +685,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.CloudProviderRateLimitBucket = vlabs.CloudProviderRateLimitBucket api.CloudProviderRateLimitQPS = vlabs.CloudProviderRateLimitQPS api.UseManagedIdentity = vlabs.UseManagedIdentity + api.UseAksExtension = vlabs.UseAksExtension api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage api.DockerEngineVersion = vlabs.DockerEngineVersion api.CustomCcmImage = vlabs.CustomCcmImage diff --git a/pkg/api/types.go b/pkg/api/types.go index f443dc2a9b..da1412123c 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -287,6 +287,7 @@ type KubernetesConfig struct { DNSServiceIP string `json:"dnsServiceIP,omitempty"` ServiceCIDR string `json:"serviceCidr,omitempty"` UseManagedIdentity bool `json:"useManagedIdentity,omitempty"` + UseAksExtension *bool `json:"UseAksExtension,omitempty"` CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 507e36d225..783e21d466 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -285,6 +285,7 @@ type KubernetesConfig struct { MaxPods int `json:"maxPods,omitempty"` DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` UseManagedIdentity bool `json:"useManagedIdentity,omitempty"` + UseAksExtension *bool `json:"UseAksExtension,omitempty"` CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` CustomCcmImage string `json:"customCcmImage,omitempty"` From 58dd4308e9e1fe87b83e30c05d3654e10aced167 Mon Sep 17 00:00:00 2001 From: Gerardo Saca Date: Sat, 7 Jul 2018 12:36:34 -0700 Subject: [PATCH 3/4] Remove UseAksExtension from autoscaler --- parts/k8s/kubernetesmastercustomdata.yml | 1 - parts/k8s/kubernetesmastervars.t | 1 - pkg/acsengine/params_k8s.go | 1 - 3 files changed, 3 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index b45ea08c93..4cf667f5b6 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -238,7 +238,6 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerUseAksExtension"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" {{end}} {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 0a26b4b0dd..eedf8250e3 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -123,7 +123,6 @@ "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]", "kubernetesClusterAutoscalerUseManagedIdentity": "[parameters('kubernetesClusterAutoscalerUseManagedIdentity')]", - "kubernetesClusterAutoscalerUseAksExtension": "[parameters('kubernetesClusterAutoscalerUseAksExtension')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index 846e59e528..e98d2a44fd 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -84,7 +84,6 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", helpers.IsTrueBoolPointer(clusterAutoscalerAddon.Enabled)) addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity))) - addValue(parametersMap, "kubernetesClusterAutoscalerUseAksExtension", helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseAksExtension)) if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { From d978b9bc32f4007ed7f1bbe8fd7d444778139303 Mon Sep 17 00:00:00 2001 From: Gerardo Saca Date: Wed, 11 Jul 2018 22:06:18 -0700 Subject: [PATCH 4/4] Auto disable AKS extension for national clouds --- docs/clusterdefinition.md | 2 -- docs/kubernetes/features.md | 4 ---- parts/k8s/kubernetesmastervars.t | 1 - parts/k8s/kuberneteswindowssetup.ps1 | 2 -- pkg/acsengine/defaults.go | 4 ---- pkg/acsengine/template_generator.go | 3 ++- pkg/api/const.go | 2 -- pkg/api/converterfromapi.go | 1 - pkg/api/convertertoapi.go | 1 - pkg/api/types.go | 1 - pkg/api/vlabs/types.go | 1 - 11 files changed, 2 insertions(+), 20 deletions(-) diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index f90ed85dd4..6b4e339eaf 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -61,8 +61,6 @@ Here are the valid values for the orchestrator types: | useInstanceMetadata | no | Use the Azure cloudprovider instance metadata service for appropriate resource discovery operations. Default is `true` | | useManagedIdentity | no | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false) | -| useAksExtension | no | TBD. Default is `true` | - #### addons `addons` describes various addons configuration. It is a child property of `kubernetesConfig`. Below is a list of currently available addons: diff --git a/docs/kubernetes/features.md b/docs/kubernetes/features.md index 84d29f57ff..554fd28297 100644 --- a/docs/kubernetes/features.md +++ b/docs/kubernetes/features.md @@ -26,10 +26,6 @@ Enable Managed Identity by adding `useManagedIdentity` in `kubernetesConfig`. } ``` - - -## AKS Billing Extension - ## Optional: Disable Kubernetes Role-Based Access Control (RBAC) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index eedf8250e3..e0c18fded6 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -138,7 +138,6 @@ "cloudProviderRatelimitQPS": "[parameters('cloudProviderRatelimitQPS')]", "cloudProviderRatelimitBucket": "[parameters('cloudProviderRatelimitBucket')]", "useManagedIdentityExtension": "{{ UseManagedIdentity }}", - "UseAksExtension": "{{ UseAksExtension }}", "useInstanceMetadata": "{{ UseInstanceMetadata }}", "kubernetesKubeDNSSpec": "[parameters('kubernetesKubeDNSSpec')]", "kubernetesDNSMasqSpec": "[parameters('kubernetesDNSMasqSpec')]", diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index da34878a1f..6acf4c4e6e 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -64,7 +64,6 @@ $global:KubeNetwork = "l2bridge" $global:KubeDnsSearchPath = "svc.cluster.local" $global:UseManagedIdentityExtension = "{{WrapAsVariable "useManagedIdentityExtension"}}" -$global:UseAksExtension = "{{WrapAsVariable "UseAksExtension"}}" $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}" $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") @@ -184,7 +183,6 @@ Write-AzureConfig() "primaryAvailabilitySetName": "$global:PrimaryAvailabilitySetName", "primaryScaleSetName": "$global:PrimaryScaleSetName", "useManagedIdentityExtension": $global:UseManagedIdentityExtension, - "UseAksExtension": $global:UseAksExtension, "useInstanceMetadata": $global:UseInstanceMetadata } "@ diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 6231a5007d..052e15643d 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -654,10 +654,6 @@ func setOrchestratorDefaults(cs *api.ContainerService) { a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = helpers.PointerToBool(api.DefaultUseInstanceMetadata) } - if a.OrchestratorProfile.KubernetesConfig.UseAksExtension == nil { - a.OrchestratorProfile.KubernetesConfig.UseAksExtension = helpers.PointerToBool(api.DefaultUseAksExtension) - } - // Configure kubelet setKubeletConfig(cs) // Configure controller-manager diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index 6064b244e8..f3f46b6cab 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -285,7 +285,8 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat return cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity }, "UseAksExtension": func() bool { - return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseAksExtension) + cloudSpecConfig := getCloudSpecConfig(cs.Location) + return cloudSpecConfig.CloudName == azurePublicCloud }, "UseInstanceMetadata": func() bool { return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata) diff --git a/pkg/api/const.go b/pkg/api/const.go index fee53e14d8..cb4f3fe697 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -97,8 +97,6 @@ const ( DefaultReschedulerAddonEnabled = false // DefaultRBACEnabled determines the acs-engine provided default for enabling kubernetes RBAC DefaultRBACEnabled = true - // DefaultUseAksExtension determines the acs-engine provided default for enabling the AKS billing extension - DefaultUseAksExtension = true // DefaultUseInstanceMetadata determines the acs-engine provided default for enabling Azure cloudprovider instance metadata service DefaultUseInstanceMetadata = true // DefaultSecureKubeletEnabled determines the acs-engine provided default for securing kubelet communications diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 6716e0e4ea..9b62a2d78a 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -728,7 +728,6 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.CloudProviderRateLimitBucket = api.CloudProviderRateLimitBucket vlabs.CloudProviderRateLimitQPS = api.CloudProviderRateLimitQPS vlabs.UseManagedIdentity = api.UseManagedIdentity - vlabs.UseAksExtension = api.UseAksExtension vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage vlabs.DockerEngineVersion = api.DockerEngineVersion vlabs.CustomCcmImage = api.CustomCcmImage diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index dbe75d9d88..66c1894308 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -685,7 +685,6 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.CloudProviderRateLimitBucket = vlabs.CloudProviderRateLimitBucket api.CloudProviderRateLimitQPS = vlabs.CloudProviderRateLimitQPS api.UseManagedIdentity = vlabs.UseManagedIdentity - api.UseAksExtension = vlabs.UseAksExtension api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage api.DockerEngineVersion = vlabs.DockerEngineVersion api.CustomCcmImage = vlabs.CustomCcmImage diff --git a/pkg/api/types.go b/pkg/api/types.go index da1412123c..f443dc2a9b 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -287,7 +287,6 @@ type KubernetesConfig struct { DNSServiceIP string `json:"dnsServiceIP,omitempty"` ServiceCIDR string `json:"serviceCidr,omitempty"` UseManagedIdentity bool `json:"useManagedIdentity,omitempty"` - UseAksExtension *bool `json:"UseAksExtension,omitempty"` CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 783e21d466..507e36d225 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -285,7 +285,6 @@ type KubernetesConfig struct { MaxPods int `json:"maxPods,omitempty"` DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` UseManagedIdentity bool `json:"useManagedIdentity,omitempty"` - UseAksExtension *bool `json:"UseAksExtension,omitempty"` CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` CustomCcmImage string `json:"customCcmImage,omitempty"`