Add UseAksExtension config

Azure · Jul 7, 2018 · a6bdf49 · a6bdf49
1 parent 9a70e8a
commit a6bdf49
Show file tree

Hide file tree

Showing 15 changed files with 34 additions and 5 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -61,6 +61,8 @@ Here are the valid values for the orchestrator types:
 | useInstanceMetadata             | no       | Use the Azure cloudprovider instance metadata service for appropriate resource discovery operations. Default is `true`                                                                                                                                                                                                                                                                                        |
 | useManagedIdentity              | no       | Includes and uses MSI identities for all interactions with the Azure Resource Manager (ARM) API. Instead of using a static service principal written to /etc/kubernetes/azure.json, Kubernetes will use a dynamic, time-limited token fetched from the MSI extension running on master and agent nodes. This support is currently alpha and requires Kubernetes v1.9.1 or newer. (boolean - default == false) |
 
+| useAksExtension                 | no       | TBD. Default is `true` |
+
 #### addons
 
 `addons` describes various addons configuration. It is a child property of `kubernetesConfig`. Below is a list of currently available addons:

diff --git a/docs/kubernetes/features.md b/docs/kubernetes/features.md
@@ -26,6 +26,10 @@ Enable Managed Identity by adding `useManagedIdentity` in `kubernetesConfig`.
 }
 ```
 
+<a name="feat-billing"></a>
+
+## AKS Billing Extension
+
 <a name="feat-managed-disks"></a>
 
 ## Optional: Disable Kubernetes Role-Based Access Control (RBAC)

diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -237,6 +237,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     sed -i "s|<kubernetesClusterAutoscalerMinNodes>|{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
     sed -i "s|<kubernetesClusterAutoscalerMaxNodes>|{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
     sed -i "s|<kubernetesClusterAutoscalerUseManagedIdentity>|{{WrapAsVariable "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
+    sed -i "s|<kubernetesClusterAutoscalerUseAksExtension>|{{WrapAsVariable "kubernetesClusterAutoscalerUseAksExtension"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml"
 {{end}}
 
 {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}}

diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t
@@ -122,6 +122,7 @@
     "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]",
     "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]",
     "kubernetesClusterAutoscalerUseManagedIdentity": "[parameters('kubernetesClusterAutoscalerUseManagedIdentity')]",
+    "kubernetesClusterAutoscalerUseAksExtension": "[parameters('kubernetesClusterAutoscalerUseAksExtension')]",
     "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]",
     "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]",
     "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]",
@@ -137,6 +138,7 @@
     "cloudProviderRatelimitQPS": "[parameters('cloudProviderRatelimitQPS')]",
     "cloudProviderRatelimitBucket": "[parameters('cloudProviderRatelimitBucket')]",
     "useManagedIdentityExtension": "{{ UseManagedIdentity }}",
+    "UseAksExtension": "{{ UseAksExtension }}",
     "useInstanceMetadata": "{{ UseInstanceMetadata }}",
     "kubernetesKubeDNSSpec": "[parameters('kubernetesKubeDNSSpec')]",
     "kubernetesDNSMasqSpec": "[parameters('kubernetesDNSMasqSpec')]",

diff --git a/parts/k8s/kuberneteswinagentresourcesvmas.t b/parts/k8s/kuberneteswinagentresourcesvmas.t
@@ -274,8 +274,9 @@
           "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',variables('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',variables('clientPrivateKey'),' -AADClientId ',variables('servicePrincipalClientId'),' -AADClientSecret ',variables('servicePrincipalClientSecret'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]"
         }
       }
-    },
-    {
+    }
+    {{if UseAksExtension}}
+    ,{
       "type": "Microsoft.Compute/virtualMachines/extensions",
       "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/computeAksLinuxBilling')]",
       "apiVersion": "[variables('apiVersionDefault')]",
@@ -295,4 +296,5 @@
         "settings": {
         }
       }
-    }
+    }
+    {{end}}
diff --git a/parts/k8s/kuberneteswinagentresourcesvmss.t b/parts/k8s/kuberneteswinagentresourcesvmss.t
@@ -113,8 +113,9 @@
                     "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',variables('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',variables('clientPrivateKey'),' -AADClientId ',variables('servicePrincipalClientId'),' -AADClientSecret ',variables('servicePrincipalClientSecret'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]"
                 }
               }
-            },
-            {
+            }
+            {{if UseAksExtension}}
+            ,{
               "name": "[concat(variables('{{.Name}}VMNamePrefix'), '-computeAksLinuxBilling')]",
               "location": "[variables('location')]",
               "properties": {
@@ -125,6 +126,7 @@
                 "settings": {}
               }
             }
+            {{end}}
             {{if UseManagedIdentity}}
             ,{
               "name": "managedIdentityExtension",

diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1
@@ -64,6 +64,7 @@ $global:KubeNetwork = "l2bridge"
 $global:KubeDnsSearchPath = "svc.cluster.local"
 
 $global:UseManagedIdentityExtension = "{{WrapAsVariable "useManagedIdentityExtension"}}"
+$global:UseAksExtension = "{{WrapAsVariable "UseAksExtension"}}"
 $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}"
 
 $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni")
@@ -183,6 +184,7 @@ Write-AzureConfig()
     "primaryAvailabilitySetName": "$global:PrimaryAvailabilitySetName",
     "primaryScaleSetName": "$global:PrimaryScaleSetName",
     "useManagedIdentityExtension": $global:UseManagedIdentityExtension,
+    "UseAksExtension": $global:UseAksExtension,
     "useInstanceMetadata": $global:UseInstanceMetadata
 }
 "@

diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go
@@ -650,6 +650,10 @@ func setOrchestratorDefaults(cs *api.ContainerService) {
 			a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata = helpers.PointerToBool(api.DefaultUseInstanceMetadata)
 		}
 
+		if a.OrchestratorProfile.KubernetesConfig.UseAksExtension == nil {
+			a.OrchestratorProfile.KubernetesConfig.UseAksExtension = helpers.PointerToBool(api.DefaultUseAksExtension)
+		}
+
 		// Configure kubelet
 		setKubeletConfig(cs)
 		// Configure controller-manager

diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go
@@ -83,6 +83,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
 				addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"])
 				addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", helpers.IsTrueBoolPointer(clusterAutoscalerAddon.Enabled))
 				addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity)))
+				addValue(parametersMap, "kubernetesClusterAutoscalerUseAksExtension", helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseAksExtension))
 				if clusterAutoscalerAddon.Containers[c].Image != "" {
 					addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image)
 				} else {

diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go
@@ -284,6 +284,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 		"UseManagedIdentity": func() bool {
 			return cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity
 		},
+		"UseAksExtension": func() bool {
+			return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseAksExtension)
+		},
 		"UseInstanceMetadata": func() bool {
 			return helpers.IsTrueBoolPointer(cs.Properties.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata)
 		},

diff --git a/pkg/api/const.go b/pkg/api/const.go
@@ -97,6 +97,8 @@ const (
 	DefaultReschedulerAddonEnabled = false
 	// DefaultRBACEnabled determines the acs-engine provided default for enabling kubernetes RBAC
 	DefaultRBACEnabled = true
+	// DefaultUseAksExtension determines the acs-engine provided default for enabling the AKS billing extension
+	DefaultUseAksExtension = true
 	// DefaultUseInstanceMetadata determines the acs-engine provided default for enabling Azure cloudprovider instance metadata service
 	DefaultUseInstanceMetadata = true
 	// DefaultSecureKubeletEnabled determines the acs-engine provided default for securing kubelet communications

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -728,6 +728,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.CloudProviderRateLimitBucket = api.CloudProviderRateLimitBucket
 	vlabs.CloudProviderRateLimitQPS = api.CloudProviderRateLimitQPS
 	vlabs.UseManagedIdentity = api.UseManagedIdentity
+	vlabs.UseAksExtension = api.UseAksExtension
 	vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage
 	vlabs.DockerEngineVersion = api.DockerEngineVersion
 	vlabs.CustomCcmImage = api.CustomCcmImage

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -685,6 +685,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.CloudProviderRateLimitBucket = vlabs.CloudProviderRateLimitBucket
 	api.CloudProviderRateLimitQPS = vlabs.CloudProviderRateLimitQPS
 	api.UseManagedIdentity = vlabs.UseManagedIdentity
+	api.UseAksExtension = vlabs.UseAksExtension
 	api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage
 	api.DockerEngineVersion = vlabs.DockerEngineVersion
 	api.CustomCcmImage = vlabs.CustomCcmImage

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -287,6 +287,7 @@ type KubernetesConfig struct {
 	DNSServiceIP                     string            `json:"dnsServiceIP,omitempty"`
 	ServiceCIDR                      string            `json:"serviceCidr,omitempty"`
 	UseManagedIdentity               bool              `json:"useManagedIdentity,omitempty"`
+	UseAksExtension                  *bool             `json:"UseAksExtension,omitempty"`
 	CustomHyperkubeImage             string            `json:"customHyperkubeImage,omitempty"`
 	DockerEngineVersion              string            `json:"dockerEngineVersion,omitempty"`
 	CustomCcmImage                   string            `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager

diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go
@@ -285,6 +285,7 @@ type KubernetesConfig struct {
 	MaxPods                         int               `json:"maxPods,omitempty"`
 	DockerBridgeSubnet              string            `json:"dockerBridgeSubnet,omitempty"`
 	UseManagedIdentity              bool              `json:"useManagedIdentity,omitempty"`
+	UseAksExtension                 *bool             `json:"UseAksExtension,omitempty"`
 	CustomHyperkubeImage            string            `json:"customHyperkubeImage,omitempty"`
 	DockerEngineVersion             string            `json:"dockerEngineVersion,omitempty"`
 	CustomCcmImage                  string            `json:"customCcmImage,omitempty"`