feat: enable multiple frontend IPs in Standard LB

docs/topics/clusterdefinitions.md

@@ -65,7 +65,8 @@ $ aks-engine get-versions
 | gcLowThreshold                   | no                        | Sets the --image-gc-low-threshold value on the kublet configuration. Default is 80. [See kubelet Garbage Collection](https://kubernetes.io/docs/concepts/cluster-administration/kubelet-garbage-collection/)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
 | kubeletConfig                    | no                        | Configure various runtime configuration for kubelet. See `kubeletConfig` [below](#feat-kubelet-config)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | kubernetesImageBase              | no                        | Specifies the default image base URL (everything preceding the actual image filename) to be used for all kubernetes-related containers such as hyperkube, cloud-controller-manager, pause, addon-manager, heapster, exechealthz etc. e.g., `k8s.gcr.io/`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
-| loadBalancerSku                  | no                        | Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`. If not set, it will be default to basic. Requires Kubernetes 1.11 or newer. NOTE: VMs behind standard SKU load balancer will not be able to access the internet without an outbound rule configured with at least one frontend IP. We have created a loadbalancer with an outbound rule and with agent nodes added to the backend pool, as described in the [Outbound NAT for internal Standard Load Balancer scenarios doc](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-rules-overview#outbound-nat-for-internal-standard-load-balancer-scenarios)                                                                     |
+| loadBalancerSku                  | no                        | Sku of Load Balancer and Public IP. Candidate values are: `basic` and `standard`. If not set, it will be default to "standard". NOTE: Because VMs behind standard SKU load balancer will not be able to access the internet without an outbound rule configured with at least one frontend IP, AKS Engine creates a Load Balancer with an outbound rule and with agent nodes added to the backend pool during cluster creation, as described in the [Outbound NAT for internal Standard Load Balancer scenarios doc](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-rules-overview#outbound-nat-for-internal-standard-load-balancer-scenarios)                                                                     |
+| loadBalancerOutboundIPs                  | no                        | Number of outbound IP addresses (e.g., 3) to use in Standard LoadBalancer configuration. If not set, AKS Engine will configure a single outbound IP address. You may want more than one outbound IP address if you are running a large cluster that is processing lots of connections. See [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#multifesnat) for more documentation about how adding more outbound IP addresses can increase the number of SNAT ports available for use by the Standard Load Balancer in your cluster.                                                                     |
 | networkPlugin                    | no                        | Specifies the network plugin implementation for the cluster. Valid values are:<br>`"azure"` (default), which provides an Azure native networking experience <br>`"kubenet"` for k8s software networking implementation. <br> `"flannel"` for using CoreOS Flannel <br> `"cilium"` for using the default Cilium CNI IPAM (requires the `"cilium"` networkPolicy as well)<br> `"antrea"` for using the Antrea network plugin (requires the `"antrea"` networkPolicy as well)                                                                                                                                                                                                                                                                                                                                                                                                                   |
 | networkPolicy                    | no                        | Specifies the network policy enforcement tool for the cluster (currently Linux-only). Valid values are:<br>`"calico"` for Calico network policy.<br>`"cilium"` for cilium network policy (uses the `"cilium"` networkPlugin exclusively).<br> `"antrea"` for Antrea network policy (uses the `"antrea"` networkPlugin exclusively).<br> `"azure"` (experimental) for Azure CNI-compliant network policy (note: Azure CNI-compliant network policy requires explicit `"networkPlugin": "azure"` configuration as well).<br>See [network policy examples](../../examples/networkpolicy) for more information.                                                                                                                                                                                                                                                                   |
 | privateCluster                   | no                        | Build a cluster without public addresses assigned. See `privateClusters` [below](#feat-private-cluster).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |

pkg/api/common/const.go

@@ -41,6 +41,8 @@ const (
 	DefaultInternalLbStaticIPOffset = 10
 	// DefaultEnableCSIProxyWindows determines if CSI proxy should be enabled by default for Windows nodes
 	DefaultEnableCSIProxyWindows = false
+	// MaxLoadBalancerOutboundIPs is the maximum number of outbound IPs in a Standard LoadBalancer frontend configuration
+	MaxLoadBalancerOutboundIPs = 16
 )
 
 // Availability profiles

pkg/api/converterfromapi.go

@@ -304,6 +304,7 @@ func convertKubernetesConfigToVLabs(apiCfg *KubernetesConfig, vlabsCfg *vlabs.Ku
 	vlabsCfg.UseInstanceMetadata = apiCfg.UseInstanceMetadata
 	vlabsCfg.LoadBalancerSku = apiCfg.LoadBalancerSku
 	vlabsCfg.ExcludeMasterFromStandardLB = apiCfg.ExcludeMasterFromStandardLB
+	vlabsCfg.LoadBalancerOutboundIPs = apiCfg.LoadBalancerOutboundIPs
 	vlabsCfg.EnableRbac = apiCfg.EnableRbac
 	vlabsCfg.EnableSecureKubelet = apiCfg.EnableSecureKubelet
 	vlabsCfg.EnableAggregatedAPIs = apiCfg.EnableAggregatedAPIs

pkg/api/convertertoapi.go

@@ -318,6 +318,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.UseInstanceMetadata = vlabs.UseInstanceMetadata
 	api.LoadBalancerSku = vlabs.LoadBalancerSku
 	api.ExcludeMasterFromStandardLB = vlabs.ExcludeMasterFromStandardLB
+	api.LoadBalancerOutboundIPs = vlabs.LoadBalancerOutboundIPs
 	api.EnableRbac = vlabs.EnableRbac
 	api.EnableSecureKubelet = vlabs.EnableSecureKubelet
 	api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs

pkg/api/types.go

@@ -492,6 +492,7 @@ type KubernetesConfig struct {
 	CtrlMgrRouteReconciliationPeriod  string                `json:"ctrlMgrRouteReconciliationPeriod,omitempty"`
 	LoadBalancerSku                   string                `json:"loadBalancerSku,omitempty"`
 	ExcludeMasterFromStandardLB       *bool                 `json:"excludeMasterFromStandardLB,omitempty"`
+	LoadBalancerOutboundIPs           *int                  `json:"loadBalancerOutboundIPs,omitempty"`
 	AzureCNIVersion                   string                `json:"azureCNIVersion,omitempty"`
 	AzureCNIURLLinux                  string                `json:"azureCNIURLLinux,omitempty"`
 	AzureCNIURLWindows                string                `json:"azureCNIURLWindows,omitempty"`

pkg/api/vlabs/types.go

@@ -368,6 +368,7 @@ type KubernetesConfig struct {
 	CloudProviderDisableOutboundSNAT  *bool                 `json:"cloudProviderDisableOutboundSNAT,omitempty"`
 	LoadBalancerSku                   string                `json:"loadBalancerSku,omitempty"`
 	ExcludeMasterFromStandardLB       *bool                 `json:"excludeMasterFromStandardLB,omitempty"`
+	LoadBalancerOutboundIPs           *int                  `json:"loadBalancerOutboundIPs,omitempty"`
 	AzureCNIVersion                   string                `json:"azureCNIVersion,omitempty"`
 	AzureCNIURLLinux                  string                `json:"azureCNIURLLinux,omitempty"`
 	AzureCNIURLWindows                string                `json:"azureCNIURLWindows,omitempty"`

pkg/api/vlabs/validate.go

@@ -315,13 +315,26 @@ func (a *Properties) ValidateOrchestratorProfile(isUpdate bool) error {
 					}
 				}
 
+				if o.KubernetesConfig.LoadBalancerSku == BasicLoadBalancerSku {
+					if o.KubernetesConfig.LoadBalancerOutboundIPs != nil {
+						return errors.Errorf("kubernetesConfig.loadBalancerOutboundIPs configuration only supported for Standard loadBalancerSku=Standard")
+					}
+				}
+
 				if o.KubernetesConfig.DockerEngineVersion != "" {
 					log.Warnf("docker-engine is deprecated in favor of moby, but you passed in a dockerEngineVersion configuration. This will be ignored.")
 				}
 
 				if o.KubernetesConfig.MaximumLoadBalancerRuleCount < 0 {
 					return errors.New("maximumLoadBalancerRuleCount shouldn't be less than 0")
 				}
+
+				if o.KubernetesConfig.LoadBalancerOutboundIPs != nil {
+					if to.Int(o.KubernetesConfig.LoadBalancerOutboundIPs) > common.MaxLoadBalancerOutboundIPs {
+						return errors.Errorf("kubernetesConfig.loadBalancerOutboundIPs was set to %d, the maximum allowed is %d", to.Int(o.KubernetesConfig.LoadBalancerOutboundIPs), common.MaxLoadBalancerOutboundIPs)
+					}
+				}
+
 				// https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-rules-overview
 				if o.KubernetesConfig.LoadBalancerSku == StandardLoadBalancerSku && o.KubernetesConfig.OutboundRuleIdleTimeoutInMinutes != 0 && (o.KubernetesConfig.OutboundRuleIdleTimeoutInMinutes < 4 || o.KubernetesConfig.OutboundRuleIdleTimeoutInMinutes > 120) {
 					return errors.New("outboundRuleIdleTimeoutInMinutes shouldn't be less than 4 or greater than 120")

pkg/api/vlabs/validate_test.go

@@ -173,6 +173,32 @@ func Test_OrchestratorProfile_Validate(t *testing.T) {
 			},
 			expectedError: "loadBalancerSku is only available in Kubernetes version 1.11.0 or greater; unable to validate for Kubernetes version 1.6.9",
 		},
+		"should error when KubernetesConfig has Basic loadBalancerSku with loadBalancerOutboundIPs config": {
+			properties: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					OrchestratorType:    "Kubernetes",
+					OrchestratorVersion: "1.18.1",
+					KubernetesConfig: &KubernetesConfig{
+						LoadBalancerSku:             BasicLoadBalancerSku,
+						ExcludeMasterFromStandardLB: to.BoolPtr(true),
+						LoadBalancerOutboundIPs:     to.IntPtr(3),
+					},
+				},
+			},
+			expectedError: "kubernetesConfig.loadBalancerOutboundIPs configuration only supported for Standard loadBalancerSku=Standard",
+		},
+		"should error when too many loadBalancerOutboundIPs are configured": {
+			properties: &Properties{
+				OrchestratorProfile: &OrchestratorProfile{
+					OrchestratorType:    "Kubernetes",
+					OrchestratorVersion: "1.18.1",
+					KubernetesConfig: &KubernetesConfig{
+						LoadBalancerOutboundIPs: to.IntPtr(17),
+					},
+				},
+			},
+			expectedError: fmt.Sprintf("kubernetesConfig.loadBalancerOutboundIPs was set to %d, the maximum allowed is %d", 17, common.MaxLoadBalancerOutboundIPs),
+		},
 		"should error when KubernetesConfig has enablePodSecurity enabled with invalid settings": {
 			properties: &Properties{
 				OrchestratorProfile: &OrchestratorProfile{

pkg/engine/armresources.go

@@ -5,6 +5,7 @@ package engine
 
 import (
 	"fmt"
+	"strconv"
 
 	"github.com/Azure/aks-engine/pkg/api"
 	"github.com/Azure/aks-engine/pkg/api/common"
@@ -61,11 +62,24 @@ func GenerateARMResources(cs *api.ContainerService) []interface{} {
 	if cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku == api.StandardLoadBalancerSku &&
 		!isHostedMaster &&
 		!cs.Properties.AnyAgentHasLoadBalancerBackendAddressPoolIDs() {
-		isForMaster := false
-		includeDNS := false
-		publicIPAddress := CreatePublicIPAddress(isForMaster, includeDNS)
+		var publicIPAddresses []PublicIPAddressARM
+		numIps := 1
+		if cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerOutboundIPs != nil {
+			numIps = *cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerOutboundIPs
+		}
+		ipAddressNamePrefix := "agentPublicIPAddressName"
+		for i := 1; i <= numIps; i++ {
+			name := ipAddressNamePrefix
+			if i > 1 {
+				name += strconv.Itoa(i)
+			}
+			publicIPAddresses = append(publicIPAddresses, CreatePublicIPAddressForNodePools(name))
+		}
 		loadBalancer := CreateStandardLoadBalancerForNodePools(cs.Properties, true)
-		armResources = append(armResources, publicIPAddress, loadBalancer)
+		for _, publicIPAddress := range publicIPAddresses {
+			armResources = append(armResources, publicIPAddress)
+		}
+		armResources = append(armResources, loadBalancer)
 	}
 
 	profiles := cs.Properties.AgentPoolProfiles

pkg/engine/armresources_test.go

@@ -2016,17 +2016,115 @@ func TestGenerateARMResourcesWithVMSSAgentPoolAndSLB(t *testing.T) {
 		t.Errorf("unexpected error while comparing ARM resources: %s", diff)
 	}
 
+	// Test with > 1 LB outbound IP address
+	cs.Properties.OrchestratorProfile.KubernetesConfig.LoadBalancerOutboundIPs = to.IntPtr(3)
+	armResources = GenerateARMResources(&cs)
+	agentPublicIPAddress2 := PublicIPAddressARM{
+		ARMResource: ARMResource{
+			APIVersion: "[variables('apiVersionNetwork')]",
+		},
+		PublicIPAddress: network.PublicIPAddress{
+			Location: to.StringPtr("[variables('location')]"),
+			Name:     to.StringPtr("[variables('agentPublicIPAddressName2')]"),
+			PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
+				PublicIPAllocationMethod: network.Static,
+			},
+			Sku: &network.PublicIPAddressSku{
+				Name: "[variables('loadBalancerSku')]",
+			},
+			Type: to.StringPtr("Microsoft.Network/publicIPAddresses"),
+		},
+	}
+	agentPublicIPAddress3 := PublicIPAddressARM{
+		ARMResource: ARMResource{
+			APIVersion: "[variables('apiVersionNetwork')]",
+		},
+		PublicIPAddress: network.PublicIPAddress{
+			Location: to.StringPtr("[variables('location')]"),
+			Name:     to.StringPtr("[variables('agentPublicIPAddressName3')]"),
+			PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{
+				PublicIPAllocationMethod: network.Static,
+			},
+			Sku: &network.PublicIPAddressSku{
+				Name: "[variables('loadBalancerSku')]",
+			},
+			Type: to.StringPtr("Microsoft.Network/publicIPAddresses"),
+		},
+	}
+	agentLoadBalancer.LoadBalancer.LoadBalancerPropertiesFormat.FrontendIPConfigurations = &[]network.FrontendIPConfiguration{
+		{
+			Name: to.StringPtr("[variables('agentLbIPConfigName')]"),
+			FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
+				PublicIPAddress: &network.PublicIPAddress{
+					ID: to.StringPtr("[resourceId('Microsoft.Network/publicIpAddresses',variables('agentPublicIPAddressName'))]"),
+				},
+			},
+		},
+		{
+			Name: to.StringPtr("[variables('agentLbIPConfigName2')]"),
+			FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
+				PublicIPAddress: &network.PublicIPAddress{
+					ID: to.StringPtr("[resourceId('Microsoft.Network/publicIpAddresses',variables('agentPublicIPAddressName2'))]"),
+				},
+			},
+		},
+		{
+			Name: to.StringPtr("[variables('agentLbIPConfigName3')]"),
+			FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{
+				PublicIPAddress: &network.PublicIPAddress{
+					ID: to.StringPtr("[resourceId('Microsoft.Network/publicIpAddresses',variables('agentPublicIPAddressName3'))]"),
+				},
+			},
+		},
+	}
+	expected = []interface{}{
+		agentLoadBalancer,
+		agentPublicIPAddress,
+		agentPublicIPAddress2,
+		agentPublicIPAddress3,
+		agentVMWithAgentLb,
+		masterAvSet,
+		virtualNetwork,
+		networkSecurityGroup,
+		publicIPAddress,
+		loadBalancer,
+		networkInterface,
+		masterVM,
+		masterVMExtension,
+		aksBillingExtension,
+	}
+
+	expectedMap = resourceSliceToMap(expected)
+	actualMap = resourceSliceToMap(armResources)
+
+	if diff := cmp.Diff(actualMap, expectedMap); diff != "" {
+		t.Errorf("unexpected error while comparing ARM resources: %s", diff)
+	}
+
 	// Now test with a validate K8s version for EnableTCPReset
-	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.14.4"
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18.1"
 	armResources = GenerateARMResources(&cs)
 	expectedCustomDataStr = getCustomDataFromJSON(tg.GetMasterCustomDataJSONObject(&cs))
 	(*agentLoadBalancer.LoadBalancer.LoadBalancerPropertiesFormat.OutboundRules)[0].OutboundRulePropertiesFormat.EnableTCPReset = to.BoolPtr(true)
+	(*agentLoadBalancer.LoadBalancer.LoadBalancerPropertiesFormat.OutboundRules)[0].OutboundRulePropertiesFormat.FrontendIPConfigurations = &[]network.SubResource{
+		{
+			ID: to.StringPtr("[variables('agentLbIPConfigID')]"),
+		},
+		{
+			ID: to.StringPtr("[variables('agentLbIPConfigID2')]"),
+		},
+		{
+			ID: to.StringPtr("[variables('agentLbIPConfigID3')]"),
+		},
+	}
 	masterVM.VirtualMachine.VirtualMachineProperties.OsProfile.CustomData = to.StringPtr(expectedCustomDataStr)
 	expectedCustomDataStr = getCustomDataFromJSON(tg.GetKubernetesLinuxNodeCustomDataJSONObject(&cs, cs.Properties.AgentPoolProfiles[0]))
 	agentVMWithAgentLb.VirtualMachineScaleSet.VirtualMachineScaleSetProperties.VirtualMachineProfile.OsProfile.CustomData = to.StringPtr(expectedCustomDataStr)
 	expected = []interface{}{
 		agentLoadBalancer,
 		agentPublicIPAddress,
+		agentPublicIPAddress2,
+		agentPublicIPAddress3,
 		agentVMWithAgentLb,
 		masterAvSet,
 		virtualNetwork,