feat: allow creation of dualstack Windows clusters

examples/dualstack/README.md

@@ -113,10 +113,8 @@ nginx-ipv6    LoadBalancer   fd00::6283       2603:1030:805:3::3   80:31140/TCP
 
 - Dual-stack clusters are supported only with kubenet and azurecni.
   - Dual-stack cluster with azurecni are only supported with `bridge` network mode.
-- Dual-stack clusters are supported only with Linux.
-- Dual-stack clusters with Windows is not supported at this time because it requires
-  - Kubernetes version 1.19+ and
-  - [backport to 2004 to support dual-stack containers](https://github.com/Azure/aks-engine/issues/3568).
+- Dual-stack clusters are supported on Windows from version 2004 (kernel version 10.0.19041.610) and Kubernetes version 1.19
+  - https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv4-ipv6-dual-stack
 - Dual-stack clusters are supported with
   - `ipvs` kube-proxy mode (Kubernetes version 1.16+)
   - `iptables` kube-proxy mode (Kubernetes version 1.18+).

examples/dualstack/kubernetes-windows.json

@@ -0,0 +1,60 @@
+{
+    "apiVersion": "vlabs",
+    "properties": {
+        "featureFlags": {
+            "enableIPv6DualStack": true
+        },
+        "orchestratorProfile": {
+            "orchestratorRelease": "1.19",
+            "kubernetesConfig": {
+                "apiServerConfig": {
+                    "--feature-gates": "IPv6DualStack=true"
+                },
+                "kubeletConfig": {
+                    "--feature-gates": "IPv6DualStack=true"
+                },
+                "controllerManagerConfig": {
+                    "--feature-gates": "IPv6DualStack=true"
+                },
+                "kubeProxyMode": "ipvs",
+                "networkPlugin": "azure",
+                "networkMode": "bridge",
+                "networkPolicy": "",
+                "useManagedIdentity": false
+            }
+        },
+        "masterProfile": {
+            "count": 1,
+            "dnsPrefix": "",
+            "vmSize": "Standard_D2_v3"
+        },
+        "agentPoolProfiles": [
+            {
+                "name": "windowspool2",
+                "count": 1,
+                "vmSize": "Standard_D2_v3",
+                "availabilityProfile": "VirtualMachineScaleSets",
+                "osType": "Windows",
+                "osDiskSizeGB": 128
+            }
+        ],
+        "windowsProfile": {
+            "windowsPublisher": "MicrosoftWindowsServer",
+            "windowsOffer": "WindowsServer",
+            "windowsSku": "Datacenter-Core-2004-with-Containers-smalldisk",
+            "imageVersion": "latest",
+            "adminUsername": "azureuser",
+            "adminPassword": "replacepassword1234$"
+        },
+        "linuxProfile": {
+            "adminUsername": "azureuser",
+            "ssh": {
+                "publicKeys": [
+                    {
+                        "keyData": ""
+                    }
+                ]
+            }
+        }
+    }
+}

pkg/api/vlabs/validate.go

@@ -435,7 +435,12 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error {
 		// validate os type is linux if dual stack feature is enabled
 		if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() {
 			if agentPoolProfile.OSType == Windows {
-				return errors.Errorf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType)
+				if a.FeatureFlags.IsIPv6DualStackEnabled() && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.19.0") {
+					return errors.Errorf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", a.OrchestratorProfile.OrchestratorVersion)
+				}
+				if a.FeatureFlags.IsIPv6OnlyEnabled() {
+					return errors.Errorf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType)
+				}
 			}
 			if agentPoolProfile.Distro == Flatcar {
 				return errors.Errorf("Dual stack and single stack IPv6 feature is currently supported only with Ubuntu, but agent pool '%s' is of distro type %s", agentPoolProfile.Name, agentPoolProfile.Distro)

pkg/api/vlabs/validate_test.go

@@ -4099,20 +4099,30 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) {
 		}
 	})
 
-	t.Run("Should not support os type other than linux for single stack ipv6 and dual stack feature", func(t *testing.T) {
+	t.Run("Should not support os type other than linux for single stack ipv6 and versions less than 1.19 for dual stack feature", func(t *testing.T) {
 		t.Parallel()
 		cs := getK8sDefaultContainerService(true)
+
+		masterProfile := cs.Properties.MasterProfile
+		masterProfile.Distro = Ubuntu
+		agentPoolProfiles := cs.Properties.AgentPoolProfiles
+
+		agentPoolProfiles[0].OSType = Windows
+		cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true}
+		cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18"
+		expectedMsg := fmt.Sprintf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion)
+		if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
+			t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
+		}
+
+		cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6Only: true}
+		expectedMsg = fmt.Sprintf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType)
+		if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
+			t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
+		}
+
 		for _, featureFlags := range []FeatureFlags{{EnableIPv6DualStack: true}, {EnableIPv6Only: true}} {
 			cs.Properties.FeatureFlags = &featureFlags
-			masterProfile := cs.Properties.MasterProfile
-
-			masterProfile.Distro = Ubuntu
-			agentPoolProfiles := cs.Properties.AgentPoolProfiles
-			agentPoolProfiles[0].OSType = Windows
-			expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType)
-			if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
-				t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
-			}
 
 			agentPoolProfiles[0].OSType = Linux
 			agentPoolProfiles[0].Distro = Flatcar
@@ -4121,7 +4131,6 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) {
 				t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
 			}
 		}
-
 	})
 }
 

staging/provisioning/windows/windowsnodereset.ps1

@@ -15,6 +15,7 @@ $global:NetworkMode = "L2Bridge"
 $global:NetworkPlugin = $Global:ClusterConfiguration.Cni.Name
 $global:ContainerRuntime = $Global:ClusterConfiguration.Cri.Name
 $UseContainerD = ($global:ContainerRuntime -eq "containerd")
+$IsDualStackEnabled = $Global:ClusterConfiguration.Kubernetes.Kubeproxy.FeatureGates -contains "IPv6DualStack=true"
 
 filter Timestamp { "$(Get-Date -Format o): $_" }
 
@@ -45,6 +46,35 @@ if ($global:EnableHostsConfigAgent) {
     Stop-Service hosts-config-agent
 }
 
+# Due to a bug in hns there is a race where it picks up the incorrect IPv6 address from the node in some cases.
+# Hns service has to be restarted after the node internal IPv6 address is available when dual-stack is enabled.
+# TODO Remove this once the bug is fixed in hns.
+function Restart-HnsService {
+    do {
+        Start-Sleep -Seconds 1
+        $nodeInternalIPv6Address = (Get-NetIPAddress | Where-Object {$_.PrefixOrigin -eq "Dhcp" -and $_.AddressFamily -eq "IPv6"}).IPAddress 
+    } while ($nodeInternalIPv6Address -eq $null)
+    Write-Log "Got node internal IPv6 address: $nodeInternalIPv6Address"
+
+    $hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6
+    Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address"
+
+    if ($hnsManagementIPv6Address -ne $nodeInternalIPv6Address) {
+        Restart-Service hns
+        Write-Log "Restarted hns service"
+
+        $hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6
+        Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address after restart"
+    }
+    else {
+        Write-Log "Hns network has correct IPv6 address, not restarting"
+    }
+}
+
+if ($IsDualStackEnabled) {
+    Restart-HnsService
+}
+
 #
 # Perform cleanup
 #