diff --git a/examples/dualstack/README.md b/examples/dualstack/README.md index 2798f6d024..0e2c06a10a 100644 --- a/examples/dualstack/README.md +++ b/examples/dualstack/README.md @@ -113,10 +113,8 @@ nginx-ipv6 LoadBalancer fd00::6283 2603:1030:805:3::3 80:31140/TCP - Dual-stack clusters are supported only with kubenet and azurecni. - Dual-stack cluster with azurecni are only supported with `bridge` network mode. -- Dual-stack clusters are supported only with Linux. -- Dual-stack clusters with Windows is not supported at this time because it requires - - Kubernetes version 1.19+ and - - [backport to 2004 to support dual-stack containers](https://github.com/Azure/aks-engine/issues/3568). +- Dual-stack clusters are supported on Windows from version 2004 (kernel version 10.0.19041.610) and Kubernetes version 1.19 + - https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv4-ipv6-dual-stack - Dual-stack clusters are supported with - `ipvs` kube-proxy mode (Kubernetes version 1.16+) - `iptables` kube-proxy mode (Kubernetes version 1.18+). diff --git a/examples/dualstack/kubernetes-windows.json b/examples/dualstack/kubernetes-windows.json new file mode 100644 index 0000000000..d8402ce9d0 --- /dev/null +++ b/examples/dualstack/kubernetes-windows.json @@ -0,0 +1,60 @@ +{ + "apiVersion": "vlabs", + "properties": { + "featureFlags": { + "enableIPv6DualStack": true + }, + "orchestratorProfile": { + "orchestratorRelease": "1.19", + "kubernetesConfig": { + "apiServerConfig": { + "--feature-gates": "IPv6DualStack=true" + }, + "kubeletConfig": { + "--feature-gates": "IPv6DualStack=true" + }, + "controllerManagerConfig": { + "--feature-gates": "IPv6DualStack=true" + }, + "kubeProxyMode": "ipvs", + "networkPlugin": "azure", + "networkMode": "bridge", + "networkPolicy": "", + "useManagedIdentity": false + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v3" + }, + "agentPoolProfiles": [ + { + "name": "windowspool2", + "count": 1, + "vmSize": "Standard_D2_v3", + "availabilityProfile": "VirtualMachineScaleSets", + "osType": "Windows", + "osDiskSizeGB": 128 + } + ], + "windowsProfile": { + "windowsPublisher": "MicrosoftWindowsServer", + "windowsOffer": "WindowsServer", + "windowsSku": "Datacenter-Core-2004-with-Containers-smalldisk", + "imageVersion": "latest", + "adminUsername": "azureuser", + "adminPassword": "replacepassword1234$" + }, + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + } + } +} diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 40f6fc1808..aa1d828120 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -435,7 +435,12 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error { // validate os type is linux if dual stack feature is enabled if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() { if agentPoolProfile.OSType == Windows { - return errors.Errorf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) + if a.FeatureFlags.IsIPv6DualStackEnabled() && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.19.0") { + return errors.Errorf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", a.OrchestratorProfile.OrchestratorVersion) + } + if a.FeatureFlags.IsIPv6OnlyEnabled() { + return errors.Errorf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) + } } if agentPoolProfile.Distro == Flatcar { return errors.Errorf("Dual stack and single stack IPv6 feature is currently supported only with Ubuntu, but agent pool '%s' is of distro type %s", agentPoolProfile.Name, agentPoolProfile.Distro) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 82caf7cc04..11d1f1a57a 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -4099,20 +4099,30 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { } }) - t.Run("Should not support os type other than linux for single stack ipv6 and dual stack feature", func(t *testing.T) { + t.Run("Should not support os type other than linux for single stack ipv6 and versions less than 1.19 for dual stack feature", func(t *testing.T) { t.Parallel() cs := getK8sDefaultContainerService(true) + + masterProfile := cs.Properties.MasterProfile + masterProfile.Distro = Ubuntu + agentPoolProfiles := cs.Properties.AgentPoolProfiles + + agentPoolProfiles[0].OSType = Windows + cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true} + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18" + expectedMsg := fmt.Sprintf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion) + if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { + t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) + } + + cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6Only: true} + expectedMsg = fmt.Sprintf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType) + if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { + t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) + } + for _, featureFlags := range []FeatureFlags{{EnableIPv6DualStack: true}, {EnableIPv6Only: true}} { cs.Properties.FeatureFlags = &featureFlags - masterProfile := cs.Properties.MasterProfile - - masterProfile.Distro = Ubuntu - agentPoolProfiles := cs.Properties.AgentPoolProfiles - agentPoolProfiles[0].OSType = Windows - expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType) - if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { - t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) - } agentPoolProfiles[0].OSType = Linux agentPoolProfiles[0].Distro = Flatcar @@ -4121,7 +4131,6 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) } } - }) } diff --git a/staging/provisioning/windows/windowsnodereset.ps1 b/staging/provisioning/windows/windowsnodereset.ps1 index 6df9d17289..98e89c7f50 100644 --- a/staging/provisioning/windows/windowsnodereset.ps1 +++ b/staging/provisioning/windows/windowsnodereset.ps1 @@ -15,6 +15,7 @@ $global:NetworkMode = "L2Bridge" $global:NetworkPlugin = $Global:ClusterConfiguration.Cni.Name $global:ContainerRuntime = $Global:ClusterConfiguration.Cri.Name $UseContainerD = ($global:ContainerRuntime -eq "containerd") +$IsDualStackEnabled = $Global:ClusterConfiguration.Kubernetes.Kubeproxy.FeatureGates -contains "IPv6DualStack=true" filter Timestamp { "$(Get-Date -Format o): $_" } @@ -45,6 +46,35 @@ if ($global:EnableHostsConfigAgent) { Stop-Service hosts-config-agent } +# Due to a bug in hns there is a race where it picks up the incorrect IPv6 address from the node in some cases. +# Hns service has to be restarted after the node internal IPv6 address is available when dual-stack is enabled. +# TODO Remove this once the bug is fixed in hns. +function Restart-HnsService { + do { + Start-Sleep -Seconds 1 + $nodeInternalIPv6Address = (Get-NetIPAddress | Where-Object {$_.PrefixOrigin -eq "Dhcp" -and $_.AddressFamily -eq "IPv6"}).IPAddress + } while ($nodeInternalIPv6Address -eq $null) + Write-Log "Got node internal IPv6 address: $nodeInternalIPv6Address" + + $hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6 + Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address" + + if ($hnsManagementIPv6Address -ne $nodeInternalIPv6Address) { + Restart-Service hns + Write-Log "Restarted hns service" + + $hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6 + Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address after restart" + } + else { + Write-Log "Hns network has correct IPv6 address, not restarting" + } +} + +if ($IsDualStackEnabled) { + Restart-HnsService +} + # # Perform cleanup #