chore: Add EnableAHUB in WindowsProfile (#3322) (#3434)

* chore: Add EnableAHUB in WindowsProfile (#3322) * Add LicenseType in WindowsProfile * fix run tests in parallel with same variable * Change LicenseType to EnableAHUB * Support LicenseType in VMAS * Update doc * Add E2E test config for Windows AHUB * fix: don't hardcode csi enableproxy in kubeclusterconfig.json (#3127) fix has been manually validated and we want to include this in v0.50.0 Co-authored-by: Mark Rossetti <[email protected]>
Azure · Jun 12, 2020 · 54ce1f0 · 54ce1f0
1 parent f442824
commit 54ce1f0
Show file tree

Hide file tree

Showing 18 changed files with 209 additions and 35 deletions.
diff --git a/docs/topics/clusterdefinitions.md b/docs/topics/clusterdefinitions.md
@@ -691,6 +691,7 @@ https://{keyvaultname}.vault.azure.net:443/secrets/{secretName}/{version}
 | imageReference.gallery        | no       | Name of a Shared Image Gallery.                                                                                                                                                                                                                                 |
 | imageReference.version        | no       | Version of an Image from a Shared Image Gallery.                                                                                                                                                                                                                |
 | sshEnabled                    | no       | If set to `true`, OpenSSH will be installed on windows nodes to allow for ssh remoting. **Only for Windows version 1809/2019 or later**. The same SSH authorized public key(s) will be added from [linuxProfile.ssh.publicKeys](#linuxProfile). Default: `true` |
+| enableAHUB                    | no       | If set to `true`, Windows nodepools was licensed on-premises. If this element is set to `true`, the value cannot be changed to `false`. <br> For more information, see [Azure Hybrid Use Benefit for Windows Server](https://docs.microsoft.com/azure/virtual-machines/virtual-machines-windows-hybrid-use-benefit-licensing?toc=%2fazure%2fvirtual-machines%2fwindows%2ftoc.json) |
 
 #### Windows Images
 

diff --git a/docs/topics/windows-and-kubernetes.md b/docs/topics/windows-and-kubernetes.md
@@ -66,6 +66,21 @@ If you want to disable automatic Windows updates, you can use the `enableAutomat
      },
 ```
 
+### Enabling Azure Hybrid Benefit for Windows Server
+
+If you want to enable [Azure hybrid benefit for Windows server](https://docs.microsoft.com/azure/virtual-machines/virtual-machines-windows-hybrid-use-benefit-licensing?toc=%2fazure%2fvirtual-machines%2fwindows%2ftoc.json), you can use the `enableAHUB` option.
+
+```json
+"windowsProfile": {
+            "adminUsername": "azureuser",
+            "adminPassword": "...",
+            "windowsPublisher": "MicrosoftWindowsServer",
+            "windowsOffer": "WindowsServerSemiAnnual",
+            "windowsSku": "Datacenter-Core-1809-with-Containers-smalldisk",
+            "enableAHUB": true
+     },
+```
+
 ## More Examples
 
 ### Using Azure Files

diff --git a/examples/windows/kubernetes-wincni.json b/examples/windows/kubernetes-wincni.json
@@ -21,15 +21,16 @@
         "osType": "Windows",
         "osDiskSizeGB": 128,
         "extensions": [
-            {
-                "name": "winrm"
-            }
+          {
+            "name": "winrm"
+          }
         ]
       }
     ],
     "windowsProfile": {
       "adminUsername": "azureuser",
-      "adminPassword": "replacepassword1234$"
+      "adminPassword": "replacepassword1234$",
+      "enableAHUB": true
     },
     "linuxProfile": {
       "adminUsername": "azureuser",
@@ -52,4 +53,4 @@
       }
     ]
   }
-}
+}
diff --git a/examples/windows/kubernetes-windows-1903.json b/examples/windows/kubernetes-windows-1903.json
@@ -26,7 +26,8 @@
       "windowsPublisher": "MicrosoftWindowsServer",
       "windowsOffer": "WindowsServer",
       "windowsSku": "Datacenter-Core-1903-with-Containers-smalldisk",
-      "imageVersion": "1903.0.20190603"
+      "imageVersion": "1903.0.20190603",
+      "enableAHUB": true
     },
     "linuxProfile": {
       "adminUsername": "azureuser",
@@ -43,4 +44,4 @@
       "secret": ""
     }
   }
-}
+}
diff --git a/parts/k8s/kuberneteswindowsfunctions.ps1 b/parts/k8s/kuberneteswindowsfunctions.ps1
@@ -243,24 +243,24 @@ function Write-KubeClusterConfig {
     }
 
     $Global:ClusterConfiguration | Add-Member -MemberType NoteProperty -Name Csi -Value @{
-        EnableProxy = "true"
+        EnableProxy = $global:EnableCsiProxy
     }
 
     $Global:ClusterConfiguration | Add-Member -MemberType NoteProperty -Name Kubernetes -Value @{
         Source       = @{
             Release = $global:KubeBinariesVersion;
         };
         ControlPlane = @{
-            IpAddress = $MasterIP;
-            Username  = "azureuser"
+            IpAddress    = $MasterIP;
+            Username     = "azureuser"
             MasterSubnet = $global:MasterSubnet
         };
-        Network = @{
+        Network      = @{
             ServiceCidr = $global:KubeServiceCIDR;
             ClusterCidr = $global:KubeClusterCIDR;
-            DnsIp = $KubeDnsServiceIp
+            DnsIp       = $KubeDnsServiceIp
         };
-        Kubelet = @{
+        Kubelet      = @{
             NodeLabels = $global:KubeletNodeLabels;
             ConfigArgs = $global:KubeletConfigArgs
         };

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/Azure/aks-engine/pkg/api/vlabs"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/blang/semver"
 )
 
@@ -210,6 +211,9 @@ func convertWindowsProfileToVLabs(api *WindowsProfile, vlabsProfile *vlabs.Windo
 		vlabsProfile.SSHEnabled = api.SSHEnabled
 	}
 	vlabsProfile.EnableAutomaticUpdates = api.EnableAutomaticUpdates
+	if api.GetEnableAHUB() {
+		vlabsProfile.EnableAHUB = to.BoolPtr(true)
+	}
 }
 
 func convertOrchestratorProfileToVLabs(api *OrchestratorProfile, o *vlabs.OrchestratorProfile) {

diff --git a/pkg/api/converterfromapi_test.go b/pkg/api/converterfromapi_test.go
@@ -773,6 +773,7 @@ func TestConvertWindowsProfileToVlabs(t *testing.T) {
 				WindowsOffer:           "WindowsServer",
 				WindowsSku:             "2019-Datacenter-Core-smalldisk",
 				WindowsDockerVersion:   "18.09",
+				EnableAHUB:             to.BoolPtr(true),
 			},
 			expected: vlabs.WindowsProfile{
 				AdminUsername:          "user",
@@ -785,6 +786,7 @@ func TestConvertWindowsProfileToVlabs(t *testing.T) {
 				WindowsSku:             "2019-Datacenter-Core-smalldisk",
 				WindowsDockerVersion:   "18.09",
 				Secrets:                []vlabs.KeyVaultSecrets{},
+				EnableAHUB:             to.BoolPtr(true),
 			},
 		},
 		{
@@ -812,9 +814,9 @@ func TestConvertWindowsProfileToVlabs(t *testing.T) {
 	}
 
 	for _, c := range cases {
+		c := c
 		t.Run(c.name, func(t *testing.T) {
 			t.Parallel()
-
 			actual := vlabs.WindowsProfile{}
 			convertWindowsProfileToVLabs(&c.w, &actual)
 

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -8,6 +8,7 @@ import (
 	"github.com/Azure/aks-engine/pkg/api/vlabs"
 	"github.com/Azure/aks-engine/pkg/helpers"
 	"github.com/Azure/go-autorest/autorest/azure"
+	"github.com/Azure/go-autorest/autorest/to"
 )
 
 ///////////////////////////////////////////////////////////
@@ -204,6 +205,9 @@ func convertVLabsWindowsProfile(vlabs *vlabs.WindowsProfile, api *WindowsProfile
 		api.SSHEnabled = vlabs.SSHEnabled
 	}
 	api.EnableAutomaticUpdates = vlabs.EnableAutomaticUpdates
+	if vlabs.EnableAHUB != nil && *vlabs.EnableAHUB {
+		api.EnableAHUB = to.BoolPtr(true)
+	}
 }
 
 func convertVLabsOrchestratorProfile(vp *vlabs.Properties, api *OrchestratorProfile, isUpdate bool) error {

diff --git a/pkg/api/convertertoapi_test.go b/pkg/api/convertertoapi_test.go
@@ -879,6 +879,7 @@ func TestConvertVLabsWindowsProfile(t *testing.T) {
 				WindowsOffer:           "WindowsServer",
 				WindowsSku:             "2019-Datacenter-Core-smalldisk",
 				WindowsDockerVersion:   "18.09",
+				EnableAHUB:             to.BoolPtr(true),
 			},
 			expected: WindowsProfile{
 				AdminUsername:          "user",
@@ -891,6 +892,7 @@ func TestConvertVLabsWindowsProfile(t *testing.T) {
 				WindowsSku:             "2019-Datacenter-Core-smalldisk",
 				WindowsDockerVersion:   "18.09",
 				Secrets:                []KeyVaultSecrets{},
+				EnableAHUB:             to.BoolPtr(true),
 			},
 		},
 		{
@@ -918,9 +920,9 @@ func TestConvertVLabsWindowsProfile(t *testing.T) {
 	}
 
 	for _, c := range cases {
+		c := c
 		t.Run(c.name, func(t *testing.T) {
 			t.Parallel()
-
 			actual := WindowsProfile{}
 			convertVLabsWindowsProfile(&c.w, &actual)
 

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -213,6 +213,11 @@ type CustomNodesDNS struct {
 	DNSServer string `json:"dnsServer,omitempty"`
 }
 
+const (
+	// WindowsLicenseTypeServer specifies that the image or disk that is being used was licensed server on-premises.
+	WindowsLicenseTypeServer string = "Windows_Server"
+)
+
 // WindowsProfile represents the windows parameters passed to the cluster
 type WindowsProfile struct {
 	AdminUsername             string            `json:"adminUsername"`
@@ -230,6 +235,7 @@ type WindowsProfile struct {
 	SSHEnabled                *bool             `json:"sshEnabled,omitempty"`
 	EnableAutomaticUpdates    *bool             `json:"enableAutomaticUpdates,omitempty"`
 	IsCredentialAutoGenerated *bool             `json:"isCredentialAutoGenerated,omitempty"`
+	EnableAHUB                *bool             `json:"enableAHUB,omitempty"`
 }
 
 // ProvisioningState represents the current state of container service resource.
@@ -1719,6 +1725,14 @@ func (w *WindowsProfile) GetIsCredentialAutoGenerated() bool {
 	return false
 }
 
+// GetEnableAHUB returns true if AHUB should be enabled for Windows nodes
+func (w *WindowsProfile) GetEnableAHUB() bool {
+	if w.EnableAHUB != nil {
+		return *w.EnableAHUB
+	}
+	return false
+}
+
 // HasSecrets returns true if the customer specified secrets to install
 func (l *LinuxProfile) HasSecrets() bool {
 	return len(l.Secrets) > 0

diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go
@@ -3284,6 +3284,11 @@ func TestWindowsProfile(t *testing.T) {
 		t.Fatalf("Expected GetIsCredentialAutoGenerated() to equal default 'false', got %t", autoGenerated)
 	}
 
+	enableAHUB := w.GetEnableAHUB()
+	if enableAHUB {
+		t.Fatalf("Expected GetEnableAHUB() to equal default 'false', got %t", enableAHUB)
+	}
+
 	w = WindowsProfile{
 		Secrets: []KeyVaultSecrets{
 			{
@@ -3298,6 +3303,7 @@ func TestWindowsProfile(t *testing.T) {
 		},
 		WindowsImageSourceURL:     "testCustomImage",
 		IsCredentialAutoGenerated: to.BoolPtr(true),
+		EnableAHUB:                to.BoolPtr(true),
 	}
 
 	if !(w.HasSecrets() && w.HasCustomImage()) {
@@ -3309,11 +3315,17 @@ func TestWindowsProfile(t *testing.T) {
 		t.Fatalf("Expected GetIsCredentialAutoGenerated() to equal default 'true', got %t", autoGenerated)
 	}
 
+	enableAHUB = w.GetEnableAHUB()
+	if !enableAHUB {
+		t.Fatalf("Expected GetEnableAHUB() to equal default 'true', got %t", enableAHUB)
+	}
+
 	w = WindowsProfile{
 		WindowsDockerVersion:      "18.03.1-ee-3",
 		WindowsSku:                "Datacenter-Core-1809-with-Containers-smalldisk",
 		SSHEnabled:                &trueVar,
 		IsCredentialAutoGenerated: to.BoolPtr(false),
+		EnableAHUB:                to.BoolPtr(false),
 	}
 
 	dv = w.GetWindowsDockerVersion()
@@ -3335,6 +3347,11 @@ func TestWindowsProfile(t *testing.T) {
 	if autoGenerated {
 		t.Fatalf("Expected GetIsCredentialAutoGenerated() to equal default 'false', got %t", autoGenerated)
 	}
+
+	enableAHUB = w.GetEnableAHUB()
+	if enableAHUB {
+		t.Fatalf("Expected GetEnableAHUB() to equal default 'false', got %t", enableAHUB)
+	}
 }
 
 func TestWindowsProfileCustomOS(t *testing.T) {

diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go
@@ -158,22 +158,32 @@ type CustomNodesDNS struct {
 	DNSServer string `json:"dnsServer,omitempty"`
 }
 
+// WindowsLicenseType represents Windows license type
+type WindowsLicenseType string
+
+const (
+	// WindowsLicenseTypeServer specifies that the image or disk that is being used was licensed server on-premises.
+	WindowsLicenseTypeServer WindowsLicenseType = "Windows_Server"
+)
+
 // WindowsProfile represents the windows parameters passed to the cluster
 type WindowsProfile struct {
-	AdminUsername          string            `json:"adminUsername,omitempty"`
-	AdminPassword          string            `json:"adminPassword,omitempty"`
-	CSIProxyURL            string            `json:"csiProxyURL,omitempty"`
-	EnableCSIProxy         *bool             `json:"enableCSIProxy,omitempty"`
-	ImageRef               *ImageReference   `json:"imageReference,omiteempty"`
-	ImageVersion           string            `json:"imageVersion,omitempty"`
-	WindowsImageSourceURL  string            `json:"WindowsImageSourceUrl"`
-	WindowsPublisher       string            `json:"WindowsPublisher"`
-	WindowsOffer           string            `json:"WindowsOffer"`
-	WindowsSku             string            `json:"WindowsSku"`
-	WindowsDockerVersion   string            `json:"windowsDockerVersion"`
-	Secrets                []KeyVaultSecrets `json:"secrets,omitempty"`
-	SSHEnabled             *bool             `json:"sshEnabled,omitempty"`
-	EnableAutomaticUpdates *bool             `json:"enableAutomaticUpdates,omitempty"`
+	AdminUsername             string            `json:"adminUsername,omitempty"`
+	AdminPassword             string            `json:"adminPassword,omitempty"`
+	CSIProxyURL               string            `json:"csiProxyURL,omitempty"`
+	EnableCSIProxy            *bool             `json:"enableCSIProxy,omitempty"`
+	ImageRef                  *ImageReference   `json:"imageReference,omitempty"`
+	ImageVersion              string            `json:"imageVersion,omitempty"`
+	WindowsImageSourceURL     string            `json:"WindowsImageSourceUrl"`
+	WindowsPublisher          string            `json:"WindowsPublisher"`
+	WindowsOffer              string            `json:"WindowsOffer"`
+	WindowsSku                string            `json:"WindowsSku"`
+	WindowsDockerVersion      string            `json:"windowsDockerVersion"`
+	Secrets                   []KeyVaultSecrets `json:"secrets,omitempty"`
+	SSHEnabled                *bool             `json:"sshEnabled,omitempty"`
+	EnableAutomaticUpdates    *bool             `json:"enableAutomaticUpdates,omitempty"`
+	IsCredentialAutoGenerated *bool             `json:"isCredentialAutoGenerated,omitempty"`
+	EnableAHUB                *bool             `json:"enableAHUB,omitempty"`
 }
 
 // ProvisioningState represents the current state of container service resource.

diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go
diff --git a/pkg/engine/virtualmachines.go b/pkg/engine/virtualmachines.go
@@ -483,6 +483,11 @@ func createAgentAvailabilitySetVM(cs *api.ContainerService, profile *api.AgentPo
 		}
 		agentCustomData := getCustomDataFromJSON(t.GetKubernetesWindowsNodeCustomDataJSONObject(cs, profile))
 		osProfile.CustomData = to.StringPtr(agentCustomData)
+
+		if cs.Properties.WindowsProfile.GetEnableAHUB() {
+			licenseType := api.WindowsLicenseTypeServer
+			virtualMachine.LicenseType = &licenseType
+		}
 	}
 
 	virtualMachine.OsProfile = &osProfile

diff --git a/pkg/engine/virtualmachines_test.go b/pkg/engine/virtualmachines_test.go
@@ -299,6 +299,21 @@ func TestCreateAgentAvailabilitySetVM(t *testing.T) {
 	if diff != "" {
 		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
 	}
+
+	cs.Properties.WindowsProfile = &api.WindowsProfile{
+		SSHEnabled: &trueVar,
+		EnableAHUB: &trueVar,
+	}
+
+	actualVM = createAgentAvailabilitySetVM(cs, profile)
+	licenseType := api.WindowsLicenseTypeServer
+	expectedVM.VirtualMachine.VirtualMachineProperties.LicenseType = &licenseType
+
+	diff = cmp.Diff(actualVM, expectedVM)
+
+	if diff != "" {
+		t.Errorf("unexpected diff while expecting equal structs: %s", diff)
+	}
 }
 
 func TestCreateVmWithCustomTags(t *testing.T) {