Fix kubeletConfig for Windows agent nodes (#3753)

examples/windows/README.md

@@ -18,5 +18,6 @@ These cluster definition examples demonstrate how to create customized Docker En
 - kubernetes.json - this is the simplest case for a 2-node Windows Kubernetes cluster
 - kubernetes-custom-image.json - example using an existing Azure Managed Disk for Windows nodes. For example if you need a prerelease OS version, you can build a VHD, upload it and use this sample.
 - kubernetes-hybrid.json - example with both Windows & Linux nodes in the same cluster
+- kubernetes-hyperv.json - example with 2 Windows nodes with the [alpha Hyper-V isolation support](https://kubernetes.io/docs/getting-started-guides/windows/#hyper-v-containers) enabled
 - kubernetes-wincni.json - example using kubenet plugin on Linux nodes and WinCNI on Windows
-- kubernetes-windows-version.json - example of how to build a cluster with a specific Windows patch version
+- kubernetes-windows-version.json - example of how to build a cluster with a specific Windows patch version

examples/windows/kubernetes-hyperv.json

@@ -0,0 +1,61 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.10",
+      "kubernetesConfig": {
+               "apiServerConfig" : {
+                 "--feature-gates": "HyperVContainer=true"
+               },
+               "kubeletConfig" : {
+                    "--feature-gates": "HyperVContainer=true"
+                }
+          }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "hypervtest",
+      "vmSize": "Standard_D2_v3"
+    },
+    "agentPoolProfiles": [
+	    {
+		"name": "windowspool",
+		"count": 2,
+		"vmSize": "Standard_D2_v3",
+		"availabilityProfile": "AvailabilitySet",
+		"osType": "Windows",
+		"osDiskSizeGB": 128,
+		"extensions": [
+		    {
+                        "name": "winrm"
+		    }
+		]
+	    }
+    ],
+    "windowsProfile": {
+      "adminUsername": "azureuser",
+      "adminPassword": "replacepassword1234$"
+    },
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    },
+    "extensionProfiles": [
+      {
+        "name": "winrm",
+        "version": "v1"
+      }
+    ]
+  }
+}

parts/k8s/kuberneteswindowssetup.ps1

@@ -4,6 +4,12 @@
 
     .DESCRIPTION
         Provisions VM as a Kubernetes agent.
+        
+
+        Notes on modifying this file:
+        - This file extension is PS1, but it is actually used as a template from pkg/acsengine/template_generator.go
+        - All of the lines that have braces in them will be modified. Please do not change them here, change them in the Go sources
+        - Single quotes are forbidden, they are reserved to delineate the different members for the ARM template concat() call
 #>
 [CmdletBinding(DefaultParameterSetName="Standard")]
 param(
@@ -45,8 +51,8 @@ $global:WindowsPackageSASURLBase = "{{WrapAsParameter "windowsPackageSASURLBase"
 $global:KubeBinariesVersion = "{{WrapAsParameter "kubeBinariesVersion"}}"
 $global:WindowsTelemetryGUID = "{{WrapAsParameter "windowsTelemetryGUID"}}"
 $global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}"
-$global:KubeletStartFile = $global:KubeDir + "\kubeletstart.ps1"
-$global:KubeProxyStartFile = $global:KubeDir + "\kubeproxystart.ps1"
+$global:KubeletStartFile = [io.path]::Combine($global:KubeDir, "kubeletstart.ps1")
+$global:KubeProxyStartFile = [io.path]::Combine($global:KubeDir, "kubeproxystart.ps1")
 $global:TenantId = "{{WrapAsVariable "tenantID"}}"
 $global:SubscriptionId = "{{WrapAsVariable "subscriptionId"}}"
 $global:ResourceGroup = "{{WrapAsVariable "resourceGroup"}}"
@@ -63,6 +69,8 @@ $global:KubeServiceCIDR = "{{WrapAsParameter "kubeServiceCidr"}}"
 $global:KubeNetwork = "l2bridge"
 $global:KubeDnsSearchPath = "svc.cluster.local"
 
+$global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh .KubernetesConfig }} )
+
 $global:UseManagedIdentityExtension = "{{WrapAsVariable "useManagedIdentityExtension"}}"
 $global:UserAssignedClientID = "{{WrapAsVariable "userAssignedClientID"}}"
 $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}"
@@ -73,7 +81,7 @@ $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni")
 $global:NetworkMode = "L2Bridge"
 $global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf")
 $global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config")
-$global:WindowsCNIKubeletOptions = " --network-plugin=cni --cni-bin-dir=$global:CNIPath --cni-conf-dir=$global:CNIConfigPath"
+$global:WindowsCNIKubeletOptions = @("--cni-bin-dir=$global:CNIPath", "--cni-conf-dir=$global:CNIConfigPath")
 $global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.psm1")
 
 $global:VolumePluginDir = [Io.path]::Combine("$global:KubeDir", "volumeplugins")
@@ -85,7 +93,7 @@ $global:VNetCNIPluginsURL = "{{WrapAsParameter "vnetCniWindowsPluginsURL"}}"
 $global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni")
 $global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin")
 $global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf")
-$global:AzureCNIKubeletOptions = " --network-plugin=cni --cni-bin-dir=$global:AzureCNIBinDir --cni-conf-dir=$global:AzureCNIConfDir"
+$global:AzureCNIKubeletOptions = @("--cni-bin-dir=$global:AzureCNIBinDir", "--cni-conf-dir=$global:AzureCNIConfDir")
 $global:AzureCNIEnabled = $false
 
 filter Timestamp {"$(Get-Date -Format o): $_"}
@@ -168,7 +176,7 @@ Update-WindowsPackages()
 function
 Write-AzureConfig()
 {
-    $azureConfigFile = $global:KubeDir + "\azure.json"
+    $azureConfigFile = [io.path]::Combine($global:KubeDir, "azure.json")
 
     $azureConfig = @"
 {
@@ -196,10 +204,18 @@ Write-AzureConfig()
     $azureConfig | Out-File -encoding ASCII -filepath "$azureConfigFile"
 }
 
+
+function
+Write-CACert()
+{
+    $caFile = [io.path]::Combine($global:KubeDir, "ca.crt")
+    [System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($global:CACertificate)) | Out-File -Encoding ascii $caFile
+}
+
 function
 Write-KubeConfig()
 {
-    $kubeConfigFile = $global:KubeDir + "\config"
+    $kubeConfigFile = [io.path]::Combine($global:KubeDir, "config")
 
     $kubeConfig = @"
 ---
@@ -313,34 +329,34 @@ Set-NetworkConfig
 function
 Write-KubernetesStartFiles($podCIDR)
 {
-    mkdir $global:VolumePluginDir
-    $KubeletArgList = @(" --node-labels=`$global:KubeletNodeLabels --hostname-override=`$global:AzureHostname","--pod-infra-container-image=kubletwin/pause","--resolv-conf=""""""""","--kubeconfig=c:\k\config","--cloud-provider=azure","--cloud-config=c:\k\azure.json")
-    $KubeletCommandLine = @"
-c:\k\kubelet.exe --hostname-override=`$env:computername --pod-infra-container-image=kubletwin/pause --resolv-conf="" --allow-privileged=true --enable-debugging-handlers --cluster-dns=`$global:KubeDnsServiceIp --cluster-domain=cluster.local  --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge --v=2 --azure-container-registry-config=c:\k\azure.json --runtime-request-timeout=10m  --cloud-provider=azure --cloud-config=c:\k\azure.json
-"@
+    mkdir $global:VolumePluginDir 
+    $KubeletArgList = $global:KubeletConfigArgs # This is the initial list passed in from acs-engine
+    $KubeletArgList += "--node-labels=`$global:KubeletNodeLabels"
+    $KubeletArgList += "--hostname-override=`$global:AzureHostname"
+    $KubeletArgList += "--volume-plugin-dir=`$global:VolumePluginDir"
+    # If you are thinking about adding another arg here, you should be considering pkg/acsengine/defaults-kubelet.go first
+    # Only args that need to be calculated or combined with other ones on the Windows agent should be added here.
+
+
     # Regex to strip version to Major.Minor.Build format such that the following check does not crash for version like x.y.z-alpha
     [regex]$regex = "^[0-9.]+"
     $KubeBinariesVersionStripped = $regex.Matches($global:KubeBinariesVersion).Value
     if ([System.Version]$KubeBinariesVersionStripped -lt [System.Version]"1.8.0")
     {
         # --api-server deprecates from 1.8.0
         $KubeletArgList += "--api-servers=https://`${global:MasterIP}:443"
-        $KubeletCommandLine += " --api-servers=https://`${global:MasterIP}:443"
     }
 
-    # more time is needed to pull windows server images
-    $KubeletCommandLine += " --image-pull-progress-deadline=20m --cgroups-per-qos=false --enforce-node-allocatable=`"`""
-    $KubeletCommandLine += " --volume-plugin-dir=`$global:VolumePluginDir"
-     # Configure kubelet to use CNI plugins if enabled.
+    # Configure kubelet to use CNI plugins if enabled.
     if ($global:AzureCNIEnabled) {
-        $KubeletCommandLine += $global:AzureCNIKubeletOptions
+        $KubeletArgList += $global:AzureCNIKubeletOptions
     } else {
-        $KubeletCommandLine += $global:WindowsCNIKubeletOptions
+        $KubeletArgList += $global:WindowsCNIKubeletOptions
     }
 
-    $KubeletArgListStr = "`"" + ($KubeletArgList -join "`",`"") + "`""
+    $KubeletArgListStr = $KubeletArgList -join " "
 
-    $KubeletArgListStr = "@`($KubeletArgListStr`)"
+    $KubeletCommandLine = "c:\k\kubelet.exe " + $KubeletArgListStr
 
     $kubeStartStr = @"
 `$global:MasterIP = "$MasterIP"
@@ -357,6 +373,7 @@ c:\k\kubelet.exe --hostname-override=`$env:computername --pod-infra-container-im
 `$global:HNSModule = "$global:HNSModule"
 `$global:VolumePluginDir = "$global:VolumePluginDir"
 `$global:NetworkPlugin="$global:NetworkPlugin"
+`$global:KubeletNodeLabels="$global:KubeletNodeLabels"
 
 "@
 
@@ -394,24 +411,24 @@ if (`$hnsNetwork)
     Remove-HnsNetwork `$hnsNetwork
     # Kill all cni instances & stale data left by cni
     # Cleanup all files related to cni
-    `$cnijson = "$global:KubeDir" + "\azure-vnet-ipam.json"
+    `$cnijson = [io.path]::Combine("$global:KubeDir", "azure-vnet-ipam.json")
     if ((Test-Path `$cnijson))
     {
         Remove-Item `$cnijson
     }
-    `$cnilock = "$global:KubeDir" + "\azure-vnet-ipam.lock"
+    `$cnilock = [io.path]::Combine("$global:KubeDir", "azure-vnet-ipam.lock")
     if ((Test-Path `$cnilock))
     {
         Remove-Item `$cnilock
     }
     taskkill /IM azure-vnet-ipam.exe /f
 
-    `$cnijson = "$global:KubeDir" + "\azure-vnet.json"
+    `$cnijson = [io.path]::Combine("$global:KubeDir", "azure-vnet.json")
     if ((Test-Path `$cnijson))
     {
         Remove-Item `$cnijson
     }
-    `$cnilock = "$global:KubeDir" + "\azure-vnet.lock"
+    `$cnilock = [io.path]::Combine("$global:KubeDir", "azure-vnet.lock")
     if ((Test-Path `$cnilock))
     {
         Remove-Item `$cnilock
@@ -425,7 +442,9 @@ Restart-Service Kubeproxy
 $KubeletCommandLine
 
 "@
-    } else {
+    } 
+    else  # using WinCNI. TODO: If WinCNI support is removed, then delete this as dead code later
+    {
         $kubeStartStr += @"
 
 function
@@ -563,7 +582,7 @@ catch
 }
 
 "@
-    }
+    } # end else using WinCNI.
 
     $kubeStartStr | Out-File -encoding ASCII -filepath $global:KubeletStartFile
 
@@ -668,6 +687,9 @@ try
         Write-Log "Write azure config"
         Write-AzureConfig
 
+        Write-Log "Write ca root"
+        Write-CACert
+
         Write-Log "Write kube config"
         Write-KubeConfig
 

pkg/acsengine/defaults-kubelet.go

@@ -26,16 +26,24 @@ func setKubeletConfig(cs *api.ContainerService) {
 		"--keep-terminated-pod-volumes": "false",
 	}
 
+	// Start with copy of Linux config
 	staticWindowsKubeletConfig := make(map[string]string)
 	for key, val := range staticLinuxKubeletConfig {
 		staticWindowsKubeletConfig[key] = val
 	}
+
+	// Add Windows-specific overrides
+	// Eventually paths should not be hardcoded here. They should be relative to $global:KubeDir in the PowerShell script
 	staticWindowsKubeletConfig["--azure-container-registry-config"] = "c:\\k\\azure.json"
 	staticWindowsKubeletConfig["--pod-infra-container-image"] = "kubletwin/pause"
 	staticWindowsKubeletConfig["--kubeconfig"] = "c:\\k\\config"
 	staticWindowsKubeletConfig["--cloud-config"] = "c:\\k\\azure.json"
 	staticWindowsKubeletConfig["--cgroups-per-qos"] = "false"
-	staticWindowsKubeletConfig["--enforce-node-allocatable"] = "\"\""
+	staticWindowsKubeletConfig["--enforce-node-allocatable"] = "\"\"\"\""
+	staticWindowsKubeletConfig["--client-ca-file"] = "c:\\k\\ca.crt"
+	staticWindowsKubeletConfig["--hairpin-mode"] = "promiscuous-bridge"
+	staticWindowsKubeletConfig["--image-pull-progress-deadline"] = "20m"
+	staticWindowsKubeletConfig["--resolv-conf"] = "\"\"\"\""
 
 	// Default Kubelet config
 	defaultKubeletConfig := map[string]string{
@@ -136,6 +144,11 @@ func setKubeletConfig(cs *api.ContainerService) {
 		}
 		setMissingKubeletValues(profile.KubernetesConfig, o.KubernetesConfig.KubeletConfig)
 
+		if profile.OSType == "Windows" {
+			// Remove Linux-specific values
+			delete(profile.KubernetesConfig.KubeletConfig, "--pod-manifest-path")
+		}
+
 		// For N Series (GPU) VMs
 		if strings.Contains(profile.VMSize, "Standard_N") {
 			if !cs.Properties.IsNVIDIADevicePluginEnabled() && !common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.11.0") {

pkg/acsengine/template_generator.go

@@ -216,6 +216,26 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			}
 			return buf.String()
 		},
+		"GetKubeletConfigKeyValsPsh": func(kc *api.KubernetesConfig) string {
+			if kc == nil {
+				return ""
+			}
+			kubeletConfig := cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig
+			if kc.KubeletConfig != nil {
+				kubeletConfig = kc.KubeletConfig
+			}
+			// Order by key for consistency
+			keys := []string{}
+			for key := range kubeletConfig {
+				keys = append(keys, key)
+			}
+			sort.Strings(keys)
+			var buf bytes.Buffer
+			for _, key := range keys {
+				buf.WriteString(fmt.Sprintf("\"%s=%s\", ", key, kubeletConfig[key]))
+			}
+			return strings.TrimSuffix(buf.String(), ", ")
+		},
 		"GetK8sRuntimeConfigKeyVals": func(config map[string]string) string {
 			// Order by key for consistency
 			keys := []string{}