diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index 7bcc399ad2..7cc2e9c42d 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -580,10 +580,11 @@ https://{keyvaultname}.vault.azure.net:443/secrets/{secretName}/{version} #### Choosing a Windows version -If you want to choose a specific Windows image, but automatically use the latest - set `windowsPublisher`, `windowsOffer`, and `windowsSku`. If you need a specific version, then add `imageVersion` too. +If you want to choose a specific Windows image, but automatically use the latest - set `windowsPublisher`, `windowsOffer`, and `windowsSku`. If you need a specific version, then add `agentWindowsVersion` too. You can find all available images with `az vm image list` + ```bash $ az vm image list --publisher MicrosoftWindowsServer --all -o table diff --git a/parts/k8s/kuberneteswindowsfunctions.ps1 b/parts/k8s/kuberneteswindowsfunctions.ps1 new file mode 100644 index 0000000000..bdc3a48909 --- /dev/null +++ b/parts/k8s/kuberneteswindowsfunctions.ps1 @@ -0,0 +1,34 @@ +# This is a temporary file to test dot-sourcing functions stored in separate scripts in a zip file + +filter Timestamp {"$(Get-Date -Format o): $_"} + +function +Write-Log($message) +{ + $msg = $message | Timestamp + Write-Output $msg +} + +function DownloadFileOverHttp +{ + Param( + [Parameter(Mandatory=$true)][string] + $Url, + [Parameter(Mandatory=$true)][string] + $DestinationPath + ) + $secureProtocols = @() + $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) + + foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) + { + if ($insecureProtocols -notcontains $protocol) + { + $secureProtocols += $protocol + } + } + [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols + + Invoke-WebRequest $Url -UseBasicParsing -OutFile $DestinationPath -Verbose + Write-Log "Downloaded file to $DestinationPath" +} \ No newline at end of file diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index e01212cf76..99f656c432 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -5,6 +5,7 @@ .DESCRIPTION Provisions VM as a Kubernetes agent. + The parameters passed in are required, and will vary per-deployment. Notes on modifying this file: - This file extension is PS1, but it is actually used as a template from pkg/acsengine/template_generator.go @@ -42,17 +43,22 @@ param( $AADClientSecret ) + + +# These globals will not change between nodes in the same cluster, so they are not +# passed as powershell parameters + +## Certificates generated by acs-engine $global:CACertificate = "{{WrapAsParameter "caCertificate"}}" $global:AgentCertificate = "{{WrapAsParameter "clientCertificate"}}" -$global:DockerServiceName = "Docker" -$global:KubeDir = "c:\k" + +## Download sources provided by acs-engine $global:KubeBinariesSASURL = "{{WrapAsParameter "kubeBinariesSASURL"}}" $global:WindowsPackageSASURLBase = "{{WrapAsParameter "windowsPackageSASURLBase"}}" $global:KubeBinariesVersion = "{{WrapAsParameter "kubeBinariesVersion"}}" + +## VM configuration passed by Azure $global:WindowsTelemetryGUID = "{{WrapAsParameter "windowsTelemetryGUID"}}" -$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}" -$global:KubeletStartFile = [io.path]::Combine($global:KubeDir, "kubeletstart.ps1") -$global:KubeProxyStartFile = [io.path]::Combine($global:KubeDir, "kubeproxystart.ps1") $global:TenantId = "{{WrapAsVariable "tenantID"}}" $global:SubscriptionId = "{{WrapAsVariable "subscriptionId"}}" $global:ResourceGroup = "{{WrapAsVariable "resourceGroup"}}" @@ -64,615 +70,54 @@ $global:VNetName = "{{WrapAsVariable "virtualNetworkName"}}" $global:RouteTableName = "{{WrapAsVariable "routeTableName"}}" $global:PrimaryAvailabilitySetName = "{{WrapAsVariable "primaryAvailabilitySetName"}}" $global:PrimaryScaleSetName = "{{WrapAsVariable "primaryScaleSetName"}}" + $global:KubeClusterCIDR = "{{WrapAsParameter "kubeClusterCidr"}}" $global:KubeServiceCIDR = "{{WrapAsParameter "kubeServiceCidr"}}" -$global:KubeNetwork = "l2bridge" -$global:KubeDnsSearchPath = "svc.cluster.local" - +$global:KubeletNodeLabels = "{{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}" $global:KubeletConfigArgs = @( {{GetKubeletConfigKeyValsPsh .KubernetesConfig }} ) $global:UseManagedIdentityExtension = "{{WrapAsVariable "useManagedIdentityExtension"}}" $global:UserAssignedClientID = "{{WrapAsVariable "userAssignedClientID"}}" $global:UseInstanceMetadata = "{{WrapAsVariable "useInstanceMetadata"}}" + $global:LoadBalancerSku = "{{WrapAsVariable "loadBalancerSku"}}" $global:ExcludeMasterFromStandardLB = "{{WrapAsVariable "excludeMasterFromStandardLB"}}" + +# Windows defaults, not changed by acs-engine +$global:KubeDir = "c:\k" +$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.psm1") + +$global:KubeDnsSearchPath = "svc.cluster.local" + $global:CNIPath = [Io.path]::Combine("$global:KubeDir", "cni") $global:NetworkMode = "L2Bridge" $global:CNIConfig = [Io.path]::Combine($global:CNIPath, "config", "`$global:NetworkMode.conf") $global:CNIConfigPath = [Io.path]::Combine("$global:CNIPath", "config") -$global:WindowsCNIKubeletOptions = @("--cni-bin-dir=$global:CNIPath", "--cni-conf-dir=$global:CNIConfigPath") -$global:HNSModule = [Io.path]::Combine("$global:KubeDir", "hns.psm1") -$global:VolumePluginDir = [Io.path]::Combine("$global:KubeDir", "volumeplugins") -#azure cni -$global:NetworkPolicy = "{{WrapAsParameter "networkPolicy"}}" -$global:NetworkPlugin = "{{WrapAsParameter "networkPlugin"}}" -$global:VNetCNIPluginsURL = "{{WrapAsParameter "vnetCniWindowsPluginsURL"}}" $global:AzureCNIDir = [Io.path]::Combine("$global:KubeDir", "azurecni") $global:AzureCNIBinDir = [Io.path]::Combine("$global:AzureCNIDir", "bin") $global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") -$global:AzureCNIKubeletOptions = @("--cni-bin-dir=$global:AzureCNIBinDir", "--cni-conf-dir=$global:AzureCNIConfDir") -$global:AzureCNIEnabled = $false - -filter Timestamp {"$(Get-Date -Format o): $_"} - -function -Write-Log($message) -{ - $msg = $message | Timestamp - Write-Output $msg -} - -function Set-TelemetrySetting() -{ - Set-ItemProperty -Path "HKLM:\Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection" -Name "CommercialId" -Value $global:WindowsTelemetryGUID -Force -} - -function Resize-OSDrive() -{ - $osDrive = ((Get-WmiObject Win32_OperatingSystem).SystemDrive).TrimEnd(":") - $size = (Get-Partition -DriveLetter $osDrive).Size - $maxSize = (Get-PartitionSupportedSize -DriveLetter $osDrive).SizeMax - if ($size -lt $maxSize) - { - Resize-Partition -DriveLetter $osDrive -Size $maxSize - } -} - -function -Get-KubeBinaries() -{ - $zipfile = "c:\k.zip" - for ($i=0; $i -le 10; $i++) - { - Start-BitsTransfer -Source $global:KubeBinariesSASURL -Destination $zipfile - if ($?) { - break - } else { - Write-Log $Error[0].Exception.Message - } - } - Expand-Archive -path $zipfile -DestinationPath C:\ -} - -function DownloadFileOverHttp($Url, $DestinationPath) -{ - $secureProtocols = @() - $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) - - foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) - { - if ($insecureProtocols -notcontains $protocol) - { - $secureProtocols += $protocol - } - } - [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols - - curl $Url -UseBasicParsing -OutFile $DestinationPath -Verbose - Write-Log "$DestinationPath updated" -} -function Get-HnsPsm1() -{ - DownloadFileOverHttp "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1" "$global:HNSModule" -} - -function Update-WinCNI() -{ - $wincni = "wincni.exe" - $wincniFile = [Io.path]::Combine($global:CNIPath, $wincni) - DownloadFileOverHttp "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/cni/wincni.exe" $wincniFile -} - -function -Update-WindowsPackages() -{ - Update-WinCNI - Get-HnsPsm1 -} - -function -Write-AzureConfig() -{ - $azureConfigFile = [io.path]::Combine($global:KubeDir, "azure.json") - - $azureConfig = @" -{ - "tenantId": "$global:TenantId", - "subscriptionId": "$global:SubscriptionId", - "aadClientId": "$AADClientId", - "aadClientSecret": "$AADClientSecret", - "resourceGroup": "$global:ResourceGroup", - "location": "$Location", - "vmType": "$global:VmType", - "subnetName": "$global:SubnetName", - "securityGroupName": "$global:SecurityGroupName", - "vnetName": "$global:VNetName", - "routeTableName": "$global:RouteTableName", - "primaryAvailabilitySetName": "$global:PrimaryAvailabilitySetName", - "primaryScaleSetName": "$global:PrimaryScaleSetName", - "useManagedIdentityExtension": $global:UseManagedIdentityExtension, - "userAssignedIdentityID": $global:UserAssignedClientID, - "useInstanceMetadata": $global:UseInstanceMetadata, - "loadBalancerSku": "$global:LoadBalancerSku", - "excludeMasterFromStandardLB": $global:ExcludeMasterFromStandardLB -} -"@ - - $azureConfig | Out-File -encoding ASCII -filepath "$azureConfigFile" -} +# Azure cni configuration +# $global:NetworkPolicy = "{{WrapAsParameter "networkPolicy"}}" # BUG: unused +$global:NetworkPlugin = "{{WrapAsParameter "networkPlugin"}}" +$global:VNetCNIPluginsURL = "{{WrapAsParameter "vnetCniWindowsPluginsURL"}}" -function -Write-CACert() -{ - $caFile = [io.path]::Combine($global:KubeDir, "ca.crt") - [System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($global:CACertificate)) | Out-File -Encoding ascii $caFile -} - -function -Write-KubeConfig() -{ - $kubeConfigFile = [io.path]::Combine($global:KubeDir, "config") - - $kubeConfig = @" ---- -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: "$global:CACertificate" - server: https://${MasterIP}:443 - name: "$MasterFQDNPrefix" -contexts: -- context: - cluster: "$MasterFQDNPrefix" - user: "$MasterFQDNPrefix-admin" - name: "$MasterFQDNPrefix" -current-context: "$MasterFQDNPrefix" -kind: Config -users: -- name: "$MasterFQDNPrefix-admin" - user: - client-certificate-data: "$global:AgentCertificate" - client-key-data: "$AgentKey" -"@ - - $kubeConfig | Out-File -encoding ASCII -filepath "$kubeConfigFile" -} - -function -New-InfraContainer() -{ - cd $global:KubeDir - $computerInfo = Get-ComputerInfo - $windowsBase = if ($computerInfo.WindowsVersion -eq "1709") { - "microsoft/nanoserver:1709" - } elseif ( ($computerInfo.WindowsVersion -eq "1803") -and ($computerInfo.WindowsBuildLabEx.StartsWith("17134")) ) { - "microsoft/nanoserver:1803" - } else { - # This is a temporary workaround. As of May 2018, Windows Server Insider builds still report 1803 which is wrong. - # Once that is fixed, add another elseif ( -eq "nnnn") instead and remove the StartsWith("17134") above - "microsoft/nanoserver-insider" - } - - "FROM $($windowsBase)" | Out-File -encoding ascii -FilePath Dockerfile - "CMD cmd /c ping -t localhost" | Out-File -encoding ascii -FilePath Dockerfile -Append - docker build -t kubletwin/pause . -} - -function -Set-VnetPluginMode($mode) -{ - # Sets Azure VNET CNI plugin operational mode. - $fileName = [Io.path]::Combine("$global:AzureCNIConfDir", "10-azure.conflist") - (Get-Content $fileName) | %{$_ -replace "`"mode`":.*", "`"mode`": `"$mode`","} | Out-File -encoding ASCII -filepath $fileName -} - -function -Install-VnetPlugins() -{ - # Create CNI directories. - mkdir $global:AzureCNIBinDir - mkdir $global:AzureCNIConfDir - - # Download Azure VNET CNI plugins. - # Mirror from https://github.com/Azure/azure-container-networking/releases - $zipfile = [Io.path]::Combine("$global:AzureCNIDir", "azure-vnet.zip") - Invoke-WebRequest -Uri $global:VNetCNIPluginsURL -OutFile $zipfile - Expand-Archive -path $zipfile -DestinationPath $global:AzureCNIBinDir - del $zipfile - - # Windows does not need a separate CNI loopback plugin because the Windows - # kernel automatically creates a loopback interface for each network namespace. - # Copy CNI network config file and set bridge mode. - move $global:AzureCNIBinDir/*.conflist $global:AzureCNIConfDir - - # Enable CNI in kubelet. - $global:AzureCNIEnabled = $true -} - -function -Set-AzureNetworkPlugin() -{ - # Azure VNET network policy requires tunnel (hairpin) mode because policy is enforced in the host. - Set-VnetPluginMode "tunnel" -} -function -Set-AzureCNIConfig() -{ - # Fill in DNS information for kubernetes. - $fileName = [Io.path]::Combine("$global:AzureCNIConfDir", "10-azure.conflist") - $configJson = Get-Content $fileName | ConvertFrom-Json - $configJson.plugins.dns.Nameservers[0] = $KubeDnsServiceIp - $configJson.plugins.dns.Search[0] = $global:KubeDnsSearchPath - $configJson.plugins.AdditionalArgs[0].Value.ExceptionList[0] = $global:KubeClusterCIDR - $configJson.plugins.AdditionalArgs[0].Value.ExceptionList[1] = $global:MasterSubnet - $configJson.plugins.AdditionalArgs[1].Value.DestinationPrefix = $global:KubeServiceCIDR - - $configJson | ConvertTo-Json -depth 20 | Out-File -encoding ASCII -filepath $fileName -} - -function -Set-NetworkConfig -{ - Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" - - # Configure network policy. - if ($global:NetworkPlugin -eq "azure") { - Install-VnetPlugins - Set-AzureCNIConfig - } -} - -function -Write-KubernetesStartFiles($podCIDR) -{ - mkdir $global:VolumePluginDir - $KubeletArgList = $global:KubeletConfigArgs # This is the initial list passed in from acs-engine - $KubeletArgList += "--node-labels=`$global:KubeletNodeLabels" - $KubeletArgList += "--hostname-override=`$global:AzureHostname" - $KubeletArgList += "--volume-plugin-dir=`$global:VolumePluginDir" - # If you are thinking about adding another arg here, you should be considering pkg/acsengine/defaults-kubelet.go first - # Only args that need to be calculated or combined with other ones on the Windows agent should be added here. - - - # Regex to strip version to Major.Minor.Build format such that the following check does not crash for version like x.y.z-alpha - [regex]$regex = "^[0-9.]+" - $KubeBinariesVersionStripped = $regex.Matches($global:KubeBinariesVersion).Value - if ([System.Version]$KubeBinariesVersionStripped -lt [System.Version]"1.8.0") - { - # --api-server deprecates from 1.8.0 - $KubeletArgList += "--api-servers=https://`${global:MasterIP}:443" - } - - # Configure kubelet to use CNI plugins if enabled. - if ($global:AzureCNIEnabled) { - $KubeletArgList += $global:AzureCNIKubeletOptions - } else { - $KubeletArgList += $global:WindowsCNIKubeletOptions - $KubeletArgList = $KubeletArgList -replace "kubenet", "cni" - } - - # Used in WinCNI version of kubeletstart.ps1 - $KubeletArgListStr = "" - $KubeletArgList | Foreach-Object { - # Since generating new code to be written to a file, need to escape quotes again - if ($KubeletArgListStr.length -gt 0) - { - $KubeletArgListStr = $KubeletArgListStr + ", " - } - $KubeletArgListStr = $KubeletArgListStr + "`"" + $_.Replace("`"`"","`"`"`"`"") + "`"" - } - $KubeletArgListStr = "@`($KubeletArgListStr`)" - - # Used in Azure-CNI version of kubeletstart.ps1 - $KubeletCommandLine = "c:\k\kubelet.exe " + ($KubeletArgList -join " ") - - $kubeStartStr = @" -`$global:MasterIP = "$MasterIP" -`$global:KubeDnsSearchPath = "svc.cluster.local" -`$global:KubeDnsServiceIp = "$KubeDnsServiceIp" -`$global:MasterSubnet = "$global:MasterSubnet" -`$global:KubeClusterCIDR = "$global:KubeClusterCIDR" -`$global:KubeServiceCIDR = "$global:KubeServiceCIDR" -`$global:KubeBinariesVersion = "$global:KubeBinariesVersion" -`$global:CNIPath = "$global:CNIPath" -`$global:NetworkMode = "$global:NetworkMode" -`$global:ExternalNetwork = "ext" -`$global:CNIConfig = "$global:CNIConfig" -`$global:HNSModule = "$global:HNSModule" -`$global:VolumePluginDir = "$global:VolumePluginDir" -`$global:NetworkPlugin="$global:NetworkPlugin" -`$global:KubeletNodeLabels="$global:KubeletNodeLabels" - -"@ - - if ($global:NetworkPlugin -eq "azure") { - $global:KubeNetwork = "azure" - $kubeStartStr += @" -Write-Host "NetworkPlugin azure, starting kubelet." - -# Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) -netsh advfirewall set allprofiles state off -# startup the service - -# Find if the primary external switch network exists. If not create one. -# This is done only once in the lifetime of the node -`$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:ExternalNetwork -if (!`$hnsNetwork) -{ - Write-Host "Creating a new hns Network" - ipmo `$global:HNSModule - # Fixme : use a smallest range possible, that will not collide with any pod space - New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name `$global:ExternalNetwork -Verbose -} - -# Find if network created by CNI exists, if yes, remove it -# This is required to keep the network non-persistent behavior -# Going forward, this would be done by HNS automatically during restart of the node - -`$hnsNetwork = Get-HnsNetwork | ? Name -EQ $global:KubeNetwork -if (`$hnsNetwork) -{ - # Cleanup all containers - docker ps -q | foreach {docker rm `$_ -f} - - Write-Host "Cleaning up old HNS network found" - Remove-HnsNetwork `$hnsNetwork - # Kill all cni instances & stale data left by cni - # Cleanup all files related to cni - `$cnijson = [io.path]::Combine("$global:KubeDir", "azure-vnet-ipam.json") - if ((Test-Path `$cnijson)) - { - Remove-Item `$cnijson - } - `$cnilock = [io.path]::Combine("$global:KubeDir", "azure-vnet-ipam.lock") - if ((Test-Path `$cnilock)) - { - Remove-Item `$cnilock - } - taskkill /IM azure-vnet-ipam.exe /f - - `$cnijson = [io.path]::Combine("$global:KubeDir", "azure-vnet.json") - if ((Test-Path `$cnijson)) - { - Remove-Item `$cnijson - } - `$cnilock = [io.path]::Combine("$global:KubeDir", "azure-vnet.lock") - if ((Test-Path `$cnilock)) - { - Remove-Item `$cnilock - } - taskkill /IM azure-vnet.exe /f -} - -# Restart Kubeproxy, which would wait, until the network is created -Restart-Service Kubeproxy - -$KubeletCommandLine - -"@ - } - else # using WinCNI. TODO: If WinCNI support is removed, then delete this as dead code later - { - $kubeStartStr += @" - -function -Get-DefaultGateway(`$CIDR) -{ - return `$CIDR.substring(0,`$CIDR.lastIndexOf(".")) + ".1" -} - -function -Get-PodCIDR() -{ - `$podCIDR = c:\k\kubectl.exe --kubeconfig=c:\k\config get nodes/`$(`$env:computername.ToLower()) -o custom-columns=podCidr:.spec.podCIDR --no-headers - return `$podCIDR -} - -function -Test-PodCIDR(`$podCIDR) -{ - return `$podCIDR.length -gt 0 -} - -function -Update-CNIConfig(`$podCIDR, `$masterSubnetGW) -{ - `$jsonSampleConfig = -"{ - ""cniVersion"": ""0.2.0"", - ""name"": """", - ""type"": ""wincni.exe"", - ""master"": ""Ethernet"", - ""capabilities"": { ""portMappings"": true }, - ""ipam"": { - ""environment"": ""azure"", - ""subnet"":"""", - ""routes"": [{ - ""GW"":"""" - }] - }, - ""dns"" : { - ""Nameservers"" : [ """" ], - ""Search"" : [ """" ] - }, - ""AdditionalArgs"" : [ - { - ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""OutBoundNAT"", ""ExceptionList"": [ """", """" ] } - }, - { - ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""ROUTE"", ""DestinationPrefix"": """", ""NeedEncap"" : true } - } - ] -}" - - `$configJson = ConvertFrom-Json `$jsonSampleConfig - `$configJson.name = `$global:NetworkMode.ToLower() - `$configJson.ipam.subnet=`$podCIDR - `$configJson.ipam.routes[0].GW = `$masterSubnetGW - `$configJson.dns.Nameservers[0] = `$global:KubeDnsServiceIp - `$configJson.dns.Search[0] = `$global:KubeDnsSearchPath - - `$configJson.AdditionalArgs[0].Value.ExceptionList[0] = `$global:KubeClusterCIDR - `$configJson.AdditionalArgs[0].Value.ExceptionList[1] = `$global:MasterSubnet - `$configJson.AdditionalArgs[1].Value.DestinationPrefix = `$global:KubeServiceCIDR - - if (Test-Path `$global:CNIConfig) - { - Clear-Content -Path `$global:CNIConfig - } - - Write-Host "Generated CNI Config [`$configJson]" - - Add-Content -Path `$global:CNIConfig -Value (ConvertTo-Json `$configJson -Depth 20) -} - -try -{ - `$masterSubnetGW = Get-DefaultGateway `$global:MasterSubnet - `$podCIDR=Get-PodCIDR - `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) - - # if the podCIDR has not yet been assigned to this node, start the kubelet process to get the podCIDR, and then promptly kill it. - if (-not `$podCidrDiscovered) - { - `$argList = $KubeletArgListStr - - `$process = Start-Process -FilePath c:\k\kubelet.exe -PassThru -ArgumentList `$argList - - # run kubelet until podCidr is discovered - Write-Host "waiting to discover pod CIDR" - while (-not `$podCidrDiscovered) - { - Write-Host "Sleeping for 10s, and then waiting to discover pod CIDR" - Start-Sleep 10 - - `$podCIDR=Get-PodCIDR - `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) - } - - # stop the kubelet process now that we have our CIDR, discard the process output - `$process | Stop-Process | Out-Null - } - - # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) - netsh advfirewall set allprofiles state off - - # startup the service - `$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:NetworkMode.ToLower() - - if (`$hnsNetwork) - { - # Kubelet has been restarted with existing network. - # Cleanup all containers - docker ps -q | foreach {docker rm `$_ -f} - # cleanup network - Write-Host "Cleaning up old HNS network found" - Remove-HnsNetwork `$hnsNetwork - Start-Sleep 10 - } - - Write-Host "Creating a new hns Network" - ipmo `$global:HNSModule - - `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$masterSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose - # New network has been created, Kubeproxy service has to be restarted - Restart-Service Kubeproxy - - Start-Sleep 10 - # Add route to all other POD networks - Update-CNIConfig `$podCIDR `$masterSubnetGW - - $KubeletCommandLine -} -catch -{ - Write-Error `$_ -} - -"@ - } # end else using WinCNI. - - $kubeStartStr | Out-File -encoding ASCII -filepath $global:KubeletStartFile - - $kubeProxyStartStr = @" -`$env:KUBE_NETWORK = "$global:KubeNetwork" -`$global:NetworkMode = "$global:NetworkMode" -`$global:HNSModule = "$global:HNSModule" -`$hnsNetwork = Get-HnsNetwork | ? Name -EQ $global:KubeNetwork -while (!`$hnsNetwork) -{ - Write-Host "Waiting for Network [$global:KubeNetwork] to be created . . ." - Start-Sleep 10 - `$hnsNetwork = Get-HnsNetwork | ? Name -EQ $global:KubeNetwork -} - -# -# cleanup the persisted policy lists -# -ipmo `$global:HNSModule -Get-HnsPolicyList | Remove-HnsPolicyList - -$global:KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$global:KubeDir\config -"@ - - $kubeProxyStartStr | Out-File -encoding ASCII -filepath $global:KubeProxyStartFile -} +# Base64 representation of ZIP archive +$zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" -function -New-NSSMService -{ - # setup kubelet - c:\k\nssm install Kubelet C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe - c:\k\nssm set Kubelet AppDirectory $global:KubeDir - c:\k\nssm set Kubelet AppParameters $global:KubeletStartFile - c:\k\nssm set Kubelet DisplayName Kubelet - c:\k\nssm set Kubelet Description Kubelet - c:\k\nssm set Kubelet Start SERVICE_AUTO_START - c:\k\nssm set Kubelet ObjectName LocalSystem - c:\k\nssm set Kubelet Type SERVICE_WIN32_OWN_PROCESS - c:\k\nssm set Kubelet AppThrottle 1500 - c:\k\nssm set Kubelet AppStdout C:\k\kubelet.log - c:\k\nssm set Kubelet AppStderr C:\k\kubelet.err.log - c:\k\nssm set Kubelet AppStdoutCreationDisposition 4 - c:\k\nssm set Kubelet AppStderrCreationDisposition 4 - c:\k\nssm set Kubelet AppRotateFiles 1 - c:\k\nssm set Kubelet AppRotateOnline 1 - c:\k\nssm set Kubelet AppRotateSeconds 86400 - c:\k\nssm set Kubelet AppRotateBytes 1048576 - - # setup kubeproxy - c:\k\nssm install Kubeproxy C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe - c:\k\nssm set Kubeproxy AppDirectory $global:KubeDir - c:\k\nssm set Kubeproxy AppParameters $global:KubeProxyStartFile - c:\k\nssm set Kubeproxy DisplayName Kubeproxy - c:\k\nssm set Kubeproxy DependOnService Kubelet - c:\k\nssm set Kubeproxy Description Kubeproxy - c:\k\nssm set Kubeproxy Start SERVICE_AUTO_START - c:\k\nssm set Kubeproxy ObjectName LocalSystem - c:\k\nssm set Kubeproxy Type SERVICE_WIN32_OWN_PROCESS - c:\k\nssm set Kubeproxy AppThrottle 1500 - c:\k\nssm set Kubeproxy AppStdout C:\k\kubeproxy.log - c:\k\nssm set Kubeproxy AppStderr C:\k\kubeproxy.err.log - c:\k\nssm set Kubeproxy AppRotateFiles 1 - c:\k\nssm set Kubeproxy AppRotateOnline 1 - c:\k\nssm set Kubeproxy AppRotateSeconds 86400 - c:\k\nssm set Kubeproxy AppRotateBytes 1048576 -} +# Extract ZIP from script +[io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles)) +Expand-Archive scripts.zip -DestinationPath "C:\\AzureData\\" -function -Set-Explorer -{ - # setup explorer so that it is usable - New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer" - New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\BrowserEmulation" - New-ItemProperty -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\BrowserEmulation" -Name IntranetCompatibilityMode -Value 0 -Type DWord - New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\Main" - New-ItemProperty -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\Main" -Name "Start Page" -Type String -Value http://bing.com -} +# Dot-source contents of zip. This should match the list in template_generator.go GetKubernetesWindowsAgentFunctions +. c:\AzureData\k8s\kuberneteswindowsfunctions.ps1 +. c:\AzureData\k8s\windowsconfigfunc.ps1 +. c:\AzureData\k8s\windowskubeletfunc.ps1 +. c:\AzureData\k8s\windowscnifunc.ps1 +. c:\AzureData\k8s\windowsazurecnifunc.ps1 try { @@ -684,40 +129,90 @@ try Write-Log "Provisioning $global:DockerServiceName... with IP $MasterIP" Write-Log "apply telemetry data setting" - Set-TelemetrySetting + Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID Write-Log "resize os drive if possible" Resize-OSDrive Write-Log "download kubelet binaries and unzip" - Get-KubeBinaries - - # This is a workaround until Windows update - Write-Log "apply Windows patch packages" - Update-WindowsPackages - - Write-Log "Write azure config" - Write-AzureConfig + Get-KubeBinaries -KubeBinariesSASURL $global:KubeBinariesSASURL + + Write-Log "Write Azure cloud provider config" + Write-AzureConfig ` + -KubeDir $global:KubeDir ` + -AADClientId $AADClientId ` + -AADClientSecret $AADClientSecret ` + -TenantId $global:TenantId ` + -SubscriptionId $global:SubscriptionId ` + -ResourceGroup $global:ResourceGroup ` + -Location $Location ` + -VmType $global:VmType ` + -SubnetName $global:SubnetName ` + -SecurityGroupName $global:SecurityGroupName ` + -VNetName $global:VNetName ` + -RouteTableName $global:RouteTableName ` + -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName ` + -PrimaryScaleSetName $global:PrimaryScaleSetName ` + -UseManagedIdentityExtension $global:UseManagedIdentityExtension ` + -UserAssignedClientID $global:UserAssignedClientID ` + -UseInstanceMetadata $global:UseInstanceMetadata ` + -LoadBalancerSku $global:LoadBalancerSku ` + -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB Write-Log "Write ca root" - Write-CACert + Write-CACert -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir Write-Log "Write kube config" - Write-KubeConfig + Write-KubeConfig -CACertificate $global:CACertificate ` + -KubeDir $global:KubeDir ` + -MasterFQDNPrefix $MasterFQDNPrefix ` + -MasterIP $MasterIP ` + -AgentKey $AgentKey ` + -AgentCertificate $global:AgentCertificate - Write-Log "Create the Pause Container kubletwin/pause" - New-InfraContainer - Write-Log "Configure networking" - Set-NetworkConfig + Write-Log "Create the Pause Container kubletwin/pause" + New-InfraContainer -KubeDir $global:KubeDir + + Write-Log "Configuring networking with NetworkPlugin:$global:NetworkPlugin" + + # Configure network policy. + if ($global:NetworkPlugin -eq "azure") { + Install-VnetPlugins -AzureCNIConfDir $global:AzureCNIConfDir ` + -AzureCNIBinDir $global:AzureCNIBinDir ` + -VNetCNIPluginsURL $global:VNetCNIPluginsURL + Set-AzureCNIConfig -AzureCNIConfDir $global:AzureCNIConfDir ` + -KubeDnsSearchPath $global:KubeDnsSearchPath ` + -KubeClusterCIDR $global:KubeClusterCIDR ` + -MasterSubnet $global:MasterSubnet ` + -KubeServiceCIDR $global:KubeServiceCIDR + } elseif ($global:NetworkPlugin -eq "kubenet") { + Update-WinCNI -CNIPath $global:CNIPath + Get-HnsPsm1 -HNSModule $global:HNSModule + } Write-Log "write kubelet startfile with pod CIDR of $podCIDR" - Write-KubernetesStartFiles $podCIDR - - Write-Log "install the NSSM service" - New-NSSMService - - Write-Log "Set Internet Explorer" + Install-KubernetesServices ` + -KubeletConfigArgs $global:KubeletConfigArgs ` + -KubeBinariesVersion $global:KubeBinariesVersion ` + -NetworkPlugin $global:NetworkPlugin ` + -NetworkMode $global:NetworkMode ` + -KubeDir $global:KubeDir ` + -AzureCNIBinDir $global:AzureCNIBinDir ` + -AzureCNIConfDir $global:AzureCNIConfDir ` + -CNIPath $global:CNIPath ` + -CNIConfig $global:CNIConfig ` + -CNIConfigPath $global:CNIConfigPath ` + -MasterIP $MasterIP ` + -KubeDnsServiceIp $KubeDnsServiceIp ` + -MasterSubnet $global:MasterSubnet ` + -KubeClusterCIDR $global:KubeClusterCIDR ` + -KubeServiceCIDR $global:KubeServiceCIDR ` + -HNSModule $global:HNSModule ` + -KubeletNodeLabels $global:KubeletNodeLabels + + Write-Log "Disable Internet Explorer compat mode and set homepage" Set-Explorer Write-Log "Start preProvisioning script" diff --git a/parts/k8s/windowsazurecnifunc.ps1 b/parts/k8s/windowsazurecnifunc.ps1 new file mode 100644 index 0000000000..d117f8c9a3 --- /dev/null +++ b/parts/k8s/windowsazurecnifunc.ps1 @@ -0,0 +1,80 @@ + + +# TODO: remove - dead code? +function +Set-VnetPluginMode() +{ + Param( + [Parameter(Mandatory=$true)][string] + $AzureCNIConfDir, + [Parameter(Mandatory=$true)][string] + $Mode + ) + # Sets Azure VNET CNI plugin operational mode. + $fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist") + (Get-Content $fileName) | %{$_ -replace "`"mode`":.*", "`"mode`": `"$Mode`","} | Out-File -encoding ASCII -filepath $fileName +} + + +function +Install-VnetPlugins +{ + Param( + [Parameter(Mandatory=$true)][string] + $AzureCNIConfDir, + [Parameter(Mandatory=$true)][string] + $AzureCNIBinDir, + [Parameter(Mandatory=$true)][string] + $VNetCNIPluginsURL + ) + # Create CNI directories. + mkdir $AzureCNIBinDir + mkdir $AzureCNIConfDir + + # Download Azure VNET CNI plugins. + # Mirror from https://github.com/Azure/azure-container-networking/releases + $zipfile = [Io.path]::Combine("$AzureCNIDir", "azure-vnet.zip") + DownloadFileOverHttp -Url $VNetCNIPluginsURL -DestinationPath $zipfile + Expand-Archive -path $zipfile -DestinationPath $AzureCNIBinDir + del $zipfile + + # Windows does not need a separate CNI loopback plugin because the Windows + # kernel automatically creates a loopback interface for each network namespace. + # Copy CNI network config file and set bridge mode. + move $AzureCNIBinDir/*.conflist $AzureCNIConfDir +} + +# TODO: remove - dead code? +function +Set-AzureNetworkPlugin() +{ + # Azure VNET network policy requires tunnel (hairpin) mode because policy is enforced in the host. + Set-VnetPluginMode "tunnel" +} + +function +Set-AzureCNIConfig +{ + Param( + [Parameter(Mandatory=$true)][string] + $AzureCNIConfDir, + [Parameter(Mandatory=$true)][string] + $KubeDnsSearchPath, + [Parameter(Mandatory=$true)][string] + $KubeClusterCIDR, + [Parameter(Mandatory=$true)][string] + $MasterSubnet, + [Parameter(Mandatory=$true)][string] + $KubeServiceCIDR + ) + # Fill in DNS information for kubernetes. + $fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist") + $configJson = Get-Content $fileName | ConvertFrom-Json + $configJson.plugins.dns.Nameservers[0] = $KubeDnsServiceIp + $configJson.plugins.dns.Search[0] = $KubeDnsSearchPath + $configJson.plugins.AdditionalArgs[0].Value.ExceptionList[0] = $KubeClusterCIDR + $configJson.plugins.AdditionalArgs[0].Value.ExceptionList[1] = $MasterSubnet + $configJson.plugins.AdditionalArgs[1].Value.DestinationPrefix = $KubeServiceCIDR + + $configJson | ConvertTo-Json -depth 20 | Out-File -encoding ASCII -filepath $fileName +} diff --git a/parts/k8s/windowscnifunc.ps1 b/parts/k8s/windowscnifunc.ps1 new file mode 100644 index 0000000000..6c9d85ae28 --- /dev/null +++ b/parts/k8s/windowscnifunc.ps1 @@ -0,0 +1,25 @@ +function Get-HnsPsm1 +{ + Param( + [string] + $HnsUrl = "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1", + [Parameter(Mandatory=$true)][string] + $HNSModule + ) + DownloadFileOverHttp -Url $HnsUrl -DestinationPath "$HNSModule" +} + +function Update-WinCNI +{ + Param( + [string] + $WinCniUrl = "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/cni/wincni.exe", + [Parameter(Mandatory=$true)][string] + $CNIPath + ) + $wincni = "wincni.exe" + $wincniFile = [Io.path]::Combine($CNIPath, $wincni) + DownloadFileOverHttp -Url $WinCniUrl -DestinationPath $wincniFile +} + +# TODO: Move the code that creates the wincni configuration file out of windowskubeletfunc.ps1 and put it here \ No newline at end of file diff --git a/parts/k8s/windowsconfigfunc.ps1 b/parts/k8s/windowsconfigfunc.ps1 new file mode 100644 index 0000000000..cac1604dfd --- /dev/null +++ b/parts/k8s/windowsconfigfunc.ps1 @@ -0,0 +1,37 @@ + + +# Set the service telemetry GUID. This is used with Windows Analytics https://docs.microsoft.com/en-us/sccm/core/clients/manage/monitor-windows-analytics +function Set-TelemetrySetting +{ + Param( + [Parameter(Mandatory=$true)][string] + $WindowsTelemetryGUID + ) + Set-ItemProperty -Path "HKLM:\Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection" -Name "CommercialId" -Value $WindowsTelemetryGUID -Force +} + +# Resize the system partition to the max available size. Azure can resize a managed disk, but the VM still needs to extend the partition boundary +function Resize-OSDrive +{ + $osDrive = ((Get-WmiObject Win32_OperatingSystem).SystemDrive).TrimEnd(":") + $size = (Get-Partition -DriveLetter $osDrive).Size + $maxSize = (Get-PartitionSupportedSize -DriveLetter $osDrive).SizeMax + if ($size -lt $maxSize) + { + Resize-Partition -DriveLetter $osDrive -Size $maxSize + } +} + +# Set the Internet Explorer to use the latest rendering mode on all sites +# https://docs.microsoft.com/en-us/windows-hardware/customize/desktop/unattend/microsoft-windows-ie-internetexplorer-intranetcompatibilitymode +# (This only affects installations with UI) +function Set-Explorer +{ + New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer" + New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\BrowserEmulation" + New-ItemProperty -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\BrowserEmulation" -Name IntranetCompatibilityMode -Value 0 -Type DWord + New-Item -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\Main" + New-ItemProperty -Path HKLM:"\\SOFTWARE\\Policies\\Microsoft\\Internet Explorer\\Main" -Name "Start Page" -Type String -Value http://bing.com +} + +# TODO: Pagefile adjustments \ No newline at end of file diff --git a/parts/k8s/windowskubeletfunc.ps1 b/parts/k8s/windowskubeletfunc.ps1 new file mode 100644 index 0000000000..3a2755a8a8 --- /dev/null +++ b/parts/k8s/windowskubeletfunc.ps1 @@ -0,0 +1,587 @@ +function +Write-AzureConfig +{ + Param( + + [Parameter(Mandatory=$true)][string] + $AADClientId, + [Parameter(Mandatory=$true)][string] + $AADClientSecret, + [Parameter(Mandatory=$true)][string] + $TenantId, + [Parameter(Mandatory=$true)][string] + $SubscriptionId, + [Parameter(Mandatory=$true)][string] + $ResourceGroup, + [Parameter(Mandatory=$true)][string] + $Location, + [Parameter(Mandatory=$true)][string] + $VmType, + [Parameter(Mandatory=$true)][string] + $SubnetName, + [Parameter(Mandatory=$true)][string] + $SecurityGroupName, + [Parameter(Mandatory=$true)][string] + $VNetName, + [Parameter(Mandatory=$true)][string] + $RouteTableName, + [Parameter(Mandatory=$false)][string] # Need one of these configured + $PrimaryAvailabilitySetName, + [Parameter(Mandatory=$false)][string] # Need one of these configured + $PrimaryScaleSetName, + [Parameter(Mandatory=$true)][string] + $UseManagedIdentityExtension, + [string] + $UserAssignedClientID, + [Parameter(Mandatory=$true)][string] + $UseInstanceMetadata, + [Parameter(Mandatory=$true)][string] + $LoadBalancerSku, + [Parameter(Mandatory=$true)][string] + $ExcludeMasterFromStandardLB, + [Parameter(Mandatory=$true)][string] + $KubeDir + ) + + if ( -Not $PrimaryAvailabilitySetName -And -Not $PrimaryScaleSetName ) + { + throw "Either PrimaryAvailabilitySetName or PrimaryScaleSetName must be set" + } + + $azureConfigFile = [io.path]::Combine($KubeDir, "azure.json") + + $azureConfig = @" +{ + "tenantId": "$TenantId", + "subscriptionId": "$SubscriptionId", + "aadClientId": "$AADClientId", + "aadClientSecret": "$AADClientSecret", + "resourceGroup": "$ResourceGroup", + "location": "$Location", + "vmType": "$VmType", + "subnetName": "$SubnetName", + "securityGroupName": "$SecurityGroupName", + "vnetName": "$VNetName", + "routeTableName": "$RouteTableName", + "primaryAvailabilitySetName": "$PrimaryAvailabilitySetName", + "primaryScaleSetName": "$PrimaryScaleSetName", + "useManagedIdentityExtension": $UseManagedIdentityExtension, + "userAssignedIdentityID": $UserAssignedClientID, + "useInstanceMetadata": $UseInstanceMetadata, + "loadBalancerSku": "$LoadBalancerSku", + "excludeMasterFromStandardLB": $ExcludeMasterFromStandardLB +} +"@ + + $azureConfig | Out-File -encoding ASCII -filepath "$azureConfigFile" +} + + +function +Write-CACert +{ + Param( + [Parameter(Mandatory=$true)][string] + $CACertificate, + [Parameter(Mandatory=$true)][string] + $KubeDir + ) + $caFile = [io.path]::Combine($KubeDir, "ca.crt") + [System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String($CACertificate)) | Out-File -Encoding ascii $caFile +} + +function +Write-KubeConfig +{ + Param( + [Parameter(Mandatory=$true)][string] + $CACertificate, + [Parameter(Mandatory=$true)][string] + $MasterFQDNPrefix, + [Parameter(Mandatory=$true)][string] + $MasterIP, + [Parameter(Mandatory=$true)][string] + $AgentKey, + [Parameter(Mandatory=$true)][string] + $AgentCertificate, + [Parameter(Mandatory=$true)][string] + $KubeDir + ) + $kubeConfigFile = [io.path]::Combine($KubeDir, "config") + + $kubeConfig = @" +--- +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: "$CACertificate" + server: https://${MasterIP}:443 + name: "$MasterFQDNPrefix" +contexts: +- context: + cluster: "$MasterFQDNPrefix" + user: "$MasterFQDNPrefix-admin" + name: "$MasterFQDNPrefix" +current-context: "$MasterFQDNPrefix" +kind: Config +users: +- name: "$MasterFQDNPrefix-admin" + user: + client-certificate-data: "$AgentCertificate" + client-key-data: "$AgentKey" +"@ + + $kubeConfig | Out-File -encoding ASCII -filepath "$kubeConfigFile" +} + +function +New-InfraContainer +{ + Param( + [Parameter(Mandatory=$true)][string] + $KubeDir + ) + cd $KubeDir + $computerInfo = Get-ComputerInfo + $windowsBase = if ($computerInfo.WindowsVersion -eq "1709") { + "microsoft/nanoserver:1709" + } elseif ($computerInfo.WindowsVersion -eq "1803") { + "microsoft/nanoserver:1803" + } elseif ($computerInfo.WindowsVersion -eq "1809") { + # TODO: unsure if 2019 will report 1809 or not + "microsoft/nanoserver:1809" + } else { + "mcr.microsoft.com/nanoserver-insider" + } + + "FROM $($windowsBase)" | Out-File -encoding ascii -FilePath Dockerfile + "CMD cmd /c ping -t localhost" | Out-File -encoding ascii -FilePath Dockerfile -Append + docker build -t kubletwin/pause . +} + + +# TODO: Deprecate this and replace with methods that get individual components instead of zip containing everything +# This expects the ZIP file to be created by scripts/build-windows-k8s.sh +function +Get-KubeBinaries +{ + Param( + [Parameter(Mandatory=$true)][string] + $KubeBinariesSASURL + ) + + $zipfile = "c:\k.zip" + for ($i=0; $i -le 10; $i++) + { + DownloadFileOverHttp -Url $KubeBinariesSASURL -DestinationPath $zipfile + if ($?) { + break + } else { + Write-Log $Error[0].Exception.Message + } + } + Expand-Archive -path $zipfile -DestinationPath C:\ +} + + +# TODO: replace KubeletStartFile with a Kubelet config, remove NSSM, and use built-in service integration +function +New-NSSMService +{ + Param( + [string] + [Parameter(Mandatory=$true)] + $KubeDir, + [string] + [Parameter(Mandatory=$true)] + $KubeletStartFile, + [string] + [Parameter(Mandatory=$true)] + $KubeProxyStartFile + ) + + # setup kubelet + & "$KubeDir\nssm.exe" install Kubelet C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe + & "$KubeDir\nssm.exe" set Kubelet AppDirectory $KubeDir + & "$KubeDir\nssm.exe" set Kubelet AppParameters $KubeletStartFile + & "$KubeDir\nssm.exe" set Kubelet DisplayName Kubelet + & "$KubeDir\nssm.exe" set Kubelet Description Kubelet + & "$KubeDir\nssm.exe" set Kubelet Start SERVICE_AUTO_START + & "$KubeDir\nssm.exe" set Kubelet ObjectName LocalSystem + & "$KubeDir\nssm.exe" set Kubelet Type SERVICE_WIN32_OWN_PROCESS + & "$KubeDir\nssm.exe" set Kubelet AppThrottle 1500 + & "$KubeDir\nssm.exe" set Kubelet AppStdout C:\k\kubelet.log + & "$KubeDir\nssm.exe" set Kubelet AppStderr C:\k\kubelet.err.log + & "$KubeDir\nssm.exe" set Kubelet AppStdoutCreationDisposition 4 + & "$KubeDir\nssm.exe" set Kubelet AppStderrCreationDisposition 4 + & "$KubeDir\nssm.exe" set Kubelet AppRotateFiles 1 + & "$KubeDir\nssm.exe" set Kubelet AppRotateOnline 1 + & "$KubeDir\nssm.exe" set Kubelet AppRotateSeconds 86400 + & "$KubeDir\nssm.exe" set Kubelet AppRotateBytes 1048576 + + # setup kubeproxy + & "$KubeDir\nssm.exe" install Kubeproxy C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe + & "$KubeDir\nssm.exe" set Kubeproxy AppDirectory $KubeDir + & "$KubeDir\nssm.exe" set Kubeproxy AppParameters $KubeProxyStartFile + & "$KubeDir\nssm.exe" set Kubeproxy DisplayName Kubeproxy + & "$KubeDir\nssm.exe" set Kubeproxy DependOnService Kubelet + & "$KubeDir\nssm.exe" set Kubeproxy Description Kubeproxy + & "$KubeDir\nssm.exe" set Kubeproxy Start SERVICE_AUTO_START + & "$KubeDir\nssm.exe" set Kubeproxy ObjectName LocalSystem + & "$KubeDir\nssm.exe" set Kubeproxy Type SERVICE_WIN32_OWN_PROCESS + & "$KubeDir\nssm.exe" set Kubeproxy AppThrottle 1500 + & "$KubeDir\nssm.exe" set Kubeproxy AppStdout C:\k\kubeproxy.log + & "$KubeDir\nssm.exe" set Kubeproxy AppStderr C:\k\kubeproxy.err.log + & "$KubeDir\nssm.exe" set Kubeproxy AppRotateFiles 1 + & "$KubeDir\nssm.exe" set Kubeproxy AppRotateOnline 1 + & "$KubeDir\nssm.exe" set Kubeproxy AppRotateSeconds 86400 + & "$KubeDir\nssm.exe" set Kubeproxy AppRotateBytes 1048576 +} + +# Renamed from Write-KubernetesStartFiles +function +Install-KubernetesServices +{ + param( + [Parameter(Mandatory=$true)][string[]] + $KubeletConfigArgs, + [Parameter(Mandatory=$true)][string] + $KubeBinariesVersion, + [Parameter(Mandatory=$true)][string] + $NetworkPlugin, + [Parameter(Mandatory=$true)][string] + $NetworkMode, + [Parameter(Mandatory=$true)][string] + $KubeDir, + [Parameter(Mandatory=$true)][string] + $AzureCNIBinDir, + [Parameter(Mandatory=$true)][string] + $AzureCNIConfDir, + [Parameter(Mandatory=$true)][string] + $CNIPath, + [Parameter(Mandatory=$true)][string] + $CNIConfig, + [Parameter(Mandatory=$true)][string] + $CNIConfigPath, + [Parameter(Mandatory=$true)][string] + $MasterIP, + [Parameter(Mandatory=$true)][string] + $KubeDnsServiceIp, + [Parameter(Mandatory=$true)][string] + $MasterSubnet, + [Parameter(Mandatory=$true)][string] + $KubeClusterCIDR, + [Parameter(Mandatory=$true)][string] + $KubeServiceCIDR, + [Parameter(Mandatory=$true)][string] + $HNSModule, + [Parameter(Mandatory=$true)][string] + $KubeletNodeLabels + ) + + # Calculate some local paths + $VolumePluginDir = [Io.path]::Combine($KubeDir, "volumeplugins") + $KubeletStartFile = [io.path]::Combine($KubeDir, "kubeletstart.ps1") + $KubeProxyStartFile = [io.path]::Combine($KubeDir, "kubeproxystart.ps1") + + mkdir $VolumePluginDir + $KubeletArgList = $KubeletConfigArgs # This is the initial list passed in from acs-engine + $KubeletArgList += "--node-labels=`$global:KubeletNodeLabels" + # $KubeletArgList += "--hostname-override=`$global:AzureHostname" TODO: remove - dead code? + $KubeletArgList += "--volume-plugin-dir=`$global:VolumePluginDir" + # If you are thinking about adding another arg here, you should be considering pkg/acsengine/defaults-kubelet.go first + # Only args that need to be calculated or combined with other ones on the Windows agent should be added here. + + + # Regex to strip version to Major.Minor.Build format such that the following check does not crash for version like x.y.z-alpha + [regex]$regex = "^[0-9.]+" + $KubeBinariesVersionStripped = $regex.Matches($KubeBinariesVersion).Value + if ([System.Version]$KubeBinariesVersionStripped -lt [System.Version]"1.8.0") + { + # --api-server deprecates from 1.8.0 + $KubeletArgList += "--api-servers=https://`${global:MasterIP}:443" + } + + # Configure kubelet to use CNI plugins if enabled. + if ($NetworkPlugin -eq "azure") { + $KubeletArgList += @("--cni-bin-dir=$AzureCNIBinDir", "--cni-conf-dir=$AzureCNIConfDir") + } elseif ($NetworkPlugin -eq "kubenet") { + $KubeletArgList += @("--cni-bin-dir=$CNIPath", "--cni-conf-dir=$CNIConfigPath") + # handle difference in naming between Linux & Windows reference plugin + $KubeletArgList = $KubeletArgList -replace "kubenet", "cni" + } else { + throw "Unknown network type $NetworkPlugin, can't configure kubelet" + } + + # Used in WinCNI version of kubeletstart.ps1 + $KubeletArgListStr = "" + $KubeletArgList | Foreach-Object { + # Since generating new code to be written to a file, need to escape quotes again + if ($KubeletArgListStr.length -gt 0) + { + $KubeletArgListStr = $KubeletArgListStr + ", " + } + $KubeletArgListStr = $KubeletArgListStr + "`"" + $_.Replace("`"`"","`"`"`"`"") + "`"" + } + $KubeletArgListStr = "@`($KubeletArgListStr`)" + + # Used in Azure-CNI version of kubeletstart.ps1 + $KubeletCommandLine = "$KubeDir\kubelet.exe " + ($KubeletArgList -join " ") + + $kubeStartStr = @" +`$global:MasterIP = "$MasterIP" +`$global:KubeDnsSearchPath = "svc.cluster.local" +`$global:KubeDnsServiceIp = "$KubeDnsServiceIp" +`$global:MasterSubnet = "$MasterSubnet" +`$global:KubeClusterCIDR = "$KubeClusterCIDR" +`$global:KubeServiceCIDR = "$KubeServiceCIDR" +`$global:KubeBinariesVersion = "$KubeBinariesVersion" +`$global:CNIPath = "$CNIPath" +`$global:NetworkMode = "$NetworkMode" +`$global:ExternalNetwork = "ext" +`$global:CNIConfig = "$CNIConfig" +`$global:HNSModule = "$HNSModule" +`$global:VolumePluginDir = "$VolumePluginDir" +`$global:NetworkPlugin="$NetworkPlugin" +`$global:KubeletNodeLabels="$KubeletNodeLabels" + +"@ + + if ($NetworkPlugin -eq "azure") { + $KubeNetwork = "azure" + $kubeStartStr += @" +Write-Host "NetworkPlugin azure, starting kubelet." + +# Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) +netsh advfirewall set allprofiles state off +# startup the service + +# Find if the primary external switch network exists. If not create one. +# This is done only once in the lifetime of the node +`$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:ExternalNetwork +if (!`$hnsNetwork) +{ + Write-Host "Creating a new hns Network" + ipmo `$global:HNSModule + # Fixme : use a smallest range possible, that will not collide with any pod space + New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix "192.168.255.0/30" -Gateway "192.168.255.1" -Name `$global:ExternalNetwork -Verbose +} + +# Find if network created by CNI exists, if yes, remove it +# This is required to keep the network non-persistent behavior +# Going forward, this would be done by HNS automatically during restart of the node + +`$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork +if (`$hnsNetwork) +{ + # Cleanup all containers + docker ps -q | foreach {docker rm `$_ -f} + + Write-Host "Cleaning up old HNS network found" + Remove-HnsNetwork `$hnsNetwork + # Kill all cni instances & stale data left by cni + # Cleanup all files related to cni + `$cnijson = [io.path]::Combine("$KubeDir", "azure-vnet-ipam.json") + if ((Test-Path `$cnijson)) + { + Remove-Item `$cnijson + } + `$cnilock = [io.path]::Combine("$KubeDir", "azure-vnet-ipam.lock") + if ((Test-Path `$cnilock)) + { + Remove-Item `$cnilock + } + taskkill /IM azure-vnet-ipam.exe /f + + `$cnijson = [io.path]::Combine("$KubeDir", "azure-vnet.json") + if ((Test-Path `$cnijson)) + { + Remove-Item `$cnijson + } + `$cnilock = [io.path]::Combine("$KubeDir", "azure-vnet.lock") + if ((Test-Path `$cnilock)) + { + Remove-Item `$cnilock + } + taskkill /IM azure-vnet.exe /f +} + +# Restart Kubeproxy, which would wait, until the network is created +Restart-Service Kubeproxy + +$KubeletCommandLine + +"@ + } + else # using WinCNI. TODO: If WinCNI support is removed, then delete this as dead code later + { + $KubeNetwork = "l2bridge" + $kubeStartStr += @" + +function +Get-DefaultGateway(`$CIDR) +{ + return `$CIDR.substring(0,`$CIDR.lastIndexOf(".")) + ".1" +} + +function +Get-PodCIDR() +{ + `$podCIDR = c:\k\kubectl.exe --kubeconfig=c:\k\config get nodes/`$(`$env:computername.ToLower()) -o custom-columns=podCidr:.spec.podCIDR --no-headers + return `$podCIDR +} + +function +Test-PodCIDR(`$podCIDR) +{ + return `$podCIDR.length -gt 0 +} + +function +Update-CNIConfig(`$podCIDR, `$masterSubnetGW) +{ + `$jsonSampleConfig = +"{ + ""cniVersion"": ""0.2.0"", + ""name"": """", + ""type"": ""wincni.exe"", + ""master"": ""Ethernet"", + ""capabilities"": { ""portMappings"": true }, + ""ipam"": { + ""environment"": ""azure"", + ""subnet"":"""", + ""routes"": [{ + ""GW"":"""" + }] + }, + ""dns"" : { + ""Nameservers"" : [ """" ], + ""Search"" : [ """" ] + }, + ""AdditionalArgs"" : [ + { + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""OutBoundNAT"", ""ExceptionList"": [ """", """" ] } + }, + { + ""Name"" : ""EndpointPolicy"", ""Value"" : { ""Type"" : ""ROUTE"", ""DestinationPrefix"": """", ""NeedEncap"" : true } + } + ] +}" + + `$configJson = ConvertFrom-Json `$jsonSampleConfig + `$configJson.name = `$global:NetworkMode.ToLower() + `$configJson.ipam.subnet=`$podCIDR + `$configJson.ipam.routes[0].GW = `$masterSubnetGW + `$configJson.dns.Nameservers[0] = `$global:KubeDnsServiceIp + `$configJson.dns.Search[0] = `$global:KubeDnsSearchPath + + `$configJson.AdditionalArgs[0].Value.ExceptionList[0] = `$global:KubeClusterCIDR + `$configJson.AdditionalArgs[0].Value.ExceptionList[1] = `$global:MasterSubnet + `$configJson.AdditionalArgs[1].Value.DestinationPrefix = `$global:KubeServiceCIDR + + if (Test-Path `$global:CNIConfig) + { + Clear-Content -Path `$global:CNIConfig + } + + Write-Host "Generated CNI Config [`$configJson]" + + Add-Content -Path `$global:CNIConfig -Value (ConvertTo-Json `$configJson -Depth 20) +} + +try +{ + `$masterSubnetGW = Get-DefaultGateway `$global:MasterSubnet + `$podCIDR=Get-PodCIDR + `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) + + # if the podCIDR has not yet been assigned to this node, start the kubelet process to get the podCIDR, and then promptly kill it. + if (-not `$podCidrDiscovered) + { + `$argList = $KubeletArgListStr + + `$process = Start-Process -FilePath c:\k\kubelet.exe -PassThru -ArgumentList `$argList + + # run kubelet until podCidr is discovered + Write-Host "waiting to discover pod CIDR" + while (-not `$podCidrDiscovered) + { + Write-Host "Sleeping for 10s, and then waiting to discover pod CIDR" + Start-Sleep 10 + + `$podCIDR=Get-PodCIDR + `$podCidrDiscovered=Test-PodCIDR(`$podCIDR) + } + + # stop the kubelet process now that we have our CIDR, discard the process output + `$process | Stop-Process | Out-Null + } + + # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) + netsh advfirewall set allprofiles state off + + # startup the service + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:NetworkMode.ToLower() + + if (`$hnsNetwork) + { + # Kubelet has been restarted with existing network. + # Cleanup all containers + docker ps -q | foreach {docker rm `$_ -f} + # cleanup network + Write-Host "Cleaning up old HNS network found" + Remove-HnsNetwork `$hnsNetwork + Start-Sleep 10 + } + + Write-Host "Creating a new hns Network" + ipmo `$global:HNSModule + + `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$masterSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose + # New network has been created, Kubeproxy service has to be restarted + Restart-Service Kubeproxy + + Start-Sleep 10 + # Add route to all other POD networks + Update-CNIConfig `$podCIDR `$masterSubnetGW + + $KubeletCommandLine +} +catch +{ + Write-Error `$_ +} + +"@ + } # end else using WinCNI. + + # Now that the script is generated, based on what CNI plugin and startup options are needed, write it to disk + $kubeStartStr | Out-File -encoding ASCII -filepath $KubeletStartFile + + $kubeProxyStartStr = @" +`$env:KUBE_NETWORK = "$KubeNetwork" +`$global:NetworkMode = "$NetworkMode" +`$global:HNSModule = "$HNSModule" +`$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork +while (!`$hnsNetwork) +{ + Write-Host "Waiting for Network [$KubeNetwork] to be created . . ." + Start-Sleep 10 + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork +} + +# +# cleanup the persisted policy lists +# +ipmo `$global:HNSModule +Get-HnsPolicyList | Remove-HnsPolicyList + +$KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config +"@ + + $kubeProxyStartStr | Out-File -encoding ASCII -filepath $KubeProxyStartFile + + New-NSSMService -KubeDir $KubeDir ` + -KubeletStartFile $KubeletStartFile ` + -KubeProxyStartFile $KubeProxyStartFile +} \ No newline at end of file diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index b34529c3bc..c571dc9dc2 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -122,7 +122,13 @@ const ( kubernetesAgentCustomDataYaml = "k8s/kubernetesagentcustomdata.yml" kubernetesJumpboxCustomDataYaml = "k8s/kubernetesjumpboxcustomdata.yml" kubeConfigJSON = "k8s/kubeconfig.json" - kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1" + // Windows custom scripts + kubernetesWindowsAgentCustomDataPS1 = "k8s/kuberneteswindowssetup.ps1" + kubernetesWindowsAgentFunctionsPS1 = "k8s/kuberneteswindowsfunctions.ps1" + kubernetesWindowsConfigFunctionsPS1 = "k8s/windowsconfigfunc.ps1" + kubernetesWindowsKubeletFunctionsPS1 = "k8s/windowskubeletfunc.ps1" + kubernetesWindowsCniFunctionsPS1 = "k8s/windowscnifunc.ps1" + kubernetesWindowsAzureCniFunctionsPS1 = "k8s/windowsazurecnifunc.ps1" // OpenShift custom scripts openshiftNodeScript = "openshift/unstable/openshiftnodescript.sh" openshiftMasterScript = "openshift/unstable/openshiftmasterscript.sh" diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index c9278150c6..bbb9b9c2a2 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -650,8 +650,8 @@ func getSecurityRules(ports []int) string { return buf.String() } -// getSingleLineForTemplate returns the file as a single line for embedding in an arm template -func (t *TemplateGenerator) getSingleLineForTemplate(textFilename string, cs *api.ContainerService, profile interface{}) (string, error) { +// getSingleLine returns the file as a single line +func (t *TemplateGenerator) getSingleLine(textFilename string, cs *api.ContainerService, profile interface{}) (string, error) { b, err := Asset(textFilename) if err != nil { return "", t.Translator.Errorf("yaml file %s does not exist", textFilename) @@ -669,6 +669,16 @@ func (t *TemplateGenerator) getSingleLineForTemplate(textFilename string, cs *ap } expandedTemplate := buffer.String() + return expandedTemplate, nil +} + +// getSingleLineForTemplate returns the file as a single line for embedding in an arm template +func (t *TemplateGenerator) getSingleLineForTemplate(textFilename string, cs *api.ContainerService, profile interface{}) (string, error) { + expandedTemplate, err := t.getSingleLine(textFilename, cs, profile) + if err != nil { + return "", err + } + textStr := escapeSingleLine(string(expandedTemplate)) return textStr, nil diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index cf2b0f8bfd..c1fd06869b 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -1,6 +1,7 @@ package acsengine import ( + "archive/zip" "bytes" "encoding/base64" "fmt" @@ -667,16 +668,54 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat str := getBase64CustomScript(swarmModeWindowsProvision) return fmt.Sprintf("\"customData\": \"%s\"", str) }, + "GetKubernetesWindowsAgentFunctions": func() string { + // Collect all the parts into a zip + var parts = []string{ + kubernetesWindowsAgentFunctionsPS1, + kubernetesWindowsConfigFunctionsPS1, + kubernetesWindowsKubeletFunctionsPS1, + kubernetesWindowsCniFunctionsPS1, + kubernetesWindowsAzureCniFunctionsPS1} + + // Create a buffer, new zip + buf := new(bytes.Buffer) + zw := zip.NewWriter(buf) + + for _, part := range parts { + f, err := zw.Create(part) + if err != nil { + panic(err) + } + partContents, err := Asset(part) + if err != nil { + panic(err) + } + _, err = f.Write([]byte(partContents)) + if err != nil { + panic(err) + } + } + err := zw.Close() + if err != nil { + panic(err) + } + return base64.StdEncoding.EncodeToString(buf.Bytes()) + }, "GetKubernetesWindowsAgentCustomData": func(profile *api.AgentPoolProfile) string { str, e := t.getSingleLineForTemplate(kubernetesWindowsAgentCustomDataPS1, cs, profile) + if e != nil { panic(e) } + preprovisionCmd := "" + if profile.PreprovisionExtension != nil { preprovisionCmd = makeAgentExtensionScriptCommands(cs, profile) } + str = strings.Replace(str, "PREPROVISION_EXTENSION", escapeSingleLine(strings.TrimSpace(preprovisionCmd)), -1) + return fmt.Sprintf("\"customData\": \"[base64(concat('%s'))]\",", str) }, "GetMasterSwarmModeCustomData": func() string {