Enabled preprovisioning on windows dcos agents (#2228)

extensions/hello-world-dcos-windows/README.md

@@ -0,0 +1,46 @@
+# hello-world-dcos Extension
+
+Sample hello-world extension.  Calls the following on the master:
+
+```
+ curl -X post http://localhost:8080/v2/apps -d "{ \"id\": \"hello-marathon\", \"cmd\": \"while [ true ] ; do echo 'Hello World' ; sleep 5 ; done\", \"cpus\": 0.1, \"mem\": 10.0, \"instances\": 1 }" -H "Content-type:application/json"
+```
+
+You can validate that the extension was run by running (make sure you have tunneled into the master):
+```
+dcos auth login
+dcos task log hello-marathon 
+```
+
+# Configuration
+|Name|Required|Acceptable Value|
+|---|---|---|
+|name|yes|hello-world-k8s|
+|version|yes|v1|
+|extensionParameters|no||
+|rootURL|optional||
+
+# Example
+``` javascript
+    "masterProfile": {
+      ...
+      "extensions": [
+        { 
+          "name": "hello-world-dcos", 
+          "singleOrAll": "single"
+        }
+     ]
+    },
+    ...
+    "extensionProfiles": [
+      { 
+        "name": "hello-world-dcos", 
+        "version": "v1"
+      }
+    ]
+
+
+```
+
+# Supported Orchestrators
+"DCOS", "DCOS173", "DCOS184", "DCOS188"

extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1

@@ -0,0 +1,20 @@
+# Script file to run hello-world in dcos
+
+#!/usr/bin/pwsh
+
+
+Write-Host "$(date) - Starting Script"
+
+# Deploy container
+Write-Host "$(date)  - Deploying hello-world"
+
+$uri =  "http://"+($env:DCOS_AGENT_IP)+":5051/metrics/snapshot"
+while($true) {
+    $obj = ((Invoke-Webrequest -Method GET -URI $uri ).Content | ConvertFrom-JSON )
+    Write-Host "$(date) - system/cpus_total = " ($obj.'system/cpus_total') ", mem free bytes = " ($obj.'system/mem_free_bytes') ", mem total bytes = " ($obj.'system/mem_total_bytes') 
+    sleep 5
+}
+
+Write-Host "$(date) - view resources in mesos UI to validate"
+Write-Host "$(date) - Script complete"
+

extensions/hello-world-dcos-windows/v1/supported-orchestrators.json

@@ -0,0 +1 @@
+["DCOS", "DCOS173", "DCOS184", "DCOS188", "DCOS190"]

extensions/hello-world-dcos-windows/v1/template-link.json

@@ -0,0 +1,33 @@
+{
+    "name": "[concat(EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET), 'HelloWorldDcos')]",
+    "type": "Microsoft.Resources/deployments",
+    "apiVersion": "[variables('apiVersionLinkDefault')]",
+    "dependsOn": [
+        "[resourceId('Microsoft.Compute/virtualMachines/extensions', concat(variables('masterVMNamePrefix'), sub(variables('masterCount'), 1)), 'waitforleader')]"
+    ],
+    "copy": {
+        "count": "EXTENSION_LOOP_COUNT",
+        "name": "helloWorldExtensionLoop"
+    },
+    "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+            "uri": "EXTENSION_URL_REPLACEextensions/hello-world-dcos-windows/v1/template.json",
+            "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+            "artifactsLocation": {
+                "value": "EXTENSION_URL_REPLACE"
+            },
+            "apiVersionDefault": {
+                "value": "[variables('apiVersionDefault')]"
+            },
+            "targetVMName": {
+                "value": "[concat(EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET))]"
+            },
+            "extensionParameters": {
+                "value": "EXTENSION_PARAMETERS_REPLACE"
+            }
+        }
+    }
+}

extensions/hello-world-dcos-windows/v1/template.json

@@ -0,0 +1,62 @@
+{
+   "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+   "contentVersion": "1.0.0.0",
+   "parameters": {  
+		"artifactsLocation": {
+			"type": "string",
+			"minLength": 1,
+			"metadata": {
+				"description": "Artifacts Location - URL"
+			}
+        },
+		"apiVersionDefault": {
+			"type": "string",
+			"minLength": 1,
+			"metadata": {
+				"description": "Compute API Version"
+			}
+		},
+		"targetVMName":{
+			"type": "string",
+			"minLength": 1,
+			"metadata": {
+				"description": "Name of the vm to run the "
+			}
+		},
+		"extensionParameters": {
+			"type": "securestring",
+			"minLength": 0,
+			"metadata": {
+				"description": "Custom Parameter for Extension - for hello-world, this is empty"
+			}
+		}
+   },
+   "variables": {  
+		"singleQuote": "'",
+		"initScriptUrl": "[concat(parameters('artifactsLocation'), 'extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1')]"
+   },
+   "resources": [  
+	{
+      "apiVersion": "[parameters('apiVersionDefault')]",
+      "dependsOn": [],
+      "location": "[resourceGroup().location]",
+      "type": "Microsoft.Compute/virtualMachines/extensions",
+	  "name": "[concat(parameters('targetVMName'),'/waitforleader')]",
+      "properties": {
+        "publisher": "Microsoft.OSTCExtensions",
+        "type": "CustomScriptForLinux",
+        "typeHandlerVersion": "1.5",
+        "autoUpgradeMinorVersion": true,
+        "settings": {
+			"fileUris": [ 
+			   "[variables('initScriptUrl')]" 
+			 ]
+		},
+        "protectedSettings": {
+			"commandToExecute": "[concat('powershell \"./hello-world-dcos.ps1 ', variables('singleQuote'), parameters('extensionParameters'), variables('singleQuote'), ' >> c:/azuredata/hello-world-dcos-provision.log 2>&1 &\" &')]"
+        }
+      }
+    }
+	],
+   "outputs": {  }
+}

parts/dcos/dcosWindowsProvision.ps1

@@ -37,7 +37,11 @@ param(
 
     [string]
     [AllowNull()]
-    $customAttrs = ""
+    $customAttrs = "",
+
+    [string]
+    [AllowNull()]
+    $preprovisionExtensionParams = ""
 )
 
 
@@ -139,7 +143,7 @@ try
     # the output.
     Write-Log "Get the install script"
 
-    Write-Log ("Parameters = isAgent = ["+ $isAgent + "] mastercount = ["+$MasterCount + "] First master ip= [" + $firstMasterIp+ "] boostrap URI = ["+ $bootstrapUri+"] Subnet = ["+ $subnet +"]" + " -customAttrs " + $customAttrs )
+    Write-Log ("Parameters = isAgent = ["+ $isAgent + "] mastercount = ["+$MasterCount + "] First master ip= [" + $firstMasterIp+ "] boostrap URI = ["+ $bootstrapUri+"] Subnet = ["+ $subnet +"]" + " -customAttrs " + $customAttrs + " -preprovisionExtensionParms = "+ $preprovisionExtensionParams )
 
     # Get the boostrap script
 
@@ -201,6 +205,8 @@ try
         Write-Log "run setup script $run_cmd"
         Invoke-Expression $run_cmd
     }
+
+    PREPROVISION_EXTENSION
 }
 catch
 {

parts/dcos/dcosagentvars.t

@@ -7,11 +7,11 @@
     {{if IsPublic .Ports}}
         "{{.Name}}VMNamePrefix": "[concat('wp', variables('winResourceNamePrefix'), add(900,variables('{{.Name}}Index')))]",
         "{{.Name}}windowsAgentCustomAttributes": "[concat(' -customAttrs ', variables('doubleSingleQuote'), '{{GetDCOSWindowsAgentCustomNodeAttributes . }}', variables('doubleSingleQuote') )]",
-        "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $true ',  variables('{{.Name}}windowsAgentCustomAttributes'), variables('singleQuote'), ' ; ')]",
+        "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $true ',  variables('{{.Name}}windowsAgentCustomAttributes'), ' -preprovisionExtensionParams \"{{GetDCOSWindowsAgentPreprovisionParameters .}}\"',  variables('singleQuote'), ' ; ')]",
     {{else}}
         "{{.Name}}VMNamePrefix": "[concat('w', variables('winResourceNamePrefix'), add(900,variables('{{.Name}}Index')))]",
         "{{.Name}}windowsAgentCustomAttributes": "[concat(' -customAttrs ', variables('doubleSingleQuote'), '{{GetDCOSWindowsAgentCustomNodeAttributes . }}', variables('doubleSingleQuote') )]",
-        "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $false ',  variables('{{.Name}}windowsAgentCustomAttributes'), variables('singleQuote'), ' ; ')]",
+        "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $false ',  variables('{{.Name}}windowsAgentCustomAttributes'), ' -preprovisionExtensionParams \"{{GetDCOSWindowsAgentPreprovisionParameters .}}\"', variables('singleQuote'), ' ; ')]",
     {{end}}
 
     "{{.Name}}windowsAgentCustomScript": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', variables('{{.Name}}windowsAgentCustomScriptArguments'), variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\dcosWindowsProvision.log 2>&1')]",

parts/dcos/dcoscustomdata110.t

@@ -71,7 +71,7 @@ mounts:
   - /dcos/volume2
 - - /dev/sdf1
   - /dcos/volume3
-runcmd:
+runcmd: PREPROVISION_EXTENSION
 - - ln
   - -s
   - /bin/rm

pkg/acsengine/engine.go

@@ -1021,12 +1021,33 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			return fmt.Sprintf("\"customData\": \"[base64(concat('#cloud-config\\n\\n', '%s'))]\",", str)
 		},
 		"GetDCOSWindowsAgentCustomData": func(profile *api.AgentPoolProfile) string {
-			str := getBase64CustomScript(dcosWindowsProvision)
+			agentPreprovisionExtension := ""
+			if profile.PreprovisionExtension != nil {
+				agentPreprovisionExtension += "\n"
+				agentPreprovisionExtension += makeAgentExtensionScriptCommands(cs, profile)
+			}
+			b, err := Asset(dcosWindowsProvision)
+			if err != nil {
+				// this should never happen and this is a bug
+				panic(fmt.Sprintf("BUG: %s", err.Error()))
+			}
+			// translate the parameters
+			csStr := string(b)
+			csStr = strings.Replace(csStr, "PREPROVISION_EXTENSION", agentPreprovisionExtension, -1)
+			csStr = strings.Replace(csStr, "\r\n", "\n", -1)
+			str := getBase64CustomScriptFromStr(csStr)
 			return fmt.Sprintf("\"customData\": \"%s\"", str)
 		},
 		"GetDCOSWindowsAgentCustomNodeAttributes": func(profile *api.AgentPoolProfile) string {
 			return getDCOSWindowsAgentCustomAttributes(profile)
 		},
+		"GetDCOSWindowsAgentPreprovisionParameters": func(profile *api.AgentPoolProfile) string {
+			agentPreprovisionExtensionParameters := ""
+			if profile.PreprovisionExtension != nil {
+				agentPreprovisionExtensionParameters = getDCOSWindowsAgentPreprovisionParameters(cs, profile)
+			}
+			return agentPreprovisionExtensionParameters
+		},
 		"GetMasterAllowedSizes": func() string {
 			if t.ClassicMode {
 				return GetClassicAllowedSizes()
@@ -1609,6 +1630,10 @@ func makeAgentExtensionScriptCommands(cs *api.ContainerService, profile *api.Age
 	if profile.IsAvailabilitySets() {
 		copyIndex = fmt.Sprintf("',copyIndex(variables('%sOffset')),'", profile.Name)
 	}
+	if profile.OSType == api.Windows {
+		return makeWindowsExtensionScriptCommands(profile.PreprovisionExtension,
+			cs.Properties.ExtensionProfiles, copyIndex)
+	}
 	return makeExtensionScriptCommands(profile.PreprovisionExtension,
 		cs.Properties.ExtensionProfiles, copyIndex)
 }
@@ -1633,6 +1658,42 @@ func makeExtensionScriptCommands(extension *api.Extension, extensionProfiles []*
 		scriptFilePath, scriptURL, scriptFilePath, scriptFilePath, extensionsParameterReference, extensionProfile.Name)
 }
 
+func makeWindowsExtensionScriptCommands(extension *api.Extension, extensionProfiles []*api.ExtensionProfile, copyIndex string) string {
+	var extensionProfile *api.ExtensionProfile
+	for _, eP := range extensionProfiles {
+		if strings.EqualFold(eP.Name, extension.Name) {
+			extensionProfile = eP
+			break
+		}
+	}
+
+	if extensionProfile == nil {
+		panic(fmt.Sprintf("%s extension referenced was not found in the extension profile", extension.Name))
+	}
+
+	scriptURL := getExtensionURL(extensionProfile.RootURL, extensionProfile.Name, extensionProfile.Version, extensionProfile.Script, extensionProfile.URLQuery)
+	scriptFileDir := fmt.Sprintf("$env:SystemDrive:/AzureData/extensions/%s", extensionProfile.Name)
+	scriptFilePath := fmt.Sprintf("%s/%s", scriptFileDir, extensionProfile.Script)
+	return fmt.Sprintf("New-Item -ItemType Directory -Force -Path \"%s\" ; Invoke-WebRequest -Uri \"%s\" -OutFile \"%s\" ; powershell \"%s %s\"\n", scriptFileDir, scriptURL, scriptFilePath, scriptFilePath, "$preprovisionExtensionParams")
+}
+
+func getDCOSWindowsAgentPreprovisionParameters(cs *api.ContainerService, profile *api.AgentPoolProfile) string {
+	extension := profile.PreprovisionExtension
+	parms := ""
+
+	var extensionProfile *api.ExtensionProfile
+
+	for _, eP := range cs.Properties.ExtensionProfiles {
+		if strings.EqualFold(eP.Name, extension.Name) {
+			extensionProfile = eP
+			break
+		}
+	}
+
+	parms = extensionProfile.ExtensionParameters
+	return parms
+}
+
 func getPackageGUID(orchestratorType string, orchestratorVersion string, masterCount int) string {
 	if orchestratorType == api.DCOS {
 		switch orchestratorVersion {