Fully automate cluster buildout. Add azure file share mount to stress…

… tests.
Azure · Oct 14, 2021 · 8ac50dc · 8ac50dc
1 parent 04becd1
commit 8ac50dc
Show file tree

Hide file tree

Showing 17 changed files with 344 additions and 48 deletions.
diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1
@@ -76,9 +76,9 @@ function DeployStressTests(
     [string]$environment = 'test',
     [string]$repository = 'images',
     [boolean]$pushImages = $false,
-    [string]$clusterGroup = 'rg-stress-test-cluster-',
+    [string]$clusterGroup = 'rg-stress-cluster-test',
     [string]$deployId = 'local',
-    [string]$subscription = 'Azure SDK Test Resources'
+    [string]$subscription = 'Azure SDK Developer Playground'
 ) {
     if ($PSCmdlet.ParameterSetName -eq 'DoLogin') {
         Login $subscription $clusterGroup $pushImages

diff --git a/tools/stress-cluster/cluster/README.md b/tools/stress-cluster/cluster/README.md
@@ -122,9 +122,11 @@ az ad sp create-for-rbac -n 'stress-test-provisioner' --role Contributor --scope
 Create an env file with the service principal values created above:
 
 ```
+AZURE_CLIENT_OID=<app object id>
 AZURE_CLIENT_ID=<app id>
 AZURE_CLIENT_SECRET=<password/secret>
 AZURE_TENANT_ID=<tenant id>
+AZURE_SUBSCRIPTION_ID=<subscription id>
 ```
 
 Upload it to the static keyvault:

diff --git a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep
@@ -4,20 +4,52 @@ param groupSuffix string
 param dnsPrefix string = 's1'
 param clusterName string
 param location string = resourceGroup().location
-param agentVMSize string = 'Standard_D2_v3'
-
-@minValue(1)
-@maxValue(50)
-@description('The number of nodes for the cluster.')
-param agentCount int = 3
+param enableHighMemAgentPool bool = false
 
 // monitoring parameters
 param enableMonitoring bool = false
 param workspaceId string
 
-var kubernetesVersion = '1.20.5'
+var kubernetesVersion = '1.21.2'
 var nodeResourceGroup = 'rg-nodes-${dnsPrefix}-${clusterName}-${groupSuffix}'
-var agentPoolName = 'agentpool01'
+
+var defaultAgentPool = {
+  name: 'default'
+  count: 3
+  minCount: 3
+  maxCount: 9
+  mode: 'System'
+  vmSize: 'Standard_D2_v3'
+  type: 'VirtualMachineScaleSets'
+  osType: 'Linux'
+  enableAutoScaling: true
+  enableEncryptionAtHost: true
+  nodeLabels: {
+      'sku': 'default'
+  }
+}
+
+var highMemAgentPool = {
+  name: 'highmemory'
+  count: 1
+  minCount: 1
+  maxCount: 3
+  mode: 'System'
+  vmSize: 'Standard_D4ds_v4'
+  type: 'VirtualMachineScaleSets'
+  osType: 'Linux'
+  enableAutoScaling: true
+  enableEncryptionAtHost: true
+  nodeLabels: {
+      'sku': 'highMem'
+  }
+}
+
+var agentPools = concat([
+        defaultAgentPool
+    ], enableHighMemAgentPool ? [
+        highMemAgentPool
+    ] : [])
 
 resource cluster 'Microsoft.ContainerService/managedClusters@2020-09-01' = {
   name: clusterName
@@ -41,17 +73,7 @@ resource cluster 'Microsoft.ContainerService/managedClusters@2020-09-01' = {
     kubernetesVersion: kubernetesVersion
     enableRBAC: true
     dnsPrefix: dnsPrefix
-    agentPoolProfiles: [
-      {
-        name: agentPoolName
-        count: agentCount
-        mode: 'System'
-        vmSize: agentVMSize
-        type: 'VirtualMachineScaleSets'
-        osType: 'Linux'
-        enableAutoScaling: false
-      }
-    ]
+    agentPoolProfiles: agentPools
     servicePrincipalProfile: {
       clientId: 'msi'
     }

diff --git a/tools/stress-cluster/cluster/azure/cluster/storage.bicep b/tools/stress-cluster/cluster/azure/cluster/storage.bicep
@@ -0,0 +1,21 @@
+param location string = resourceGroup().location
+param storageName string
+param fileShareName string
+
+resource storage 'Microsoft.Storage/storageAccounts@2019-06-01' = {
+  name: storageName
+  location: location
+  kind: 'StorageV2'
+  sku: {
+    name: 'Standard_LRS'
+  }
+}
+
+resource fileshare 'Microsoft.Storage/storageAccounts/fileServices/shares@2021-04-01' = {
+      name: '${storage.name}/default/${fileShareName}'
+      properties: { }
+}
+
+output name string = storage.name
+output key string = storage.listKeys().keys[0].value
+output fileShareName string = fileShareName
diff --git a/tools/stress-cluster/cluster/azure/main.bicep b/tools/stress-cluster/cluster/azure/main.bicep
@@ -1,5 +1,6 @@
 targetScope = 'subscription'
 
+param subscriptionId string = ''
 param groupSuffix string
 param clusterName string
 param clusterLocation string = 'westus2'
@@ -8,15 +9,19 @@ param staticTestSecretsKeyvaultGroup string
 param monitoringLocation string = 'centralus'
 param tags object
 param enableMonitoring bool = false
+param enableHighMemAgentPool bool = false
+param enableDebugStorage bool = false
 
 // Azure Developer Platform Team Group
 // https://ms.portal.azure.com/#blade/Microsoft_AAD_IAM/GroupDetailsMenuBlade/Overview/groupId/56709ad9-8962-418a-ad0d-4b25fa962bae
 param accessGroups array = [
     '56709ad9-8962-418a-ad0d-4b25fa962bae'
 ]
 
+var groupName = 'rg-stress-cluster-${groupSuffix}'
+
 resource group 'Microsoft.Resources/resourceGroups@2020-10-01' = {
-    name: 'rg-stress-test-cluster-${groupSuffix}'
+    name: groupName
     location: clusterLocation
     tags: tags
 }
@@ -52,6 +57,7 @@ module cluster 'cluster/cluster.bicep' = {
         tags: tags
         groupSuffix: groupSuffix
         enableMonitoring: enableMonitoring
+        enableHighMemAgentPool: enableHighMemAgentPool
         workspaceId: enableMonitoring ? logWorkspace.outputs.id : ''
     }
 }
@@ -60,15 +66,34 @@ module containerRegistry 'cluster/acr.bicep' = {
     name: 'containerRegistry'
     scope: group
     params: {
-        registryName: '${replace(clusterName, '-', '')}registry'
+        registryName: '${replace(clusterName, '-', '')}${resourceSuffix}'
         location: clusterLocation
         objectIds: concat(accessGroups, array(cluster.outputs.kubeletIdentityObjectId))
     }
 }
 
+module storage 'cluster/storage.bicep' = if (enableDebugStorage) {
+    name: 'storage'
+    scope: group
+    params: {
+        storageName: 'stressdebug${resourceSuffix}'
+        fileShareName: 'stressfiles${resourceSuffix}'
+        location: clusterLocation
+    }
+}
+
 var appInsightsInstrumentationKeySecretName = 'appInsightsInstrumentationKey-${resourceSuffix}'
+// Value is in dotenv format as it will be appended to stress test container dotenv files
 var appInsightsInstrumentationKeySecretValue = 'APPINSIGHTS_INSTRUMENTATIONKEY=${appInsights.outputs.instrumentationKey}\n'
 
+// Storage account information used for kubernetes fileshare volume mounting via the azure files csi driver
+// See https://docs.microsoft.com/en-us/azure/aks/azure-files-volume#create-a-kubernetes-secret
+// See https://docs.microsoft.com/en-us/azure/aks/azure-files-csi
+var debugStorageKeySecretName = 'debugStorageKey-${resourceSuffix}'
+var debugStorageKeySecretValue = '${storage.outputs.key}'
+var debugStorageAccountSecretName = 'debugStorageAccount-${resourceSuffix}'
+var debugStorageAccountSecretValue = '${storage.outputs.name}'
+
 module keyvault 'cluster/keyvault.bicep' = if (enableMonitoring) {
     name: 'keyvault'
     scope: group
@@ -83,6 +108,14 @@ module keyvault 'cluster/keyvault.bicep' = if (enableMonitoring) {
                     secretName: appInsightsInstrumentationKeySecretName
                     secretValue: appInsightsInstrumentationKeySecretValue
                 }
+                {
+                    secretName: debugStorageKeySecretName
+                    secretValue: debugStorageKeySecretValue
+                }
+                {
+                    secretName: debugStorageAccountSecretName
+                    secretValue: debugStorageAccountSecretValue
+                }
             ]
         }
     }
@@ -99,10 +132,14 @@ module accessPolicy 'cluster/static-vault-access-policy.bicep' = {
 }
 
 output STATIC_TEST_SECRETS_KEYVAULT string = staticTestSecretsKeyvaultName
-output CLUSTER_KEYVAULT string = keyvault.outputs.keyvaultName
+output CLUSTER_TEST_SECRETS_KEYVAULT string = keyvault.outputs.keyvaultName
 output SECRET_PROVIDER_CLIENT_ID string = cluster.outputs.secretProviderClientId
 output CLUSTER_NAME string = cluster.outputs.clusterName
 output CONTAINER_REGISTRY_NAME string = containerRegistry.outputs.containerRegistryName
 output APPINSIGHTS_KEY_SECRET_NAME string = appInsightsInstrumentationKeySecretName
+output DEBUG_STORAGE_KEY_SECRET_NAME string = debugStorageKeySecretName
+output DEBUG_STORAGE_ACCOUNT_SECRET_NAME string = debugStorageAccountSecretName
+output DEBUG_FILESHARE_NAME string = storage.outputs.fileShareName
 output RESOURCE_GROUP string = group.name
+output SUBSCRIPTION_ID string = subscriptionId
 output TENANT_ID string = subscription().tenantId
diff --git a/tools/stress-cluster/cluster/azure/parameters/dev.json b/tools/stress-cluster/cluster/azure/parameters/dev.json
@@ -2,7 +2,10 @@
   "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
   "contentVersion": "1.0.0.0",
   "parameters": {
-    "groupName": {
+    "subscriptionId": {
+      "value": // add me
+    },
+    "groupSuffix": {
       "value": // add me
     },
     "clusterName": {
@@ -12,10 +15,13 @@
       "value": "westus2"
     },
     "staticTestSecretsKeyvaultName": {
-      "value": "StressTestSecrets"
+      "value": "stress-secrets-dev"
     },
     "staticTestSecretsKeyvaultGroup": {
-      "value": "rg-StressTestSecrets"
+      "value": "rg-stress-secrets-dev"
+    },
+    "enableDebugStorage": {
+        "value": true
     },
     "tags": {
       "value": {

diff --git a/tools/stress-cluster/cluster/azure/parameters/prod.json b/tools/stress-cluster/cluster/azure/parameters/prod.json
@@ -2,6 +2,9 @@
   "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
   "contentVersion": "1.0.0.0",
   "parameters": {
+    "subscriptionId": {
+      "value": "2cd617ea-1866-46b1-90e3-fffb087ebf9b"
+    },
     "groupSuffix": {
       "value": "prod"
     },
@@ -15,10 +18,10 @@
       "value": "centralus"
     },
     "staticTestSecretsKeyvaultName": {
-      "value": "StressTestSecrets"
+      "value": "stress-secrets-prod"
     },
     "staticTestSecretsKeyvaultGroup": {
-      "value": "rg-StressTestSecrets"
+      "value": "rg-stress-secrets-prod"
     },
     "enableMonitoring": {
         "value": true

diff --git a/tools/stress-cluster/cluster/azure/parameters/test.json b/tools/stress-cluster/cluster/azure/parameters/test.json
@@ -2,8 +2,11 @@
   "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
   "contentVersion": "1.0.0.0",
   "parameters": {
+    "subscriptionId": {
+      "value": "faa080af-c1d8-40ad-9cce-e1a450ca5b57"
+    },
     "groupSuffix": {
-      "value": ""
+      "value": "test"
     },
     "clusterName": {
       "value": "stress-test"
@@ -12,14 +15,20 @@
       "value": "westus2"
     },
     "staticTestSecretsKeyvaultName": {
-      "value": "StressTestSecrets"
+      "value": "stress-secrets-test"
     },
     "staticTestSecretsKeyvaultGroup": {
-      "value": "rg-StressTestSecrets"
+      "value": "rg-stress-secrets-test"
     },
     "enableMonitoring": {
         "value": true
     },
+    "enableDebugStorage": {
+        "value": true
+    },
+    "enableHighMemAgentPool": {
+        "value": true
+    },
     "tags": {
       "value": {
         "environment": "Test",

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_container_env.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_container_env.tpl
@@ -2,7 +2,11 @@
 env:
   - name: ENV_FILE
     value: /mnt/outputs/.env
+  - name: DEBUG_SHARE
+    value: /mnt/share/
 volumeMounts:
   - name: test-env-{{ lower .Scenario }}-{{ .Release.Name }}-{{ .Release.Revision }}
     mountPath: /mnt/outputs
+  - name: debug-file-share-{{ .Release.Name }}
+    mountPath: /mnt/share
 {{- end -}}
diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_deploy_configmap.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_deploy_configmap.tpl
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: "{{ .Release.Name }}-test-resources"
+  name: "{{ .Release.Name }}-{{ .Release.Revision }}-test-resources"
   namespace: {{ .Release.Namespace }}
 data:
   template: |

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_deploy_volumes.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_deploy_volumes.tpl
@@ -1,7 +1,7 @@
 {{ define "stress-test-addons.deploy-volumes" }}
-- name: {{ .Release.Name }}-test-resources
+- name: {{ .Release.Name }}-{{ .Release.Revision }}-test-resources
   configMap:
-    name: "{{ .Release.Name }}-test-resources"
+    name: "{{ .Release.Name }}-{{ .Release.Revision }}-test-resources"
     items:
       - key: template
         path: test-resources.json

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_deploy.tpl
@@ -1,5 +1,5 @@
 {{ define "stress-test-addons.init-deploy" }}
-- name: azure-deployer
+- name: init-azure-deployer
   image: mcr.microsoft.com/azure-cli
   command: ['bash', '-c']
   args:

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_env.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_init_env.tpl
@@ -1,9 +1,10 @@
 {{ define "stress-test-addons.init-env" }}
-- name: test-env-initializer
+- name: init-test-env
   image: k8s.gcr.io/e2e-test-images/busybox:1.29
   command: ['sh', '-c']
   args:
-    # Merge all mounted keyvault secrets into env file
+    # Merge all mounted keyvault secrets into env file.
+    # Secret values are expected to be in format <key>=<value>
     - 'cat /mnt/secrets/static/* /mnt/secrets/cluster/* > $ENV_FILE'
   env:
     - name: ENV_FILE