Migrate stress cluster to use workload identity (#8278)

* Migrate stress cluster to use workload identity * Update stress watcher tests and chart lock files * Only sleep to sync fed creds when namespace is new * Skip stress provision env outputs in what if mode
Azure · May 23, 2024 · 69fcb1d · 69fcb1d
1 parent 4d84241
commit 69fcb1d
Show file tree

Hide file tree

Showing 41 changed files with 719 additions and 275 deletions.
diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1
@@ -213,9 +213,17 @@ function DeployStressPackage(
     $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)"
 
     if (!$Template) {
-        Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..."
-        kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
-        if ($LASTEXITCODE) {exit $LASTEXITCODE}
+        Write-Host "Checking for namespace $($pkg.Namespace)"
+        kubectl get namespace $pkg.Namespace
+        if ($LASTEXITCODE) {
+            Write-Host "Creating namespace $($pkg.Namespace) ..."
+            kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f -
+            if ($LASTEXITCODE) {exit $LASTEXITCODE}
+            # Give a few seconds for stress watcher to initialize the federated identity credential
+            # and create the service account before we reference it
+            Write-Host "Waiting 15 seconds for namespace federated credentials to be created and synced"
+            Start-Sleep 15
+        }
         Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)"
         $limitRangeSpec | kubectl apply -n $pkg.Namespace -f -
         if ($LASTEXITCODE) {exit $LASTEXITCODE}

diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.1
-digest: sha256:28e374f8db5c46447b2a1491d4361ceb126536c425cbe54be49017120fe7b27d
-generated: "2024-01-17T15:38:17.871619598-05:00"
+  version: 0.3.2
+digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
+generated: "2024-05-23T11:38:32.810490735-04:00"
diff --git a/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock b/tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.1
-digest: sha256:28e374f8db5c46447b2a1491d4361ceb126536c425cbe54be49017120fe7b27d
-generated: "2024-01-17T15:39:38.364921715-05:00"
+  version: 0.3.2
+digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
+generated: "2024-05-23T11:38:01.807752664-04:00"
diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock b/tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.1
-digest: sha256:28e374f8db5c46447b2a1491d4361ceb126536c425cbe54be49017120fe7b27d
-generated: "2024-01-17T15:40:00.504665427-05:00"
+  version: 0.3.2
+digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
+generated: "2024-05-23T11:38:47.628996062-04:00"
diff --git a/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml b/tools/stress-cluster/chaos/examples/parallel-pod-example/templates/parallel-pod.yaml
@@ -11,7 +11,7 @@ spec:
       args:
         - |
             source $ENV_FILE &&
-            az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID &&
+            az login --federated-token "$(cat $AZURE_FEDERATED_TOKEN_FILE)" --service-principal -u "$AZURE_CLIENT_ID" -t "$AZURE_TENANT_ID" &&
             az appconfig show -n $APP_CONFIG_NAME -g $RESOURCE_GROUP --subscription $AZURE_SUBSCRIPTION_ID -o table &&
             echo "Completed pod instance $JOB_COMPLETION_INDEX"
       {{- include "stress-test-addons.container-env" . | nindent 6 }}

diff --git a/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.1
-digest: sha256:28e374f8db5c46447b2a1491d4361ceb126536c425cbe54be49017120fe7b27d
-generated: "2024-01-17T15:39:47.856708817-05:00"
+  version: 0.3.2
+digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
+generated: "2024-05-23T11:38:19.251210631-04:00"
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock b/tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock
@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.1
-digest: sha256:28e374f8db5c46447b2a1491d4361ceb126536c425cbe54be49017120fe7b27d
-generated: "2024-01-17T15:39:23.757382734-05:00"
+  version: 0.3.2
+digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
+generated: "2024-05-23T11:37:41.371010465-04:00"
diff --git a/tools/stress-cluster/chaos/examples/stress-deployment-example/templates/deploy-job.yaml b/tools/stress-cluster/chaos/examples/stress-deployment-example/templates/deploy-job.yaml
@@ -13,7 +13,7 @@ spec:
       args:
         - |
             source $ENV_FILE &&
-            az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID &&
+            az login --federated-token "$(cat $AZURE_FEDERATED_TOKEN_FILE)" --service-principal -u "$AZURE_CLIENT_ID" -t "$AZURE_TENANT_ID" &&
             az account set -s $AZURE_SUBSCRIPTION_ID &&
             az appconfig show -n $APP_CONFIG_NAME -g $RESOURCE_GROUP -o json
       {{- include "stress-test-addons.container-env" . | nindent 6 }}

diff --git a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep
@@ -24,7 +24,7 @@ var systemAgentPool = {
   mode: 'System'
   vmSize: 'Standard_D4ds_v4'
   type: 'VirtualMachineScaleSets'
-  osType: 'AzureLinux'
+  osType: 'Linux'
   enableAutoScaling: true
   enableEncryptionAtHost: true
   nodeLabels: {
@@ -40,7 +40,7 @@ var defaultAgentPool = {
   mode: 'User'
   vmSize: 'Standard_D8a_v4'
   type: 'VirtualMachineScaleSets'
-  osType: 'AzureLinux'
+  osType: 'Linux'
   osDiskType: 'Ephemeral'
   enableAutoScaling: true
   enableEncryptionAtHost: true
@@ -87,6 +87,14 @@ resource newCluster 'Microsoft.ContainerService/managedClusters@2023-02-02-previ
     servicePrincipalProfile: {
       clientId: 'msi'
     }
+    oidcIssuerProfile: {
+      enabled: true
+    }
+    securityProfile: {
+      workloadIdentity: {
+        enabled: true
+      }
+    }
     nodeResourceGroup: nodeResourceGroup
   }
 }
@@ -151,4 +159,5 @@ resource metricsPublisher 'Microsoft.Authorization/roleAssignments@2020-04-01-pr
 output secretProviderObjectId string = cluster.properties.addonProfiles.azureKeyvaultSecretsProvider.identity.objectId
 output secretProviderClientId string = cluster.properties.addonProfiles.azureKeyvaultSecretsProvider.identity.clientId
 output kubeletIdentityObjectId string = cluster.properties.identityProfile.kubeletidentity.objectId
+output workloadAppIssuer string = cluster.properties.oidcIssuerProfile.issuerURL
 output clusterName string = clusterName
diff --git a/tools/stress-cluster/cluster/azure/cluster/static-vault-access-policy.bicep b/tools/stress-cluster/cluster/azure/cluster/static-vault-access-policy.bicep
diff --git a/tools/stress-cluster/cluster/azure/cluster/workloadappidentities.bicep b/tools/stress-cluster/cluster/azure/cluster/workloadappidentities.bicep
@@ -0,0 +1,35 @@
+param groupSuffix string
+param location string
+
+param infraNamespace string
+param infraWorkloadServiceAccountName string
+param workloadAppIssuer string
+param workloadAppPoolCount int
+
+resource infraWorkloadApp 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = {
+  name: 'stress-infra-workload-${groupSuffix}'
+  location: location
+
+  resource creds 'federatedIdentityCredentials' = {
+    name: 'stress-infra-federated-${groupSuffix}'
+    properties: {
+      issuer: workloadAppIssuer
+      audiences: ['api://AzureADTokenExchange']
+      subject: 'system:serviceaccount:${infraNamespace}:${infraWorkloadServiceAccountName}'
+    }
+  }
+}
+
+resource workloadApps 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = [for i in range(0, workloadAppPoolCount): {
+  name: 'stress-app-workload-${groupSuffix}-${i}'
+  location: location
+}]
+
+output infraWorkloadAppClientId string = infraWorkloadApp.properties.clientId
+output infraWorkloadAppObjectId string = infraWorkloadApp.properties.principalId
+
+output workloadAppInfo array = [for i in range(0, workloadAppPoolCount): {
+  name: 'stress-app-workload-${groupSuffix}-${i}'
+  clientId: workloadApps[i].properties.clientId
+  objectId: workloadApps[i].properties.principalId
+}]
diff --git a/tools/stress-cluster/cluster/azure/cluster/workloadapproles.bicep b/tools/stress-cluster/cluster/azure/cluster/workloadapproles.bicep
@@ -0,0 +1,54 @@
+targetScope = 'subscription'
+
+param infraWorkloadAppObjectId string
+param workloadApps array
+
+var serviceBusDataOwnerRoleId = '090c5cfd-751d-490a-894a-3ce6f1109419'
+var eventHubsDataOwnerRoleId = 'f526a384-b230-433a-b45c-95f59c4a2dec'
+var contributorRoleId = 'b24988ac-6180-42a0-ab88-20f7382dd24c'
+var userAccessAdministratorRoleId = '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9'
+
+resource infraWorkloadAppContrib 'Microsoft.Authorization/roleAssignments@2021-04-01-preview' = {
+  name: guid('infraWorkloadAppContrib', subscription().id, infraWorkloadAppObjectId)
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', contributorRoleId)
+    principalId: infraWorkloadAppObjectId
+    principalType: 'ServicePrincipal'
+  }
+}
+
+resource infraWorkloadAppUA 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = {
+  name: guid('infraWorkloadAppUA', subscription().id, infraWorkloadAppObjectId)
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', userAccessAdministratorRoleId)
+    principalId: infraWorkloadAppObjectId
+    principalType: 'ServicePrincipal'
+  }
+}
+
+resource workloadAppContrib 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for i in range(0, length(workloadApps)): {
+  name: guid('workloadAppContrib', subscription().id, workloadApps[i].objectId)
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', contributorRoleId)
+    principalId: workloadApps[i].objectId
+    principalType: 'ServicePrincipal'
+  }
+}]
+
+resource workloadAppEHDataOwner 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for i in range(0, length(workloadApps)): {
+  name: guid('workloadAppEHDataOwner', subscription().id, workloadApps[i].objectId)
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', eventHubsDataOwnerRoleId)
+    principalId: workloadApps[i].objectId
+    principalType: 'ServicePrincipal'
+  }
+}]
+
+resource workloadAppSBDataOwner 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for i in range(0, length(workloadApps)): {
+  name: guid('workloadAppSBDataOwner', subscription().id, workloadApps[i].objectId)
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', serviceBusDataOwnerRoleId)
+    principalId: workloadApps[i].objectId
+    principalType: 'ServicePrincipal'
+  }
+}]
diff --git a/tools/stress-cluster/cluster/azure/main.bicep b/tools/stress-cluster/cluster/azure/main.bicep
@@ -3,9 +3,8 @@ targetScope = 'subscription'
 param subscriptionId string = ''
 param groupSuffix string
 param clusterName string
+param infraNamespace string = 'stress-infra'
 param clusterLocation string = 'westus3'
-param staticTestKeyvaultName string
-param staticTestKeyvaultGroup string
 param monitoringLocation string = 'centralus'
 param defaultAgentPoolMinNodes int = 6
 param defaultAgentPoolMaxNodes int = 20
@@ -14,6 +13,8 @@ param tags object
 // AKS does not allow agentPool updates via existing managed cluster resources
 param updateNodes bool = false
 
+var workloadAppPoolCount = 5
+
 // Azure Developer Platform Team Group
 // https://ms.portal.azure.com/#blade/Microsoft_AAD_IAM/GroupDetailsMenuBlade/Overview/groupId/56709ad9-8962-418a-ad0d-4b25fa962bae
 param accessGroups array = [
@@ -154,17 +155,28 @@ module keyvault 'cluster/keyvault.bicep' = {
     }
 }
 
-module accessPolicy 'cluster/static-vault-access-policy.bicep' = {
-    name: 'accessPolicy'
-    scope: resourceGroup(staticTestKeyvaultGroup)
-    params: {
-        vaultName: staticTestKeyvaultName
-        tenantId: subscription().tenantId
-        objectId: cluster.outputs.secretProviderObjectId
-    }
+module workloadAppIdentities 'cluster/workloadappidentities.bicep' = if (!updateNodes) {
+  name: 'workloadAppIdentities'
+  scope: group
+  params: {
+    groupSuffix: groupSuffix
+    location: clusterLocation
+    infraNamespace: infraNamespace
+    infraWorkloadServiceAccountName: 'workload-svc'
+    workloadAppIssuer: cluster.outputs.workloadAppIssuer
+    workloadAppPoolCount: workloadAppPoolCount
+  }
+}
+
+module workloadAppRoles 'cluster/workloadapproles.bicep' = if (!updateNodes) {
+  name: 'workloadAppRoles'
+  scope: subscription()
+  params: {
+    infraWorkloadAppObjectId: workloadAppIdentities.outputs.infraWorkloadAppObjectId
+    workloadApps: workloadAppIdentities.outputs.workloadAppInfo
+  }
 }
 
-output STATIC_TEST_SECRETS_KEYVAULT string = staticTestKeyvaultName
 output CLUSTER_TEST_SECRETS_KEYVAULT string = keyvault.outputs.keyvaultName
 output SECRET_PROVIDER_CLIENT_ID string = cluster.outputs.secretProviderClientId
 output CLUSTER_NAME string = cluster.outputs.clusterName
@@ -181,3 +193,8 @@ output STATUS_DASHBOARD_LINK string = 'https://ms.portal.azure.com/#@microsoft.o
 output RESOURCE_GROUP string = group.name
 output SUBSCRIPTION_ID string = subscriptionId
 output TENANT_ID string = subscription().tenantId
+output INFRA_WORKLOAD_APP_SERVICE_ACCOUNT_NAME string = 'workload-svc'
+output INFRA_WORKLOAD_APP_CLIENT_ID string = workloadAppIdentities.outputs.infraWorkloadAppClientId
+output INFRA_WORKLOAD_APP_OBJECT_ID string = workloadAppIdentities.outputs.infraWorkloadAppObjectId
+output WORKLOAD_APP_ISSUER string = cluster.outputs.workloadAppIssuer
+output WORKLOAD_APPS string = string(workloadAppIdentities.outputs.workloadAppInfo)
diff --git a/tools/stress-cluster/cluster/azure/parameters/pg.json b/tools/stress-cluster/cluster/azure/parameters/pg.json
@@ -14,12 +14,6 @@
     "clusterLocation": {
       "value": "westus3"
     },
-    "staticTestKeyvaultName": {
-      "value": "stress-secrets-pg"
-    },
-    "staticTestKeyvaultGroup": {
-      "value": "rg-stress-secrets-pg"
-    },
     "defaultAgentPoolMinNodes": {
       "value": 6
     },

diff --git a/tools/stress-cluster/cluster/azure/parameters/prod.json b/tools/stress-cluster/cluster/azure/parameters/prod.json
@@ -17,12 +17,6 @@
     "monitoringLocation": {
       "value": "centralus"
     },
-    "staticTestKeyvaultName": {
-      "value": "stress-secrets-prod"
-    },
-    "staticTestKeyvaultGroup": {
-      "value": "rg-stress-secrets-prod"
-    },
     "defaultAgentPoolMinNodes": {
       "value": 2
     },

diff --git a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock
@@ -4,6 +4,6 @@ dependencies:
   version: 2.6.3
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.1
-digest: sha256:b954ab9bea8f484f3110a2051e384621f6cfb0188cccd3911299da3aad6fd951
-generated: "2024-05-08T14:47:49.4174026-07:00"
+  version: 0.3.2
+digest: sha256:59235c0eac423267e28d9ac61392532ea74fb37a1be2567e4ac83277d62d8761
+generated: "2024-05-23T11:44:39.658622055-04:00"
diff --git a/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/templates/infra-svc.yaml b/tools/stress-cluster/cluster/kubernetes/stress-infrastructure/templates/infra-svc.yaml
@@ -0,0 +1,8 @@
+{{- $addons := get .Values "stress-test-addons" -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations:
+    azure.workload.identity/client-id: {{ get $addons.infraWorkloadAppClientId $addons.env }}
+  name: {{ get $addons.infraWorkloadAppServiceAccountName $addons.env }}
+  namespace: {{ .Release.Namespace }}
diff --git a/...luster/kubernetes/stress-infrastructure/templates/stresswatcher-cluster-role-binding.yaml b/...luster/kubernetes/stress-infrastructure/templates/stresswatcher-cluster-role-binding.yaml
@@ -1,3 +1,4 @@
+{{- $addons := get .Values "stress-test-addons" -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -6,6 +7,9 @@ subjects:
 - namespace: {{ .Release.Namespace }}
   kind: ServiceAccount
   name: default
+- namespace: {{ .Release.Namespace }}
+  kind: ServiceAccount
+  name: {{ get $addons.infraWorkloadAppServiceAccountName $addons.env }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

diff --git a/...luster/cluster/kubernetes/stress-infrastructure/templates/stresswatcher-cluster-role.yaml b/...luster/cluster/kubernetes/stress-infrastructure/templates/stresswatcher-cluster-role.yaml
@@ -11,5 +11,6 @@ rules:
   - get
   - list
   - watch
+  - create
   - update
   - patch
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,5 +11,6 @@ rules: @@
       - get
       - list
       - watch
+      - create
       - update
       - patch