diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index bc028f26aa1..8abaa40d0cb 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -1,16 +1,14 @@ -# Set a default parameter set here so we can call this script without requiring -Login and -Subscription, -# but if it IS called with either of those, then both parameters need to be required. Not defining a -# default parameter set makes Login/Subscription required all the time. +# Not defining a default parameter set makes SkipLogin/Subscription required all the time. [CmdletBinding(DefaultParameterSetName = 'Default')] param( [string]$SearchDirectory, [hashtable]$Filters, [string]$Environment, [string]$Repository, - [switch]$PushImages, + [switch]$SkipPushImages, [string]$ClusterGroup, [string]$DeployId, - [switch]$Login, + [switch]$SkipLogin, [string]$Subscription, # Default to true in Azure Pipelines environments diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 5ad163c6a84..f8763de3502 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -43,7 +43,7 @@ function RunOrExitOnFailure() } } -function Login([string]$subscription, [string]$clusterGroup, [switch]$pushImages) +function Login([string]$subscription, [string]$clusterGroup, [switch]$skipPushImages) { Write-Host "Logging in to subscription, cluster and container registry" az account show *> $null @@ -73,7 +73,7 @@ function Login([string]$subscription, [string]$clusterGroup, [switch]$pushImages RunOrExitOnFailure kubectl config set-context $clusterName --namespace $defaultNamespace } - if ($pushImages) { + if (!$skipPushImages) { $registry = RunOrExitOnFailure az acr list -g $clusterGroup --subscription $subscription -o json $registryName = ($registry | ConvertFrom-Json).name RunOrExitOnFailure az acr login -n $registryName @@ -86,10 +86,10 @@ function DeployStressTests( # Default to playground environment [string]$environment = 'pg', [string]$repository = '', - [switch]$pushImages, + [switch]$skipPushImages, [string]$clusterGroup = '', [string]$deployId = '', - [switch]$login, + [switch]$skipLogin, [string]$subscription = '', [switch]$CI, [string]$Namespace, @@ -125,8 +125,8 @@ function DeployStressTests( throw "clusterGroup and subscription parameters must be specified when deploying to an environment that is not pg or prod." } - if ($login) { - Login -subscription $subscription -clusterGroup $clusterGroup -pushImages:$pushImages + if (!$skipLogin) { + Login -subscription $subscription -clusterGroup $clusterGroup -skipPushImages:$skipPushImages } $chartRepoName = 'stress-test-charts' @@ -162,8 +162,8 @@ function DeployStressTests( -deployId $deployer ` -environment $environment ` -repositoryBase $repository ` - -pushImages:$pushImages ` - -login:$login ` + -skipPushImages:$skipPushImages ` + -skipLogin:$skipLogin ` -clusterGroup $clusterGroup ` -subscription $subscription } @@ -189,8 +189,8 @@ function DeployStressPackage( [string]$deployId, [string]$environment, [string]$repositoryBase, - [switch]$pushImages, - [switch]$login, + [switch]$skipPushImages, + [switch]$skipLogin, [string]$clusterGroup, [string]$subscription ) { @@ -267,7 +267,7 @@ function DeployStressPackage( } $dockerfileName = ($dockerFilePath -split { $_ -in '\', '/' })[-1].ToLower() $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}" - if ($pushImages) { + if (!$skipPushImages) { Write-Host "Building and pushing stress test docker image '$imageTag'" $dockerFile = Get-ChildItem $dockerFilePath @@ -290,8 +290,8 @@ function DeployStressPackage( Run docker push $imageTag if ($LASTEXITCODE) { - if ($login) { - Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + if (!$skipLogin) { + Write-Warning "If docker push is failing due to authentication issues, try calling this script without '-SkipLogin'" } } } diff --git a/eng/pipelines/templates/jobs/stress-test-release.yml b/eng/pipelines/templates/jobs/stress-test-release.yml index 84e9e3e8339..e97de6d7249 100644 --- a/eng/pipelines/templates/jobs/stress-test-release.yml +++ b/eng/pipelines/templates/jobs/stress-test-release.yml @@ -74,7 +74,5 @@ jobs: -Filters $(Filters) -Environment '${{ parameters.Environment }}' -Repository '$(Agent.JobName)' - -PushImages - -Login -DeployId '$(Build.BuildNumber)' -CI diff --git a/tools/stress-cluster/chaos/README.md b/tools/stress-cluster/chaos/README.md index c73547f395e..b277d40cf83 100644 --- a/tools/stress-cluster/chaos/README.md +++ b/tools/stress-cluster/chaos/README.md @@ -52,17 +52,12 @@ package dependencies, and building and pushing docker images. The script must be If using bash or another linux terminal, a [powershell core](https://docs.microsoft.com/powershell/scripting/install/installing-powershell-core-on-linux?view=powershell-7.1) shell can be invoked via `pwsh`. -The first invocation of the script must be run with the `-Login` flag to set up cluster and container registry access. - ``` cd - -/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 ` - -Login ` - -PushImages +/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 ``` -To re-deploy more quickly, the script can be run without `-Login` and/or without `-PushImages` (if no code changes were +To re-deploy more quickly, the script can be run with `-SkipLogin` and/or with `-SkipPushImages` (if no code changes were made). ``` @@ -622,7 +617,7 @@ Follow the below commands to execute a sample test. ``` cd ./examples/network_stress_example -pwsh ../../../../../eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -Login -PushImages +pwsh ../../../../../eng/common/scripts/stress-testing/deploy-stress-tests.ps1 ``` Verify the pods in the job have booted and are running ok (with chaos network failures): diff --git a/tools/stress-cluster/cluster/README.md b/tools/stress-cluster/cluster/README.md index a51fbadb832..4f63e7d643f 100644 --- a/tools/stress-cluster/cluster/README.md +++ b/tools/stress-cluster/cluster/README.md @@ -78,8 +78,7 @@ resource values from the newly provisioned dev environment that are required by Avoid checking in the updated dev values, they are for local use only. ``` -# -Login only needs to be run once or if the azure container registry credentials have expired (~24 hours) -/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -Login -Environment dev +/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -Environment dev ``` ## Playground Cluster @@ -122,7 +121,7 @@ Steps for deploying the stress test addons helm chart: 1. Increment the version number in stress test addons' [Chart.yaml](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml) (e.g. 0.1.0 -> 0.1.1). 1. Run [deploy.ps1](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/cluster/kubernetes/stress-test-addons/deploy.ps1). 1. Update all the helm chart versions for stress-test-addons dependency references in `azure-sdk-tools/tools/stress-cluster/chaos/examples/**/Chart.yaml`. -1. Run azure-sdk-tools\eng\common\scripts\stress-testing\deploy-stress-tests.ps1 script in the [examples](https://github.com/Azure/azure-sdk-tools/tree/main/tools/stress-cluster/chaos/examples) directory, this will update all the nested helm charts (-login tag is needed for the first run). +1. Run azure-sdk-tools\eng\common\scripts\stress-testing\deploy-stress-tests.ps1 script in the [examples](https://github.com/Azure/azure-sdk-tools/tree/main/tools/stress-cluster/chaos/examples) directory, this will update all the nested helm charts (the -SkipLogin parameter can be used to speed up the script or if interactive login isn't supported by the shell). 1. Run `kubectl get pods -n examples -w` to monitor the status of each pod and look for Running/Completed and make sure there are no errors. 1. Update all the stress tests' Chart.yaml files across the other repos in the same manner. @@ -166,8 +165,7 @@ may still be an invalid kubernetes manifest, so the example stress test should a the full set of changes: ``` -# -Login only needs to be run once or if the azure container registry credentials have expired (~24 hours) -/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -Login +/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 ``` For more helm debugging info, see [here](https://helm.sh/docs/chart_template_guide/debugging/). diff --git a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep index 54b227135b4..a13edc5fd2a 100644 --- a/tools/stress-cluster/cluster/azure/cluster/cluster.bicep +++ b/tools/stress-cluster/cluster/azure/cluster/cluster.bicep @@ -4,6 +4,8 @@ param groupSuffix string param dnsPrefix string = 's1' param clusterName string param location string = resourceGroup().location +param defaultAgentPoolMinNodes int = 6 +param defaultAgentPoolMaxNodes int = 20 // AKS does not allow agentPool updates via existing managed cluster resources param updateNodes bool = false @@ -31,9 +33,9 @@ var systemAgentPool = { var defaultAgentPool = { name: 'default' - count: 6 - minCount: 6 - maxCount: 20 + count: defaultAgentPoolMinNodes + minCount: defaultAgentPoolMinNodes + maxCount: defaultAgentPoolMaxNodes mode: 'User' vmSize: 'Standard_D8a_v4' type: 'VirtualMachineScaleSets' diff --git a/tools/stress-cluster/cluster/azure/main.bicep b/tools/stress-cluster/cluster/azure/main.bicep index 878569287c9..a35840a96cd 100644 --- a/tools/stress-cluster/cluster/azure/main.bicep +++ b/tools/stress-cluster/cluster/azure/main.bicep @@ -7,6 +7,8 @@ param clusterLocation string = 'westus3' param staticTestKeyvaultName string param staticTestKeyvaultGroup string param monitoringLocation string = 'centralus' +param defaultAgentPoolMinNodes int = 6 +param defaultAgentPoolMaxNodes int = 20 param tags object // AKS does not allow agentPool updates via existing managed cluster resources param updateNodes bool = false @@ -75,6 +77,8 @@ module cluster 'cluster/cluster.bicep' = { updateNodes: updateNodes location: clusterLocation clusterName: clusterName + defaultAgentPoolMinNodes: defaultAgentPoolMinNodes + defaultAgentPoolMaxNodes: defaultAgentPoolMaxNodes tags: tags groupSuffix: groupSuffix workspaceId: logWorkspace.outputs.id diff --git a/tools/stress-cluster/cluster/azure/parameters/pg.json b/tools/stress-cluster/cluster/azure/parameters/pg.json index e1ce19e6e4d..a50b9843267 100644 --- a/tools/stress-cluster/cluster/azure/parameters/pg.json +++ b/tools/stress-cluster/cluster/azure/parameters/pg.json @@ -20,6 +20,12 @@ "staticTestKeyvaultGroup": { "value": "rg-stress-secrets-pg" }, + "defaultAgentPoolMinNodes": { + "value": 2 + }, + "defaultAgentPoolMaxNodes": { + "value": 10 + }, "tags": { "value": { "environment": "pg", diff --git a/tools/stress-cluster/cluster/azure/parameters/prod.json b/tools/stress-cluster/cluster/azure/parameters/prod.json index 6800e78c30f..1b89cdd2ad7 100644 --- a/tools/stress-cluster/cluster/azure/parameters/prod.json +++ b/tools/stress-cluster/cluster/azure/parameters/prod.json @@ -23,6 +23,12 @@ "staticTestKeyvaultGroup": { "value": "rg-stress-secrets-prod" }, + "defaultAgentPoolMinNodes": { + "value": 2 + }, + "defaultAgentPoolMaxNodes": { + "value": 10 + }, "tags": { "value": { "environment": "Prod", diff --git a/tools/stress-cluster/cluster/kubernetes/generator/Stress.Generator/Program.cs b/tools/stress-cluster/cluster/kubernetes/generator/Stress.Generator/Program.cs index 42f04531bd0..dc2f26cc529 100644 --- a/tools/stress-cluster/cluster/kubernetes/generator/Stress.Generator/Program.cs +++ b/tools/stress-cluster/cluster/kubernetes/generator/Stress.Generator/Program.cs @@ -44,7 +44,7 @@ public static void Main(string[] args) Console.WriteLine($"2. Add test code to ./src/"); Console.WriteLine($"3. Update 'Dockerfile' (see contents for help)."); Console.WriteLine($"4. Run the following command from within your language repository to deploy the package:"); - Console.WriteLine($" pwsh -c $(git rev-parse --show-toplevel)/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -Login -PushImages"); + Console.WriteLine($" pwsh -c $(git rev-parse --show-toplevel)/eng/common/scripts/stress-testing/deploy-stress-tests.ps1"); Console.WriteLine($"********************************************************************************"); }); }