diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 8abaa40d0cb4..61d8f947d800 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -31,7 +31,10 @@ param( [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, + + # Prevent kubernetes from deleting nodes or rebalancing pods related to this test for N days + [Parameter(Mandatory=$False)][ValidateRange(1, 14)][int]$LockDeletionForDays ) . $PSScriptRoot/stress-test-deployment-lib.ps1 diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index bdaec9e711a9..e956508c4392 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -59,7 +59,7 @@ function Login([string]$subscription, [string]$clusterGroup, [switch]$skipPushIm $kubeContext = (RunOrExitOnFailure kubectl config view -o json) | ConvertFrom-Json -AsHashtable $defaultNamespace = $null $targetContext = $kubeContext.contexts.Where({ $_.name -eq $clusterName }) | Select -First 1 - if ($targetContext -ne $null -and $targetContext.PSObject.Properties.Name -match "namespace") { + if ($targetContext -ne $null -and $targetContext.Contains('context') -and $targetContext.Contains('namespace')) { $defaultNamespace = $targetContext.context.namespace } @@ -107,7 +107,8 @@ function DeployStressTests( [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, + [Parameter(Mandatory=$False)][int]$LockDeletionForDays ) { if ($environment -eq 'pg') { if ($clusterGroup -or $subscription) { @@ -168,7 +169,7 @@ function DeployStressTests( -subscription $subscription } - if ($FailedCommands.Count -lt $pkgs.Count) { + if ($FailedCommands.Count -lt $pkgs.Count -and !$Template) { Write-Host "Releases deployed by $deployer" Run helm list --all-namespaces -l deployId=$deployer } @@ -211,12 +212,14 @@ function DeployStressPackage( } $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)" - Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." - kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} - Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)" - $limitRangeSpec | kubectl apply -n $pkg.Namespace -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} + if (!$Template) { + Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." + kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} + Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)" + $limitRangeSpec | kubectl apply -n $pkg.Namespace -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} + } $dockerBuildConfigs = @() @@ -317,8 +320,18 @@ function DeployStressPackage( $generatedConfigPath = Join-Path $pkg.Directory generatedValues.yaml $subCommand = $Template ? "template" : "upgrade" - $installFlag = $Template ? "" : "--install" - $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $installFlag, "--set", "stress-test-addons.env=$environment", "--values", $generatedConfigPath + $subCommandFlag = $Template ? "--debug" : "--install" + $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $subCommandFlag, "--values", $generatedConfigPath, "--set", "stress-test-addons.env=$environment" + + if ($LockDeletionForDays) { + $date = (Get-Date).AddDays($LockDeletionForDays).ToUniversalTime() + $isoDate = $date.ToString("o") + # Tell kubernetes job to run only on this specific future time. Technically it will run once per year. + $cron = "$($date.Minute) $($date.Hour) $($date.Day) $($date.Month) *" + + Write-Host "PodDisruptionBudget will be set to prevent deletion until $isoDate" + $helmCommandArg += "--set", "PodDisruptionBudgetExpiry=$($isoDate)", "--set", "PodDisruptionBudgetExpiryCron=$cron" + } $result = (Run @helmCommandArg) 2>&1 | Write-Host @@ -342,7 +355,7 @@ function DeployStressPackage( # Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around # specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources. # There is not currently support for setting these labels via the helm cli. - if(!$Template) { + if (!$Template) { $helmReleaseConfig = RunOrExitOnFailure kubectl get secrets ` -n $pkg.Namespace ` -l "status=deployed,name=$releaseName" `