From 99c5776c4c5f886d5d2bfb231daaba4079ace6be Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Thu, 21 Sep 2023 17:53:58 -0400 Subject: [PATCH 1/3] Add LockDeletionForDays parameter to set PodDisruptionBudget and cleanup job --- .../stress-testing/deploy-stress-tests.ps1 | 5 ++- .../stress-test-deployment-lib.ps1 | 35 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 8abaa40d0cb4..7e1fe579f263 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -31,7 +31,10 @@ param( [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, + + # Prevent kubernetes from deleting nodes or rebalancing pods related to this test for N days + [Parameter(Mandatory=$False)][int]$LockDeletionForDays ) . $PSScriptRoot/stress-test-deployment-lib.ps1 diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index bdaec9e711a9..1a5c5b408469 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -107,7 +107,8 @@ function DeployStressTests( [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, - [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, + [Parameter(Mandatory=$False)][int]$LockDeletionForDays ) { if ($environment -eq 'pg') { if ($clusterGroup -or $subscription) { @@ -168,7 +169,7 @@ function DeployStressTests( -subscription $subscription } - if ($FailedCommands.Count -lt $pkgs.Count) { + if ($FailedCommands.Count -lt $pkgs.Count -and !$Template) { Write-Host "Releases deployed by $deployer" Run helm list --all-namespaces -l deployId=$deployer } @@ -211,12 +212,14 @@ function DeployStressPackage( } $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)" - Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." - kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} - Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)" - $limitRangeSpec | kubectl apply -n $pkg.Namespace -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} + if (!$Template) { + Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." + kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} + Write-Host "Adding default resource requests to namespace/$($pkg.Namespace)" + $limitRangeSpec | kubectl apply -n $pkg.Namespace -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} + } $dockerBuildConfigs = @() @@ -317,8 +320,18 @@ function DeployStressPackage( $generatedConfigPath = Join-Path $pkg.Directory generatedValues.yaml $subCommand = $Template ? "template" : "upgrade" - $installFlag = $Template ? "" : "--install" - $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $installFlag, "--set", "stress-test-addons.env=$environment", "--values", $generatedConfigPath + $subCommandFlag = $Template ? "--debug" : "--install" + $helmCommandArg = "helm", $subCommand, $releaseName, $pkg.Directory, "-n", $pkg.Namespace, $subCommandFlag, "--values", $generatedConfigPath, "--set", "stress-test-addons.env=$environment" + + if ($LockDeletionForDays) { + $date = (Get-Date).AddDays($LockDeletionForDays).ToUniversalTime() + $isoDate = $date.ToString("o") + # Tell kubernetes job to run only on this specific future time. Technically it will run once per year. + $cron = "$($date.Minute) $($date.Hour) $($date.Day) $($date.Month) *" + + Write-Host "PodDisruptionBudget will be set to prevent deletion until $isoDate" + $helmCommandArg += "--set", "PodDisruptionBudgetExpiry=$($isoDate)", "--set", "PodDisruptionBudgetExpiryCron=$cron" + } $result = (Run @helmCommandArg) 2>&1 | Write-Host @@ -342,7 +355,7 @@ function DeployStressPackage( # Helm 3 stores release information in kubernetes secrets. The only way to add extra labels around # specific releases (thereby enabling filtering on `helm list`) is to label the underlying secret resources. # There is not currently support for setting these labels via the helm cli. - if(!$Template) { + if (!$Template) { $helmReleaseConfig = RunOrExitOnFailure kubectl get secrets ` -n $pkg.Namespace ` -l "status=deployed,name=$releaseName" ` From 4840810aa5287f8e62dab6462fee257631cd7c66 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 22 Sep 2023 16:18:28 -0400 Subject: [PATCH 2/3] Use matrix for parallel tests. PDB improvements+docs. --- eng/common/scripts/stress-testing/deploy-stress-tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 7e1fe579f263..61d8f947d800 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -34,7 +34,7 @@ param( [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters, # Prevent kubernetes from deleting nodes or rebalancing pods related to this test for N days - [Parameter(Mandatory=$False)][int]$LockDeletionForDays + [Parameter(Mandatory=$False)][ValidateRange(1, 14)][int]$LockDeletionForDays ) . $PSScriptRoot/stress-test-deployment-lib.ps1 From 2a5d149836523b1154de5a3e7e519952b22d7af2 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 22 Sep 2023 16:47:58 -0400 Subject: [PATCH 3/3] Fix kubectl namespace context preservation on login --- .../scripts/stress-testing/stress-test-deployment-lib.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 1a5c5b408469..e956508c4392 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -59,7 +59,7 @@ function Login([string]$subscription, [string]$clusterGroup, [switch]$skipPushIm $kubeContext = (RunOrExitOnFailure kubectl config view -o json) | ConvertFrom-Json -AsHashtable $defaultNamespace = $null $targetContext = $kubeContext.contexts.Where({ $_.name -eq $clusterName }) | Select -First 1 - if ($targetContext -ne $null -and $targetContext.PSObject.Properties.Name -match "namespace") { + if ($targetContext -ne $null -and $targetContext.Contains('context') -and $targetContext.Contains('namespace')) { $defaultNamespace = $targetContext.context.namespace }