diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 1a2a0a90083..d9a5afaab1b 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -22,9 +22,9 @@ function Update-PSModulePathForCI() $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) }) # Add any "az_" paths from the agent which is the lastest set of azure modules - $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator - if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) { - $modulePaths += $AzModuleCachPath + $AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator + if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) { + $modulePaths += $AzModuleCachePath } $env:PSModulePath = $modulePaths -join $moduleSeperator diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 index 6eba7e8795c..ea4923daa42 100644 --- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) { Write-Error "ConfigPath '$ConfigPath' does not exist." exit 1 } -$config = GetMatrixConfigFromJson (Get-Content $ConfigPath) +$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) # Strip empty string filters in order to be able to use azure pipelines yaml join() $Filters = $Filters | Where-Object { $_ } @@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ } $serialized = SerializePipelineMatrix $matrix Write-Output $serialized.pretty -Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" + +if ($null -ne $env:SYSTEM_TEAMPROJECTID) { + Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" +} \ No newline at end of file diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 8822d7ce723..0926185910b 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -84,6 +84,7 @@ class MatrixParameter { } } +. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1) $IMPORT_KEYWORD = '$IMPORT' function GenerateMatrix( @@ -146,7 +147,7 @@ function ProcessNonSparseParameters( function FilterMatrixDisplayName([array]$matrix, [string]$filter) { return $matrix | Where-Object { $_ } | ForEach-Object { - if ($_.Name -match $filter) { + if ($_.ContainsKey("Name") -and $_.Name -match $filter) { return $_ } } @@ -168,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) { # Default all regex checks to go against empty string when keys are missing. # This simplifies the filter syntax/interface to be regex only. $value = "" - if ($null -ne $entry -and $entry.parameters.Contains($key)) { + if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) { $value = $entry.parameters[$key] } if ($value -notmatch $regex) { @@ -190,11 +191,34 @@ function ParseFilter([string]$filter) { } } -# Importing the JSON as PSCustomObject preserves key ordering, -# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfigFromFile([String] $config) +{ + [MatrixConfig]$config = try{ + GetMatrixConfigFromJson $config + } catch { + GetMatrixConfigFromYaml $config + } + return $config +} + +function GetMatrixConfigFromYaml([String] $yamlConfig) +{ + Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + # ConvertTo then from json is to make sure the nested values are in PSCustomObject + [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json + return GetMatrixConfig $config +} + function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + return GetMatrixConfig $config +} + +# Importing the JSON as PSCustomObject preserves key ordering, +# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfig([MatrixConfig]$config) +{ $config.matrixParameters = @() $config.displayNamesLookup = @{} $include = [MatrixParameter[]]@() @@ -359,7 +383,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n Write-Error "`$IMPORT path '$importPath' does not exist." exit 1 } - $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) + $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) $importedMatrix = GenerateMatrix ` -config $importedMatrixConfig ` -selectFromMatrixType $selection ` diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 01920bdcbfe..c71e6294315 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -24,7 +24,15 @@ param( [string]$Namespace, # Override remote stress-test-addons with local on-disk addons for development - [System.IO.FileInfo]$LocalAddonsPath + [System.IO.FileInfo]$LocalAddonsPath, + + # Matrix generation parameters + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) . $PSScriptRoot/stress-test-deployment-lib.ps1 diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index 24c27da485f..f949b03ad8a 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -12,20 +12,42 @@ class StressTestPackageInfo { [string]$Deployer } +. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1 +. $PSScriptRoot/generate-scenario-matrix.ps1 + function FindStressPackages( [string]$directory, [hashtable]$filters = @{}, [switch]$CI, - [string]$namespaceOverride + [string]$namespaceOverride, + [string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) { # Bare minimum filter for stress tests $filters['stressTest'] = 'true' - $packages = @() $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory + if (!$MatrixFileName) { + $MatrixFileName = '/scenarios-matrix.yaml' + } foreach ($chartFile in $chartFiles) { $chart = ParseChart $chartFile if (matchesAnnotations $chart $filters) { + $matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName) + if (Test-Path $matrixFilePath) { + GenerateScenarioMatrix ` + -matrixFilePath $matrixFilePath ` + -Selection $MatrixSelection ` + -DisplayNameFilter $MatrixDisplayNameFilter ` + -Filters $MatrixFilters ` + -Replace $MatrixReplace ` + -NonSparseParameters $MatrixNonSparseParameters + } + $packages += NewStressTestPackageInfo ` -chart $chart ` -chartFile $chartFile ` @@ -80,8 +102,8 @@ function NewStressTestPackageInfo( Namespace = $namespace.ToLower() Directory = $chartFile.DirectoryName ReleaseName = $chart.name - Dockerfile = $chart.annotations.dockerfile - DockerBuildDir = $chart.annotations.dockerbuilddir + Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null + DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null } } diff --git a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 new file mode 100644 index 00000000000..aba8be3451d --- /dev/null +++ b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 @@ -0,0 +1,86 @@ +param( + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters +) + +function GenerateScenarioMatrix( + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters +) { + $yamlConfig = Get-Content $matrixFilePath -Raw + + $prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` + -ConfigPath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters + Write-Host $prettyMatrix + $prettyMatrix = $prettyMatrix | ConvertFrom-Json + + $scenariosMatrix = @() + foreach($permutation in $prettyMatrix.psobject.properties) { + $entry = @{} + $entry.Name = $permutation.Name -replace '_', '-' + $entry.Scenario = $entry.Name + $entry.Remove("Name") + foreach ($param in $permutation.value.psobject.properties) { + $entry.add($param.Name, $param.value) + } + $scenariosMatrix += $entry + } + + $valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml') + $values = $valuesYaml | ConvertFrom-Yaml -Ordered + if (!$values) {$values = @{}} + + if ($values.ContainsKey('Scenarios')) { + throw "Please use matrix generation for stress test scenarios." + } + + $values.scenarios = $scenariosMatrix + $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml') +} + +function NewStressTestPackageInfo( + [hashtable]$chart, + [System.IO.FileInfo]$chartFile, + [switch]$CI, + [object]$namespaceOverride +) { + $namespace = if ($namespaceOverride) { + $namespaceOverride + } elseif ($CI) { + $chart.annotations.namespace + } else { + GetUsername + } + + return [StressTestPackageInfo]@{ + Namespace = $namespace.ToLower() + Directory = $chartFile.DirectoryName + ReleaseName = $chart.name + Dockerfile = $chart.annotations.dockerfile + DockerBuildDir = $chart.annotations.dockerbuilddir + } +} + +# Don't call functions when the script is being dot sourced +if ($MyInvocation.InvocationName -ne ".") { + GenerateScenarioMatrix ` + -matrixFilePath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters +} diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 9369dcc6baf..a0824777797 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -79,7 +79,13 @@ function DeployStressTests( } return $true })] - [System.IO.FileInfo]$LocalAddonsPath + [System.IO.FileInfo]$LocalAddonsPath, + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse", + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) { if ($environment -eq 'pg') { if ($clusterGroup -or $subscription) { @@ -115,9 +121,17 @@ function DeployStressTests( Run helm repo update if ($LASTEXITCODE) { return $LASTEXITCODE } - $deployer = if ($deployId) { $deployId } else { GetUsername } - $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace + $pkgs = @(FindStressPackages ` + -directory $searchDirectory ` + -filters $filters ` + -CI:$CI ` + -namespaceOverride $Namespace ` + -MatrixSelection $MatrixSelection ` + -MatrixFileName $MatrixFileName ` + -MatrixFilters $MatrixFilters ` + -MatrixReplace $MatrixReplace ` + -MatrixNonSparseParameters $MatrixNonSparseParameters) Write-Host "" "Found $($pkgs.Length) stress test packages:" Write-Host $pkgs.Directory "" foreach ($pkg in $pkgs) { @@ -164,59 +178,96 @@ function DeployStressPackage( if ($LASTEXITCODE) { return } } - $imageTag = "${registryName}.azurecr.io" + $imageTagBase = "${registryName}.azurecr.io" if ($repositoryBase) { - $imageTag += "/$repositoryBase" + $imageTagBase += "/$repositoryBase" } - $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}" + $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)" - $dockerFilePath = if ($pkg.Dockerfile) { - Join-Path $pkg.Directory $pkg.Dockerfile - } else { - "$($pkg.Directory)/Dockerfile" - } - $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) - - if ($pushImages -and (Test-Path $dockerFilePath)) { - Write-Host "Building and pushing stress test docker image '$imageTag'" - $dockerFile = Get-ChildItem $dockerFilePath - $dockerBuildFolder = if ($pkg.DockerBuildDir) { - Join-Path $pkg.Directory $pkg.DockerBuildDir - } else { - $dockerFile.DirectoryName - } - $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim() + Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." + kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} - Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder - if ($LASTEXITCODE) { return } + $dockerBuildConfigs = @() + + $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" + $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered + if (Test-Path $genValFile) { + $scenarios = $genVal.Scenarios + foreach ($scenario in $scenarios) { + if ("image" -in $scenario.keys) { + $dockerFilePath = Join-Path $pkg.Directory $scenario.image + } else { + $dockerFilePath = "$($pkg.Directory)/Dockerfile" + } + $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath).Trim() - Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue - Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue - Write-Host " docker run -it $imageTag " -ForegroundColor DarkBlue - Write-Host "To show installed container images:" -ForegroundColor Blue - Write-Host " docker image ls" -ForegroundColor DarkBlue - Write-Host "To show running containers:" -ForegroundColor Blue - Write-Host " docker ps" -ForegroundColor DarkBlue - - Run docker push $imageTag - if ($LASTEXITCODE) { - if ($login) { - Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + if ("imageBuildDir" -in $scenario.keys) { + $dockerBuildDir = Join-Path $pkg.Directory $scenario.imageBuildDir + } else { + $dockerBuildDir = Split-Path $dockerFilePath } - return + $dockerBuildDir = [System.IO.Path]::GetFullPath($dockerBuildDir).Trim() + $dockerBuildConfigs += @{"dockerFilePath"=$dockerFilePath; "dockerBuildDir"=$dockerBuildDir} } } + if ($pkg.Dockerfile -or $pkg.DockerBuildDir) { + throw "The chart.yaml docker config is depracated, please use the scenarios matrix instead." + } + - Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." - kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} + foreach ($dockerBuildConfig in $dockerBuildConfigs) { + $dockerFilePath = $dockerBuildConfig.dockerFilePath + $dockerBuildFolder = $dockerBuildConfig.dockerBuildDir + if (!(Test-Path $dockerFilePath)) { + throw "Invalid dockerfile path, cannot find dockerfile at ${dockerFilePath}" + } + if (!(Test-Path $dockerBuildFolder)) { + throw "Invalid docker build directory, cannot find directory ${dockerBuildFolder}" + } + $dockerfileName = ($dockerFilePath -split { $_ -in '\', '/' })[-1].ToLower() + $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}" + if ($pushImages) { + Write-Host "Building and pushing stress test docker image '$imageTag'" + $dockerFile = Get-ChildItem $dockerFilePath + + Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder + + Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue + Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue + Write-Host " docker run -it $imageTag " -ForegroundColor DarkBlue + Write-Host "To show installed container images:" -ForegroundColor Blue + Write-Host " docker image ls" -ForegroundColor DarkBlue + Write-Host "To show running containers:" -ForegroundColor Blue + Write-Host " docker ps" -ForegroundColor DarkBlue + + Run docker push $imageTag + if ($LASTEXITCODE) { + if ($login) { + Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + } + } + } + $genVal.scenarios = foreach ($scenario in $genVal.scenarios) { + $dockerPath = Join-Path $pkg.Directory $scenario.image + if ("image" -notin $scenario) { + $dockerPath = $dockerFilePath + } + if ([System.IO.Path]::GetFullPath($dockerPath) -eq $dockerFilePath) { + $scenario.imageTag = $imageTag + } + $scenario + } + + $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile + } Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)" Run helm upgrade $pkg.ReleaseName $pkg.Directory ` -n $pkg.Namespace ` --install ` - --set image=$imageTag ` - --set stress-test-addons.env=$environment + --set stress-test-addons.env=$environment ` + --values generatedValues.yaml if ($LASTEXITCODE) { # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress' # can be the result of cancelled `upgrade` operations (e.g. ctrl-c). diff --git a/tools/stress-cluster/chaos/examples/network-stress-example/templates/testjob.yaml b/tools/stress-cluster/chaos/examples/network-stress-example/templates/testjob.yaml index c89d5f67dc1..d9ec2025d40 100644 --- a/tools/stress-cluster/chaos/examples/network-stress-example/templates/testjob.yaml +++ b/tools/stress-cluster/chaos/examples/network-stress-example/templates/testjob.yaml @@ -12,6 +12,6 @@ spec: - name: network-example command: ["bash", "poll.sh"] imagePullPolicy: Always - image: {{ .Values.image }} + image: {{ .Stress.image }} {{- include "stress-test-addons.container-env" . | nindent 6 }} {{- end -}} diff --git a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_util.tpl b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_util.tpl index 626dfeee56a..a619725bd9c 100644 --- a/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_util.tpl +++ b/tools/stress-cluster/cluster/kubernetes/stress-test-addons/templates/_util.tpl @@ -35,11 +35,12 @@ See https://github.com/Masterminds/sprig/tree/master/docs for template function {{- /* Copy scenario name into top level keys of global context */}} {{- $_global := index . 0 -}} {{- $_scenario := index . 1 -}} -{{- $resourceGroupName := lower (print $_global.Release.Namespace "-" $_scenario "-" $_global.Release.Name "-" $_global.Release.Revision) -}} +{{- $resourceGroupName := lower (print $_global.Release.Namespace "-" $_scenario.Scenario "-" $_global.Release.Name "-" $_global.Release.Revision) -}} {{- /* Use lowercase alphanumeric characters beginning with a letter for maximum azure resource naming compatibility */ -}} {{- $uniqueTestId := lower (print "s" (trunc 5 (sha1sum $resourceGroupName) ) ) -}} {{- /* Create add Stress context to top level keys of global context */}} -{{- $_stress := dict "Scenario" $_scenario "ResourceGroupName" $resourceGroupName "BaseName" $uniqueTestId -}} +{{- $_stress := dict "ResourceGroupName" $resourceGroupName "BaseName" $uniqueTestId -}} +{{- $_stress := merge $_stress $_scenario -}} {{- $_instance := deepCopy ($_global | merge (dict "Stress" $_stress )) -}} {{ toYaml ($_instance) }} {{- end -}}