From 90f1ff9364fdbb3b297043e9a0a999a3ba9eeecc Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 20 Oct 2022 16:30:02 -0700 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 3790 (#4041) * Added yaml support for job matrix creation * autogen scenario matrix for stress test * Temporary Working State * update to default sparse * pr comments and some error handling * custom matrixfilename and ordering of generatedValues.yaml * common module import * JobMatrix write host Co-authored-by: Albert Cheng --- .../scripts/Helpers/PSModule-Helpers.ps1 | 6 +- .../scripts/job-matrix/Create-JobMatrix.ps1 | 7 +- .../job-matrix/job-matrix-functions.ps1 | 34 ++++- .../stress-testing/deploy-stress-tests.ps1 | 10 +- .../find-all-stress-packages.ps1 | 30 +++- .../generate-scenario-matrix.ps1 | 86 +++++++++++ .../stress-test-deployment-lib.ps1 | 135 ++++++++++++------ 7 files changed, 251 insertions(+), 57 deletions(-) create mode 100644 eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 1a2a0a9008..d9a5afaab1 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -22,9 +22,9 @@ function Update-PSModulePathForCI() $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) }) # Add any "az_" paths from the agent which is the lastest set of azure modules - $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator - if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) { - $modulePaths += $AzModuleCachPath + $AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator + if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) { + $modulePaths += $AzModuleCachePath } $env:PSModulePath = $modulePaths -join $moduleSeperator diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 index 6eba7e8795..ea4923daa4 100644 --- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) { Write-Error "ConfigPath '$ConfigPath' does not exist." exit 1 } -$config = GetMatrixConfigFromJson (Get-Content $ConfigPath) +$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) # Strip empty string filters in order to be able to use azure pipelines yaml join() $Filters = $Filters | Where-Object { $_ } @@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ } $serialized = SerializePipelineMatrix $matrix Write-Output $serialized.pretty -Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" + +if ($null -ne $env:SYSTEM_TEAMPROJECTID) { + Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" +} \ No newline at end of file diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 8822d7ce72..0926185910 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -84,6 +84,7 @@ class MatrixParameter { } } +. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1) $IMPORT_KEYWORD = '$IMPORT' function GenerateMatrix( @@ -146,7 +147,7 @@ function ProcessNonSparseParameters( function FilterMatrixDisplayName([array]$matrix, [string]$filter) { return $matrix | Where-Object { $_ } | ForEach-Object { - if ($_.Name -match $filter) { + if ($_.ContainsKey("Name") -and $_.Name -match $filter) { return $_ } } @@ -168,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) { # Default all regex checks to go against empty string when keys are missing. # This simplifies the filter syntax/interface to be regex only. $value = "" - if ($null -ne $entry -and $entry.parameters.Contains($key)) { + if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) { $value = $entry.parameters[$key] } if ($value -notmatch $regex) { @@ -190,11 +191,34 @@ function ParseFilter([string]$filter) { } } -# Importing the JSON as PSCustomObject preserves key ordering, -# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfigFromFile([String] $config) +{ + [MatrixConfig]$config = try{ + GetMatrixConfigFromJson $config + } catch { + GetMatrixConfigFromYaml $config + } + return $config +} + +function GetMatrixConfigFromYaml([String] $yamlConfig) +{ + Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + # ConvertTo then from json is to make sure the nested values are in PSCustomObject + [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json + return GetMatrixConfig $config +} + function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + return GetMatrixConfig $config +} + +# Importing the JSON as PSCustomObject preserves key ordering, +# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfig([MatrixConfig]$config) +{ $config.matrixParameters = @() $config.displayNamesLookup = @{} $include = [MatrixParameter[]]@() @@ -359,7 +383,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n Write-Error "`$IMPORT path '$importPath' does not exist." exit 1 } - $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) + $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) $importedMatrix = GenerateMatrix ` -config $importedMatrixConfig ` -selectFromMatrixType $selection ` diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 01920bdcbf..c71e629431 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -24,7 +24,15 @@ param( [string]$Namespace, # Override remote stress-test-addons with local on-disk addons for development - [System.IO.FileInfo]$LocalAddonsPath + [System.IO.FileInfo]$LocalAddonsPath, + + # Matrix generation parameters + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) . $PSScriptRoot/stress-test-deployment-lib.ps1 diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index 24c27da485..f949b03ad8 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -12,20 +12,42 @@ class StressTestPackageInfo { [string]$Deployer } +. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1 +. $PSScriptRoot/generate-scenario-matrix.ps1 + function FindStressPackages( [string]$directory, [hashtable]$filters = @{}, [switch]$CI, - [string]$namespaceOverride + [string]$namespaceOverride, + [string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) { # Bare minimum filter for stress tests $filters['stressTest'] = 'true' - $packages = @() $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory + if (!$MatrixFileName) { + $MatrixFileName = '/scenarios-matrix.yaml' + } foreach ($chartFile in $chartFiles) { $chart = ParseChart $chartFile if (matchesAnnotations $chart $filters) { + $matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName) + if (Test-Path $matrixFilePath) { + GenerateScenarioMatrix ` + -matrixFilePath $matrixFilePath ` + -Selection $MatrixSelection ` + -DisplayNameFilter $MatrixDisplayNameFilter ` + -Filters $MatrixFilters ` + -Replace $MatrixReplace ` + -NonSparseParameters $MatrixNonSparseParameters + } + $packages += NewStressTestPackageInfo ` -chart $chart ` -chartFile $chartFile ` @@ -80,8 +102,8 @@ function NewStressTestPackageInfo( Namespace = $namespace.ToLower() Directory = $chartFile.DirectoryName ReleaseName = $chart.name - Dockerfile = $chart.annotations.dockerfile - DockerBuildDir = $chart.annotations.dockerbuilddir + Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null + DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null } } diff --git a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 new file mode 100644 index 0000000000..aba8be3451 --- /dev/null +++ b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 @@ -0,0 +1,86 @@ +param( + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters +) + +function GenerateScenarioMatrix( + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters +) { + $yamlConfig = Get-Content $matrixFilePath -Raw + + $prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` + -ConfigPath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters + Write-Host $prettyMatrix + $prettyMatrix = $prettyMatrix | ConvertFrom-Json + + $scenariosMatrix = @() + foreach($permutation in $prettyMatrix.psobject.properties) { + $entry = @{} + $entry.Name = $permutation.Name -replace '_', '-' + $entry.Scenario = $entry.Name + $entry.Remove("Name") + foreach ($param in $permutation.value.psobject.properties) { + $entry.add($param.Name, $param.value) + } + $scenariosMatrix += $entry + } + + $valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml') + $values = $valuesYaml | ConvertFrom-Yaml -Ordered + if (!$values) {$values = @{}} + + if ($values.ContainsKey('Scenarios')) { + throw "Please use matrix generation for stress test scenarios." + } + + $values.scenarios = $scenariosMatrix + $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml') +} + +function NewStressTestPackageInfo( + [hashtable]$chart, + [System.IO.FileInfo]$chartFile, + [switch]$CI, + [object]$namespaceOverride +) { + $namespace = if ($namespaceOverride) { + $namespaceOverride + } elseif ($CI) { + $chart.annotations.namespace + } else { + GetUsername + } + + return [StressTestPackageInfo]@{ + Namespace = $namespace.ToLower() + Directory = $chartFile.DirectoryName + ReleaseName = $chart.name + Dockerfile = $chart.annotations.dockerfile + DockerBuildDir = $chart.annotations.dockerbuilddir + } +} + +# Don't call functions when the script is being dot sourced +if ($MyInvocation.InvocationName -ne ".") { + GenerateScenarioMatrix ` + -matrixFilePath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters +} diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 9369dcc6ba..a082477779 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -79,7 +79,13 @@ function DeployStressTests( } return $true })] - [System.IO.FileInfo]$LocalAddonsPath + [System.IO.FileInfo]$LocalAddonsPath, + [Parameter(Mandatory=$False)][string]$MatrixFileName, + [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse", + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) { if ($environment -eq 'pg') { if ($clusterGroup -or $subscription) { @@ -115,9 +121,17 @@ function DeployStressTests( Run helm repo update if ($LASTEXITCODE) { return $LASTEXITCODE } - $deployer = if ($deployId) { $deployId } else { GetUsername } - $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace + $pkgs = @(FindStressPackages ` + -directory $searchDirectory ` + -filters $filters ` + -CI:$CI ` + -namespaceOverride $Namespace ` + -MatrixSelection $MatrixSelection ` + -MatrixFileName $MatrixFileName ` + -MatrixFilters $MatrixFilters ` + -MatrixReplace $MatrixReplace ` + -MatrixNonSparseParameters $MatrixNonSparseParameters) Write-Host "" "Found $($pkgs.Length) stress test packages:" Write-Host $pkgs.Directory "" foreach ($pkg in $pkgs) { @@ -164,59 +178,96 @@ function DeployStressPackage( if ($LASTEXITCODE) { return } } - $imageTag = "${registryName}.azurecr.io" + $imageTagBase = "${registryName}.azurecr.io" if ($repositoryBase) { - $imageTag += "/$repositoryBase" + $imageTagBase += "/$repositoryBase" } - $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}" + $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)" - $dockerFilePath = if ($pkg.Dockerfile) { - Join-Path $pkg.Directory $pkg.Dockerfile - } else { - "$($pkg.Directory)/Dockerfile" - } - $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) - - if ($pushImages -and (Test-Path $dockerFilePath)) { - Write-Host "Building and pushing stress test docker image '$imageTag'" - $dockerFile = Get-ChildItem $dockerFilePath - $dockerBuildFolder = if ($pkg.DockerBuildDir) { - Join-Path $pkg.Directory $pkg.DockerBuildDir - } else { - $dockerFile.DirectoryName - } - $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim() + Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." + kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} - Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder - if ($LASTEXITCODE) { return } + $dockerBuildConfigs = @() + + $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" + $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered + if (Test-Path $genValFile) { + $scenarios = $genVal.Scenarios + foreach ($scenario in $scenarios) { + if ("image" -in $scenario.keys) { + $dockerFilePath = Join-Path $pkg.Directory $scenario.image + } else { + $dockerFilePath = "$($pkg.Directory)/Dockerfile" + } + $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath).Trim() - Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue - Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue - Write-Host " docker run -it $imageTag " -ForegroundColor DarkBlue - Write-Host "To show installed container images:" -ForegroundColor Blue - Write-Host " docker image ls" -ForegroundColor DarkBlue - Write-Host "To show running containers:" -ForegroundColor Blue - Write-Host " docker ps" -ForegroundColor DarkBlue - - Run docker push $imageTag - if ($LASTEXITCODE) { - if ($login) { - Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + if ("imageBuildDir" -in $scenario.keys) { + $dockerBuildDir = Join-Path $pkg.Directory $scenario.imageBuildDir + } else { + $dockerBuildDir = Split-Path $dockerFilePath } - return + $dockerBuildDir = [System.IO.Path]::GetFullPath($dockerBuildDir).Trim() + $dockerBuildConfigs += @{"dockerFilePath"=$dockerFilePath; "dockerBuildDir"=$dockerBuildDir} } } + if ($pkg.Dockerfile -or $pkg.DockerBuildDir) { + throw "The chart.yaml docker config is depracated, please use the scenarios matrix instead." + } + - Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." - kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} + foreach ($dockerBuildConfig in $dockerBuildConfigs) { + $dockerFilePath = $dockerBuildConfig.dockerFilePath + $dockerBuildFolder = $dockerBuildConfig.dockerBuildDir + if (!(Test-Path $dockerFilePath)) { + throw "Invalid dockerfile path, cannot find dockerfile at ${dockerFilePath}" + } + if (!(Test-Path $dockerBuildFolder)) { + throw "Invalid docker build directory, cannot find directory ${dockerBuildFolder}" + } + $dockerfileName = ($dockerFilePath -split { $_ -in '\', '/' })[-1].ToLower() + $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}" + if ($pushImages) { + Write-Host "Building and pushing stress test docker image '$imageTag'" + $dockerFile = Get-ChildItem $dockerFilePath + + Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder + + Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue + Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue + Write-Host " docker run -it $imageTag " -ForegroundColor DarkBlue + Write-Host "To show installed container images:" -ForegroundColor Blue + Write-Host " docker image ls" -ForegroundColor DarkBlue + Write-Host "To show running containers:" -ForegroundColor Blue + Write-Host " docker ps" -ForegroundColor DarkBlue + + Run docker push $imageTag + if ($LASTEXITCODE) { + if ($login) { + Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + } + } + } + $genVal.scenarios = foreach ($scenario in $genVal.scenarios) { + $dockerPath = Join-Path $pkg.Directory $scenario.image + if ("image" -notin $scenario) { + $dockerPath = $dockerFilePath + } + if ([System.IO.Path]::GetFullPath($dockerPath) -eq $dockerFilePath) { + $scenario.imageTag = $imageTag + } + $scenario + } + + $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile + } Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)" Run helm upgrade $pkg.ReleaseName $pkg.Directory ` -n $pkg.Namespace ` --install ` - --set image=$imageTag ` - --set stress-test-addons.env=$environment + --set stress-test-addons.env=$environment ` + --values generatedValues.yaml if ($LASTEXITCODE) { # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress' # can be the result of cancelled `upgrade` operations (e.g. ctrl-c).