From 515dd113a81319ecaed108250341354fce5069a6 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Tue, 12 Jul 2022 15:30:47 -0700 Subject: [PATCH 1/8] Added yaml support for job matrix creation --- eng/common/scripts/job-matrix/job-matrix-functions.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 8822d7ce72..95a7ef9d16 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -190,6 +190,12 @@ function ParseFilter([string]$filter) { } } +function GetMatrixConfigFromYaml([String] $yamlConfig) +{ + $config = $yamlConfig | ConvertFrom-Yaml + return GetMatrixConfigFromJson($config | ConvertTo-Json) +} + # Importing the JSON as PSCustomObject preserves key ordering, # whereas ConvertFrom-Json -AsHashtable does not function GetMatrixConfigFromJson([String]$jsonConfig) From c96c6854fcdb1f4102d5de58901e3af9c29d8fe8 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Fri, 29 Jul 2022 09:45:03 -0700 Subject: [PATCH 2/8] autogen scenario matrix for stress test --- .../job-matrix/job-matrix-functions.ps1 | 2 +- .../find-all-stress-packages.ps1 | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 95a7ef9d16..fd7b3e8458 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -193,7 +193,7 @@ function ParseFilter([string]$filter) { function GetMatrixConfigFromYaml([String] $yamlConfig) { $config = $yamlConfig | ConvertFrom-Yaml - return GetMatrixConfigFromJson($config | ConvertTo-Json) + return GetMatrixConfigFromJson (ConvertTo-Json $config) } # Importing the JSON as PSCustomObject preserves key ordering, diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index 24c27da485..ae7cf3eda7 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -12,6 +12,8 @@ class StressTestPackageInfo { [string]$Deployer } +. $PSScriptRoot/../job-matrix/job-matrix-functions.ps1 + function FindStressPackages( [string]$directory, [hashtable]$filters = @{}, @@ -26,6 +28,11 @@ function FindStressPackages( foreach ($chartFile in $chartFiles) { $chart = ParseChart $chartFile if (matchesAnnotations $chart $filters) { + $matrixFilePath = (Join-Path $chartFile.Directory.FullName '/matrix.yaml') + if (Test-Path $matrixFilePath) { + ScenariosMatrixGeneration $matrixFilePath + } + $packages += NewStressTestPackageInfo ` -chart $chart ` -chartFile $chartFile ` @@ -62,6 +69,31 @@ function GetUsername() { return $stressUser.ToLower() } +function ScenariosMatrixGeneration( + [string]$matrixFilePath +) { + $yamlConfig = Get-Content $matrixFilePath -Raw + $matrix = GenerateMatrix (GetMatrixConfigFromYaml $yamlConfig) "sparse" + $serializedMatrix = SerializePipelineMatrix $matrix + $prettyMatrix = $serializedMatrix.pretty | ConvertFrom-Json -AsHashtable + + $scenariosMatrix = @() + foreach($permutation in $prettyMatrix.GetEnumerator()) { + $entry = @{} + $entry.Name = $permutation.key + foreach ($param in $permutation.value.GetEnumerator()) { + $entry.add($param.key, $param.value) + } + $scenariosMatrix += $entry + } + + $valuesYaml = Get-Content (Join-Path $matrixFilePath '../values.yaml') -Raw + $values = $valuesYaml | ConvertFrom-Yaml + + $values.Scenarios += $scenariosMatrix + $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml') +} + function NewStressTestPackageInfo( [hashtable]$chart, [System.IO.FileInfo]$chartFile, From ee899c1e8dfceb3b12851feeb7ce5f7cdc097821 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 29 Sep 2022 17:00:46 -0700 Subject: [PATCH 3/8] Temporary Working State --- .../scripts/job-matrix/Create-JobMatrix.ps1 | 7 +- .../job-matrix/job-matrix-functions.ps1 | 26 +++- .../stress-testing/deploy-stress-tests.ps1 | 9 +- .../find-all-stress-packages.ps1 | 46 +++++-- .../stress-test-deployment-lib.ps1 | 123 ++++++++++++------ 5 files changed, 153 insertions(+), 58 deletions(-) diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 index 6eba7e8795..3f14b9a1d9 100644 --- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -23,7 +23,7 @@ if (!(Test-Path $ConfigPath)) { Write-Error "ConfigPath '$ConfigPath' does not exist." exit 1 } -$config = GetMatrixConfigFromJson (Get-Content $ConfigPath) +$config = GetMatrixConfigFromFile (Get-Content $ConfigPath -Raw) # Strip empty string filters in order to be able to use azure pipelines yaml join() $Filters = $Filters | Where-Object { $_ } @@ -38,4 +38,7 @@ $Filters = $Filters | Where-Object { $_ } $serialized = SerializePipelineMatrix $matrix Write-Output $serialized.pretty -Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" + +if ($null -ne $env:SYSTEM_TEAMPROJECTID) { + Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" +} diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index fd7b3e8458..7c96762be4 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -190,17 +190,33 @@ function ParseFilter([string]$filter) { } } +function GetMatrixConfigFromFile([String] $config) +{ + [MatrixConfig]$config = try{ + GetMatrixConfigFromJson($config) + } catch { + GetMatrixConfigFromYaml($config) + } + return $config +} + function GetMatrixConfigFromYaml([String] $yamlConfig) { - $config = $yamlConfig | ConvertFrom-Yaml - return GetMatrixConfigFromJson (ConvertTo-Json $config) + # ConvertTo then from json is to make sure the nested values are in PSCustomObject + [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json + return GetMatrixConfig $config } -# Importing the JSON as PSCustomObject preserves key ordering, -# whereas ConvertFrom-Json -AsHashtable does not function GetMatrixConfigFromJson([String]$jsonConfig) { [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + return GetMatrixConfig $config +} + +# Importing the JSON as PSCustomObject preserves key ordering, +# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfig([MatrixConfig]$config) +{ $config.matrixParameters = @() $config.displayNamesLookup = @{} $include = [MatrixParameter[]]@() @@ -365,7 +381,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n Write-Error "`$IMPORT path '$importPath' does not exist." exit 1 } - $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) + $importedMatrixConfig = GetMatrixConfigFromYaml (Get-Content $importPath) $importedMatrix = GenerateMatrix ` -config $importedMatrixConfig ` -selectFromMatrixType $selection ` diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index 01920bdcbf..dadf2a05e7 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -24,7 +24,14 @@ param( [string]$Namespace, # Override remote stress-test-addons with local on-disk addons for development - [System.IO.FileInfo]$LocalAddonsPath + [System.IO.FileInfo]$LocalAddonsPath, + + # Matrix generation parameters + [string]$MatrixSelection = "sparse", + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) . $PSScriptRoot/stress-test-deployment-lib.ps1 diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index ae7cf3eda7..803dc965b8 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -18,7 +18,12 @@ function FindStressPackages( [string]$directory, [hashtable]$filters = @{}, [switch]$CI, - [string]$namespaceOverride + [string]$namespaceOverride, + [string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) { # Bare minimum filter for stress tests $filters['stressTest'] = 'true' @@ -30,7 +35,13 @@ function FindStressPackages( if (matchesAnnotations $chart $filters) { $matrixFilePath = (Join-Path $chartFile.Directory.FullName '/matrix.yaml') if (Test-Path $matrixFilePath) { - ScenariosMatrixGeneration $matrixFilePath + ScenariosMatrixGeneration ` + -matrixFilePath $matrixFilePath ` + -Selection $MatrixSelection ` + -DisplayNameFilter $MatrixDisplayNameFilter ` + -Filters $MatrixFilters ` + -Replace $MatrixReplace ` + -NonSparseParameters $MatrixNonSparseParameters } $packages += NewStressTestPackageInfo ` @@ -70,17 +81,31 @@ function GetUsername() { } function ScenariosMatrixGeneration( - [string]$matrixFilePath + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters ) { $yamlConfig = Get-Content $matrixFilePath -Raw - $matrix = GenerateMatrix (GetMatrixConfigFromYaml $yamlConfig) "sparse" - $serializedMatrix = SerializePipelineMatrix $matrix - $prettyMatrix = $serializedMatrix.pretty | ConvertFrom-Json -AsHashtable + + $prettySerializedMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` + -ConfigPath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters + + $prettyMatrix = $prettySerializedMatrix | ConvertFrom-Json -AsHashtable $scenariosMatrix = @() foreach($permutation in $prettyMatrix.GetEnumerator()) { $entry = @{} - $entry.Name = $permutation.key + $entry.Name = $permutation.key -replace '_', '-' + $entry.Scenario = $entry.Name + $entry.Remove("Name") foreach ($param in $permutation.value.GetEnumerator()) { $entry.add($param.key, $param.value) } @@ -89,8 +114,13 @@ function ScenariosMatrixGeneration( $valuesYaml = Get-Content (Join-Path $matrixFilePath '../values.yaml') -Raw $values = $valuesYaml | ConvertFrom-Yaml + if (!$values) {$values = @{}} + + if ($values.ContainsKey('Scenarios')) { + throw "Please use matrix generation for stress test scenarios." + } - $values.Scenarios += $scenariosMatrix + $values.scenarios = $scenariosMatrix $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml') } diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 9369dcc6ba..9b4cbddfbc 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -79,7 +79,12 @@ function DeployStressTests( } return $true })] - [System.IO.FileInfo]$LocalAddonsPath + [System.IO.FileInfo]$LocalAddonsPath, + [string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, + [Parameter(Mandatory=$False)][array]$MatrixFilters, + [Parameter(Mandatory=$False)][array]$MatrixReplace, + [Parameter(Mandatory=$False)][array]$MatrixNonSparseParameters ) { if ($environment -eq 'pg') { if ($clusterGroup -or $subscription) { @@ -115,9 +120,16 @@ function DeployStressTests( Run helm repo update if ($LASTEXITCODE) { return $LASTEXITCODE } - $deployer = if ($deployId) { $deployId } else { GetUsername } - $pkgs = FindStressPackages -directory $searchDirectory -filters $filters -CI:$CI -namespaceOverride $Namespace + $pkgs = @(FindStressPackages ` + -directory $searchDirectory ` + -filters $filters ` + -CI:$CI ` + -namespaceOverride $Namespace ` + -MatrixSelection $MatrixSelection ` + -MatrixFilters $MatrixFilters ` + -MatrixReplace $MatrixReplace ` + -MatrixNonSparseParameters $MatrixNonSparseParameters) Write-Host "" "Found $($pkgs.Length) stress test packages:" Write-Host $pkgs.Directory "" foreach ($pkg in $pkgs) { @@ -164,59 +176,86 @@ function DeployStressPackage( if ($LASTEXITCODE) { return } } - $imageTag = "${registryName}.azurecr.io" + $imageTagBase = "${registryName}.azurecr.io" if ($repositoryBase) { - $imageTag += "/$repositoryBase" + $imageTagBase += "/$repositoryBase" } - $imageTag += "/$($pkg.Namespace)/$($pkg.ReleaseName):${deployId}" + $imageTagBase += "/$($pkg.Namespace)/$($pkg.ReleaseName)" - $dockerFilePath = if ($pkg.Dockerfile) { - Join-Path $pkg.Directory $pkg.Dockerfile - } else { - "$($pkg.Directory)/Dockerfile" - } - $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) + Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." + kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - + if ($LASTEXITCODE) {exit $LASTEXITCODE} - if ($pushImages -and (Test-Path $dockerFilePath)) { - Write-Host "Building and pushing stress test docker image '$imageTag'" - $dockerFile = Get-ChildItem $dockerFilePath - $dockerBuildFolder = if ($pkg.DockerBuildDir) { - Join-Path $pkg.Directory $pkg.DockerBuildDir + $dockerFilePaths = @() + + $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" + $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml + if (Test-Path $genValFile) { + $scenarios = $genVal.Scenarios + foreach ($scenario in $scenarios) { + if ($scenario.image) { + $dockerfilePath = Join-Path $pkg.Directory $scenario.image + $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) + $dockerFilePaths += $dockerFilePath + } + } + } else { + if ($pkg.Dockerfile) { + $dockerFilePath = Join-Path $pkg.Directory $pkg.Dockerfile } else { - $dockerFile.DirectoryName + $dockerFilePath = "$($pkg.Directory)/Dockerfile" } - $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim() - - Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder - if ($LASTEXITCODE) { return } - - Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue - Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue - Write-Host " docker run -it $imageTag " -ForegroundColor DarkBlue - Write-Host "To show installed container images:" -ForegroundColor Blue - Write-Host " docker image ls" -ForegroundColor DarkBlue - Write-Host "To show running containers:" -ForegroundColor Blue - Write-Host " docker ps" -ForegroundColor DarkBlue - - Run docker push $imageTag - if ($LASTEXITCODE) { - if ($login) { - Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) + $dockerFilePaths += $dockerFilePath + } + + + foreach ($dockerFilePath in $dockerFilePaths) { + if (!(Test-Path $dockerFilePath)) {continue} + $dockerfileName = ($dockerfilePath -split { $_ -in '\', '/' })[-1].ToLower() + $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}" + if ($pushImages) { + Write-Host "Building and pushing stress test docker image '$imageTag'" + $dockerFile = Get-ChildItem $dockerFilePath + $dockerBuildFolder = if ($pkg.DockerBuildDir) { + Join-Path $pkg.Directory $pkg.DockerBuildDir + } else { + $dockerFile.DirectoryName + } + $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim() + + Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder + if ($LASTEXITCODE) { return } + + Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue + Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue + Write-Host " docker run -it $imageTag " -ForegroundColor DarkBlue + Write-Host "To show installed container images:" -ForegroundColor Blue + Write-Host " docker image ls" -ForegroundColor DarkBlue + Write-Host "To show running containers:" -ForegroundColor Blue + Write-Host " docker ps" -ForegroundColor DarkBlue + + Run docker push $imageTag + if ($LASTEXITCODE) { + if ($login) { + Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" + } + return } - return } + ($genVal.scenarios | Where-Object { + $dockerPath = Join-Path $pkg.Directory $_.image; + [System.IO.Path]::GetFullPath($dockerPath) -eq + $dockerFilePath }).ForEach({$_.imageTag = $imageTag}) + $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile } - Write-Host "Creating namespace $($pkg.Namespace) if it does not exist..." - kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - - if ($LASTEXITCODE) {exit $LASTEXITCODE} - Write-Host "Installing or upgrading stress test $($pkg.ReleaseName) from $($pkg.Directory)" Run helm upgrade $pkg.ReleaseName $pkg.Directory ` -n $pkg.Namespace ` --install ` - --set image=$imageTag ` - --set stress-test-addons.env=$environment + --set stress-test-addons.env=$environment ` + --values generatedValues.yaml if ($LASTEXITCODE) { # Issues like 'UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress' # can be the result of cancelled `upgrade` operations (e.g. ctrl-c). From 63246f949a84918df7d0b85fad88bb7476ab29b2 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 29 Sep 2022 17:20:15 -0700 Subject: [PATCH 4/8] update to default sparse --- eng/common/scripts/stress-testing/deploy-stress-tests.ps1 | 2 +- .../scripts/stress-testing/stress-test-deployment-lib.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index dadf2a05e7..bf51920865 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -27,7 +27,7 @@ param( [System.IO.FileInfo]$LocalAddonsPath, # Matrix generation parameters - [string]$MatrixSelection = "sparse", + [Parameter(Mandatory=$False)][string]$MatrixSelection, [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 9b4cbddfbc..9879882c82 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -80,7 +80,7 @@ function DeployStressTests( return $true })] [System.IO.FileInfo]$LocalAddonsPath, - [string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse", [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, From 3352a48972438ad5f979eff02d23f568f555b5da Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 6 Oct 2022 16:50:41 -0700 Subject: [PATCH 5/8] pr comments and some error handling --- .../job-matrix/job-matrix-functions.ps1 | 6 +- .../find-all-stress-packages.ps1 | 53 ++---------- .../generate-scenario-matrix.ps1 | 86 +++++++++++++++++++ .../stress-test-deployment-lib.ps1 | 68 ++++++++------- 4 files changed, 133 insertions(+), 80 deletions(-) create mode 100644 eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 7c96762be4..27a08ebd6e 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -193,9 +193,9 @@ function ParseFilter([string]$filter) { function GetMatrixConfigFromFile([String] $config) { [MatrixConfig]$config = try{ - GetMatrixConfigFromJson($config) + GetMatrixConfigFromJson $config } catch { - GetMatrixConfigFromYaml($config) + GetMatrixConfigFromYaml $config } return $config } @@ -381,7 +381,7 @@ function ProcessImport([MatrixParameter[]]$matrix, [String]$selection, [Array]$n Write-Error "`$IMPORT path '$importPath' does not exist." exit 1 } - $importedMatrixConfig = GetMatrixConfigFromYaml (Get-Content $importPath) + $importedMatrixConfig = GetMatrixConfigFromFile (Get-Content -Raw $importPath) $importedMatrix = GenerateMatrix ` -config $importedMatrixConfig ` -selectFromMatrixType $selection ` diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index 803dc965b8..b1aaa8db6c 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -13,6 +13,7 @@ class StressTestPackageInfo { } . $PSScriptRoot/../job-matrix/job-matrix-functions.ps1 +. $PSScriptRoot/generate-scenario-matrix.ps1 function FindStressPackages( [string]$directory, @@ -33,9 +34,9 @@ function FindStressPackages( foreach ($chartFile in $chartFiles) { $chart = ParseChart $chartFile if (matchesAnnotations $chart $filters) { - $matrixFilePath = (Join-Path $chartFile.Directory.FullName '/matrix.yaml') + $matrixFilePath = (Join-Path $chartFile.Directory.FullName '/scenarios-matrix.yaml') if (Test-Path $matrixFilePath) { - ScenariosMatrixGeneration ` + GenerateScenarioMatrix ` -matrixFilePath $matrixFilePath ` -Selection $MatrixSelection ` -DisplayNameFilter $MatrixDisplayNameFilter ` @@ -80,50 +81,6 @@ function GetUsername() { return $stressUser.ToLower() } -function ScenariosMatrixGeneration( - [string]$matrixFilePath, - [string]$Selection, - [Parameter(Mandatory=$False)][string]$DisplayNameFilter, - [Parameter(Mandatory=$False)][array]$Filters, - [Parameter(Mandatory=$False)][array]$Replace, - [Parameter(Mandatory=$False)][array]$NonSparseParameters -) { - $yamlConfig = Get-Content $matrixFilePath -Raw - - $prettySerializedMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` - -ConfigPath $matrixFilePath ` - -Selection $Selection ` - -DisplayNameFilter $DisplayNameFilter ` - -Filters $Filters ` - -Replace $Replace ` - -NonSparseParameters $NonSparseParameters - - $prettyMatrix = $prettySerializedMatrix | ConvertFrom-Json -AsHashtable - - $scenariosMatrix = @() - foreach($permutation in $prettyMatrix.GetEnumerator()) { - $entry = @{} - $entry.Name = $permutation.key -replace '_', '-' - $entry.Scenario = $entry.Name - $entry.Remove("Name") - foreach ($param in $permutation.value.GetEnumerator()) { - $entry.add($param.key, $param.value) - } - $scenariosMatrix += $entry - } - - $valuesYaml = Get-Content (Join-Path $matrixFilePath '../values.yaml') -Raw - $values = $valuesYaml | ConvertFrom-Yaml - if (!$values) {$values = @{}} - - if ($values.ContainsKey('Scenarios')) { - throw "Please use matrix generation for stress test scenarios." - } - - $values.scenarios = $scenariosMatrix - $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml') -} - function NewStressTestPackageInfo( [hashtable]$chart, [System.IO.FileInfo]$chartFile, @@ -142,8 +99,8 @@ function NewStressTestPackageInfo( Namespace = $namespace.ToLower() Directory = $chartFile.DirectoryName ReleaseName = $chart.name - Dockerfile = $chart.annotations.dockerfile - DockerBuildDir = $chart.annotations.dockerbuilddir + Dockerfile = "dockerfile" -in $chart.annotations.keys ? $chart.annotations.dockerfile : $null + DockerBuildDir = "dockerbuilddir" -in $chart.annotations.keys ? $chart.annotations.dockerbuilddir : $null } } diff --git a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 new file mode 100644 index 0000000000..9d7028df30 --- /dev/null +++ b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 @@ -0,0 +1,86 @@ +param( + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters +) + +function GenerateScenarioMatrix( + [string]$matrixFilePath, + [string]$Selection, + [Parameter(Mandatory=$False)][string]$DisplayNameFilter, + [Parameter(Mandatory=$False)][array]$Filters, + [Parameter(Mandatory=$False)][array]$Replace, + [Parameter(Mandatory=$False)][array]$NonSparseParameters +) { + $yamlConfig = Get-Content $matrixFilePath -Raw + + $prettySerializedMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` + -ConfigPath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters + + $prettyMatrix = $prettySerializedMatrix | ConvertFrom-Json -AsHashtable + + $scenariosMatrix = @() + foreach($permutation in $prettyMatrix.GetEnumerator()) { + $entry = @{} + $entry.Name = $permutation.key -replace '_', '-' + $entry.Scenario = $entry.Name + $entry.Remove("Name") + foreach ($param in $permutation.value.GetEnumerator()) { + $entry.add($param.key, $param.value) + } + $scenariosMatrix += $entry + } + + $valuesYaml = Get-Content -Raw (Join-Path (Split-Path $matrixFilePath) 'values.yaml') + $values = $valuesYaml | ConvertFrom-Yaml -Ordered + if (!$values) {$values = @{}} + + if ($values.ContainsKey('Scenarios')) { + throw "Please use matrix generation for stress test scenarios." + } + + $values.scenarios = $scenariosMatrix + $values | ConvertTo-Yaml | Out-File -FilePath (Join-Path $matrixFilePath '../generatedValues.yaml') +} + +function NewStressTestPackageInfo( + [hashtable]$chart, + [System.IO.FileInfo]$chartFile, + [switch]$CI, + [object]$namespaceOverride +) { + $namespace = if ($namespaceOverride) { + $namespaceOverride + } elseif ($CI) { + $chart.annotations.namespace + } else { + GetUsername + } + + return [StressTestPackageInfo]@{ + Namespace = $namespace.ToLower() + Directory = $chartFile.DirectoryName + ReleaseName = $chart.name + Dockerfile = $chart.annotations.dockerfile + DockerBuildDir = $chart.annotations.dockerbuilddir + } +} + +# Don't call functions when the script is being dot sourced +if ($MyInvocation.InvocationName -ne ".") { + GenerateScenarioMatrix ` + -matrixFilePath $matrixFilePath ` + -Selection $Selection ` + -DisplayNameFilter $DisplayNameFilter ` + -Filters $Filters ` + -Replace $Replace ` + -NonSparseParameters $NonSparseParameters +} diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 9879882c82..4db81bc9ab 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -186,46 +186,50 @@ function DeployStressPackage( kubectl create namespace $pkg.Namespace --dry-run=client -o yaml | kubectl apply -f - if ($LASTEXITCODE) {exit $LASTEXITCODE} - $dockerFilePaths = @() + $dockerBuildConfigs = @() $genValFile = Join-Path $pkg.Directory "generatedValues.yaml" - $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml + $genVal = Get-Content $genValFile -Raw | ConvertFrom-Yaml -Ordered if (Test-Path $genValFile) { $scenarios = $genVal.Scenarios foreach ($scenario in $scenarios) { - if ($scenario.image) { - $dockerfilePath = Join-Path $pkg.Directory $scenario.image - $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) - $dockerFilePaths += $dockerFilePath + if ("image" -in $scenario.keys) { + $dockerFilePath = Join-Path $pkg.Directory $scenario.image + } else { + $dockerFilePath = "$($pkg.Directory)/Dockerfile" } + $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath).Trim() + + if ("imageBuildDir" -in $scenario.keys) { + $dockerBuildDir = Join-Path $pkg.Directory $scenario.imageBuildDir + } else { + $dockerBuildDir = Split-Path $dockerFilePath + } + $dockerBuildDir = [System.IO.Path]::GetFullPath($dockerBuildDir).Trim() + $dockerBuildConfigs += @{"dockerFilePath"=$dockerFilePath; "dockerBuildDir"=$dockerBuildDir} } - } else { - if ($pkg.Dockerfile) { - $dockerFilePath = Join-Path $pkg.Directory $pkg.Dockerfile - } else { - $dockerFilePath = "$($pkg.Directory)/Dockerfile" - } - $dockerFilePath = [System.IO.Path]::GetFullPath($dockerFilePath) - $dockerFilePaths += $dockerFilePath + } + if ($pkg.Dockerfile -or $pkg.DockerBuildDir) { + throw "The chart.yaml docker config is depracated, please use the scenarios matrix instead." } - foreach ($dockerFilePath in $dockerFilePaths) { - if (!(Test-Path $dockerFilePath)) {continue} - $dockerfileName = ($dockerfilePath -split { $_ -in '\', '/' })[-1].ToLower() + foreach ($dockerBuildConfig in $dockerBuildConfigs) { + $dockerFilePath = $dockerBuildConfig.dockerFilePath + $dockerBuildFolder = $dockerBuildConfig.dockerBuildDir + if (!(Test-Path $dockerFilePath)) { + throw "Invalid dockerfile path, cannot find dockerfile at ${dockerFilePath}" + } + if (!(Test-Path $dockerBuildFolder)) { + throw "Invalid docker build directory, cannot find directory ${dockerBuildFolder}" + } + $dockerfileName = ($dockerFilePath -split { $_ -in '\', '/' })[-1].ToLower() $imageTag = $imageTagBase + "/${dockerfileName}:${deployId}" if ($pushImages) { Write-Host "Building and pushing stress test docker image '$imageTag'" $dockerFile = Get-ChildItem $dockerFilePath - $dockerBuildFolder = if ($pkg.DockerBuildDir) { - Join-Path $pkg.Directory $pkg.DockerBuildDir - } else { - $dockerFile.DirectoryName - } - $dockerBuildFolder = [System.IO.Path]::GetFullPath($dockerBuildFolder).Trim() Run docker build -t $imageTag -f $dockerFile $dockerBuildFolder - if ($LASTEXITCODE) { return } Write-Host "`nContainer image '$imageTag' successfully built. To run commands on the container locally:" -ForegroundColor Blue Write-Host " docker run -it $imageTag" -ForegroundColor DarkBlue @@ -240,13 +244,19 @@ function DeployStressPackage( if ($login) { Write-Warning "If docker push is failing due to authentication issues, try calling this script with '-Login'" } - return } } - ($genVal.scenarios | Where-Object { - $dockerPath = Join-Path $pkg.Directory $_.image; - [System.IO.Path]::GetFullPath($dockerPath) -eq - $dockerFilePath }).ForEach({$_.imageTag = $imageTag}) + $genVal.scenarios = foreach ($scenario in $genVal.scenarios) { + $dockerPath = Join-Path $pkg.Directory $scenario.image + if ("image" -notin $scenario) { + $dockerPath = $dockerFilePath + } + if ([System.IO.Path]::GetFullPath($dockerPath) -eq $dockerFilePath) { + $scenario.imageTag = $imageTag + } + $scenario + } + $genVal | ConvertTo-Yaml | Out-File -FilePath $genValFile } From 10e2d12e9eb712e5212c64c793adeadf98264e0a Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 13 Oct 2022 15:51:46 -0700 Subject: [PATCH 6/8] custom matrixfilename and ordering of generatedValues.yaml --- .../scripts/job-matrix/job-matrix-functions.ps1 | 1 + .../scripts/stress-testing/deploy-stress-tests.ps1 | 1 + .../stress-testing/find-all-stress-packages.ps1 | 7 +++++-- .../stress-testing/generate-scenario-matrix.ps1 | 13 ++++++------- .../stress-testing/stress-test-deployment-lib.ps1 | 2 ++ 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 27a08ebd6e..93bebfdef5 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -202,6 +202,7 @@ function GetMatrixConfigFromFile([String] $config) function GetMatrixConfigFromYaml([String] $yamlConfig) { + Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module # ConvertTo then from json is to make sure the nested values are in PSCustomObject [MatrixConfig]$config = ConvertFrom-Yaml $yamlConfig -Ordered | ConvertTo-Json -Depth 100 | ConvertFrom-Json return GetMatrixConfig $config diff --git a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 index bf51920865..c71e629431 100644 --- a/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 +++ b/eng/common/scripts/stress-testing/deploy-stress-tests.ps1 @@ -27,6 +27,7 @@ param( [System.IO.FileInfo]$LocalAddonsPath, # Matrix generation parameters + [Parameter(Mandatory=$False)][string]$MatrixFileName, [Parameter(Mandatory=$False)][string]$MatrixSelection, [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, diff --git a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 index b1aaa8db6c..f949b03ad8 100644 --- a/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 +++ b/eng/common/scripts/stress-testing/find-all-stress-packages.ps1 @@ -21,6 +21,7 @@ function FindStressPackages( [switch]$CI, [string]$namespaceOverride, [string]$MatrixSelection, + [Parameter(Mandatory=$False)][string]$MatrixFileName, [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, [Parameter(Mandatory=$False)][array]$MatrixReplace, @@ -28,13 +29,15 @@ function FindStressPackages( ) { # Bare minimum filter for stress tests $filters['stressTest'] = 'true' - $packages = @() $chartFiles = Get-ChildItem -Recurse -Filter 'Chart.yaml' $directory + if (!$MatrixFileName) { + $MatrixFileName = '/scenarios-matrix.yaml' + } foreach ($chartFile in $chartFiles) { $chart = ParseChart $chartFile if (matchesAnnotations $chart $filters) { - $matrixFilePath = (Join-Path $chartFile.Directory.FullName '/scenarios-matrix.yaml') + $matrixFilePath = (Join-Path $chartFile.Directory.FullName $MatrixFileName) if (Test-Path $matrixFilePath) { GenerateScenarioMatrix ` -matrixFilePath $matrixFilePath ` diff --git a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 index 9d7028df30..26507386a7 100644 --- a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 +++ b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 @@ -17,24 +17,23 @@ function GenerateScenarioMatrix( ) { $yamlConfig = Get-Content $matrixFilePath -Raw - $prettySerializedMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` + $prettyMatrix = &"$PSScriptRoot/../job-matrix/Create-JobMatrix.ps1" ` -ConfigPath $matrixFilePath ` -Selection $Selection ` -DisplayNameFilter $DisplayNameFilter ` -Filters $Filters ` -Replace $Replace ` -NonSparseParameters $NonSparseParameters - - $prettyMatrix = $prettySerializedMatrix | ConvertFrom-Json -AsHashtable + $prettyMatrix = $prettyMatrix | ConvertFrom-Json $scenariosMatrix = @() - foreach($permutation in $prettyMatrix.GetEnumerator()) { + foreach($permutation in $prettyMatrix.psobject.properties) { $entry = @{} - $entry.Name = $permutation.key -replace '_', '-' + $entry.Name = $permutation.Name -replace '_', '-' $entry.Scenario = $entry.Name $entry.Remove("Name") - foreach ($param in $permutation.value.GetEnumerator()) { - $entry.add($param.key, $param.value) + foreach ($param in $permutation.value.psobject.properties) { + $entry.add($param.Name, $param.value) } $scenariosMatrix += $entry } diff --git a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 index 4db81bc9ab..a082477779 100644 --- a/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 +++ b/eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1 @@ -80,6 +80,7 @@ function DeployStressTests( return $true })] [System.IO.FileInfo]$LocalAddonsPath, + [Parameter(Mandatory=$False)][string]$MatrixFileName, [Parameter(Mandatory=$False)][string]$MatrixSelection = "sparse", [Parameter(Mandatory=$False)][string]$MatrixDisplayNameFilter, [Parameter(Mandatory=$False)][array]$MatrixFilters, @@ -127,6 +128,7 @@ function DeployStressTests( -CI:$CI ` -namespaceOverride $Namespace ` -MatrixSelection $MatrixSelection ` + -MatrixFileName $MatrixFileName ` -MatrixFilters $MatrixFilters ` -MatrixReplace $MatrixReplace ` -MatrixNonSparseParameters $MatrixNonSparseParameters) From 7f5a36a4fd3d69350f90582b7b524a58963f401e Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Wed, 19 Oct 2022 11:03:06 -0700 Subject: [PATCH 7/8] common module import --- eng/common/scripts/job-matrix/job-matrix-functions.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index 93bebfdef5..dfc1fa44b0 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -84,6 +84,7 @@ class MatrixParameter { } } +. (Join-Path $PSScriptRoot "../Helpers" PSModule-Helpers.ps1) $IMPORT_KEYWORD = '$IMPORT' function GenerateMatrix( From 3dcf4e0b27ed05c97d2f4328993345ca5d298d82 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 20 Oct 2022 12:49:37 -0700 Subject: [PATCH 8/8] JobMatrix write host --- eng/common/scripts/Helpers/PSModule-Helpers.ps1 | 6 +++--- eng/common/scripts/job-matrix/Create-JobMatrix.ps1 | 2 +- eng/common/scripts/job-matrix/job-matrix-functions.ps1 | 4 ++-- .../scripts/stress-testing/generate-scenario-matrix.ps1 | 1 + 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 index 1a2a0a9008..d9a5afaab1 100644 --- a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -22,9 +22,9 @@ function Update-PSModulePathForCI() $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) }) # Add any "az_" paths from the agent which is the lastest set of azure modules - $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator - if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) { - $modulePaths += $AzModuleCachPath + $AzModuleCachePath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator + if ($AzModuleCachePath -and $env:PSModulePath -notcontains $AzModuleCachePath) { + $modulePaths += $AzModuleCachePath } $env:PSModulePath = $modulePaths -join $moduleSeperator diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 index 3f14b9a1d9..ea4923daa4 100644 --- a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -41,4 +41,4 @@ Write-Output $serialized.pretty if ($null -ne $env:SYSTEM_TEAMPROJECTID) { Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" -} +} \ No newline at end of file diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 index dfc1fa44b0..0926185910 100644 --- a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -147,7 +147,7 @@ function ProcessNonSparseParameters( function FilterMatrixDisplayName([array]$matrix, [string]$filter) { return $matrix | Where-Object { $_ } | ForEach-Object { - if ($_.Name -match $filter) { + if ($_.ContainsKey("Name") -and $_.Name -match $filter) { return $_ } } @@ -169,7 +169,7 @@ function MatchesFilters([hashtable]$entry, [array]$filters) { # Default all regex checks to go against empty string when keys are missing. # This simplifies the filter syntax/interface to be regex only. $value = "" - if ($null -ne $entry -and $entry.parameters.Contains($key)) { + if ($null -ne $entry -and $entry.ContainsKey("parameters") -and $entry.parameters.Contains($key)) { $value = $entry.parameters[$key] } if ($value -notmatch $regex) { diff --git a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 index 26507386a7..aba8be3451 100644 --- a/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 +++ b/eng/common/scripts/stress-testing/generate-scenario-matrix.ps1 @@ -24,6 +24,7 @@ function GenerateScenarioMatrix( -Filters $Filters ` -Replace $Replace ` -NonSparseParameters $NonSparseParameters + Write-Host $prettyMatrix $prettyMatrix = $prettyMatrix | ConvertFrom-Json $scenariosMatrix = @()