From c770daa2a646766323f720d5e2db9d3abd51ed91 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 19 Feb 2021 17:52:45 -0500 Subject: [PATCH] Add job matrix generation scripts (#1429) This PR is a port of functionality that is currently duplicated across the net/java/python repositories. The intent was to settle on an implementation before moving it to the /eng/common/scripts directory. After merge to this location, I'll update the net/java/python and js repos to point to the `eng/common/scripts` location and remove the scripts from `eng/scripts`. Here is the PR text used against the other repos for reference: This adds scripts, docs and samples supporting dynamic, cross-product matrix generation for azure pipeline jobs. It aims to replicate the [cross-product matrix functionality in github actions](https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-running-with-more-than-one-version-of-nodejs), but also adds some additional features like sparse matrix generation, cross-product includes and excludes, parameter grouping and matrix filters. This functionality is made possible by the ability for the azure pipelines yaml to take a [dynamic variable as an input for a job matrix definition](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration) (see the code sample at the bottom of the linked section). See the README.md file for more details on the config file syntax and usage, as well as implementation details. The tests (`test-matrix-functions.tests.ps1`) contain a lot of detail on expected data structures at various processing stages. The `-`test-matrix-functions.ps1` file could perhaps be split up or use some more organization, so let me know if it's hard to navigate. Example: ``` { "displayNames": { "true": "TestFromSource" }, "matrix": { "Agent": { "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }, "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }, "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } }, "JavaTestVersion": [ "1.8", "1.11" ], "AZURE_TEST_HTTP_CLIENTS": [ "okhttp", "netty" ] }, "include": [ { "Agent": { "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" } }, "JavaTestVersion": "1.11", "AZURE_TEST_HTTP_CLIENTS": "netty", "TestFromSource": true } ] } ``` Sparse matrix job generation in a pipeline: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=705622&view=results ![image](https://user-images.githubusercontent.com/1020379/106040177-151e1f80-60a8-11eb-823c-2af96b5e84aa.png) Related discussion: https://github.com/microsoft/azure-pipelines-yaml/issues/20 --- .../jobs/archetype-sdk-tests-generate.yml | 73 +++ .../scripts/job-matrix/Create-JobMatrix.ps1 | 35 ++ eng/common/scripts/job-matrix/README.md | 461 ++++++++++++++ .../job-matrix/job-matrix-functions.ps1 | 553 +++++++++++++++++ .../job-matrix/samples/matrix-job-sample.yml | 42 ++ .../job-matrix/samples/matrix-test.yml | 26 + .../scripts/job-matrix/samples/matrix.json | 28 + .../job-matrix-functions.filter.tests.ps1 | 84 +++ ...ob-matrix-functions.modification.tests.ps1 | 219 +++++++ .../tests/job-matrix-functions.tests.ps1 | 562 ++++++++++++++++++ .../job-matrix/tests/test-import-matrix.json | 11 + 11 files changed, 2094 insertions(+) create mode 100644 eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml create mode 100644 eng/common/scripts/job-matrix/Create-JobMatrix.ps1 create mode 100644 eng/common/scripts/job-matrix/README.md create mode 100644 eng/common/scripts/job-matrix/job-matrix-functions.ps1 create mode 100644 eng/common/scripts/job-matrix/samples/matrix-job-sample.yml create mode 100644 eng/common/scripts/job-matrix/samples/matrix-test.yml create mode 100644 eng/common/scripts/job-matrix/samples/matrix.json create mode 100644 eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 create mode 100644 eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 create mode 100644 eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 create mode 100644 eng/common/scripts/job-matrix/tests/test-import-matrix.json diff --git a/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml new file mode 100644 index 00000000000..cbe0c6bb6af --- /dev/null +++ b/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml @@ -0,0 +1,73 @@ +parameters: +- name: AdditionalParameters + type: object +- name: CloudConfig + type: object + default: {} +- name: MatrixConfigs + type: object + default: [] +- name: MatrixFilters + type: object + default: [] +- name: JobTemplatePath + type: string + +jobs: +- job: generate_matrix + variables: + displayNameFilter: $[ coalesce(variables.jobMatrixFilter, '.*') ] + pool: + name: Azure Pipelines + vmImage: ubuntu-18.04 + displayName: Generate Job Matrix + steps: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter "$(displayNameFilter)" + -Filters "${{ join('","', parameters.MatrixFilters) }}","container=^$","SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}" + -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}" + displayName: Generate VM Job Matrix ${{ config.Name }} + name: generate_vm_job_matrix_${{ config.Name }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter "$(displayNameFilter)" + -Filters "${{ join('","', parameters.MatrixFilters) }}", "container=.*", "SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}" + -NonSparseParameters "${{ join('","', config.NonSparseParameters) }}" + displayName: Generate Container Job Matrix + name: generate_container_job_matrix_${{ config.Name }} + +- ${{ each config in parameters.MatrixConfigs }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + Matrix: dependencies.generate_matrix.outputs['generate_vm_job_matrix_${{ config.Name }}.matrix'] + DependsOn: generate_matrix + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: true + Matrix: dependencies.generate_matrix.outputs['generate_container_job_matrix_${{ config.Name }}.matrix'] + DependsOn: generate_matrix + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} diff --git a/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 new file mode 100644 index 00000000000..3e075c3e04e --- /dev/null +++ b/eng/common/scripts/job-matrix/Create-JobMatrix.ps1 @@ -0,0 +1,35 @@ +<# + .SYNOPSIS + Generates a JSON object representing an Azure Pipelines Job Matrix. + See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec + + .EXAMPLE + ./eng/common/scripts/Create-JobMatrix $context +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory=$True)][string] $ConfigPath, + [Parameter(Mandatory=$True)][string] $Selection, + [Parameter(Mandatory=$False)][string] $DisplayNameFilter, + [Parameter(Mandatory=$False)][array] $Filters, + [Parameter(Mandatory=$False)][array] $NonSparseParameters +) + +. $PSScriptRoot/job-matrix-functions.ps1 + +$config = GetMatrixConfigFromJson (Get-Content $ConfigPath) +# Strip empty string filters in order to be able to use azure pipelines yaml join() +$Filters = $Filters | Where-Object { $_ } + +[array]$matrix = GenerateMatrix ` + -config $config ` + -selectFromMatrixType $Selection ` + -displayNameFilter $DisplayNameFilter ` + -filters $Filters ` + -nonSparseParameters $NonSparseParameters + +$serialized = SerializePipelineMatrix $matrix + +Write-Output $serialized.pretty +Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" diff --git a/eng/common/scripts/job-matrix/README.md b/eng/common/scripts/job-matrix/README.md new file mode 100644 index 00000000000..4f468e4ccb3 --- /dev/null +++ b/eng/common/scripts/job-matrix/README.md @@ -0,0 +1,461 @@ +# Azure Pipelines Matrix Generator + +* [Usage in a pipeline](#usage-in-a-pipeline) +* [Matrix config file syntax](#matrix-config-file-syntax) + * [Fields](#fields) + * [matrix](#matrix) + * [include](#include) + * [exclude](#exclude) + * [displayNames](#displaynames) + * [$IMPORT](#import) +* [Matrix Generation behavior](#matrix-generation-behavior) + * [all](#all) + * [sparse](#sparse) + * [include/exclude](#includeexclude) + * [displayNames](#displaynames-1) + * [Filters](#filters) + * [NonSparseParameters](#nonsparseparameters) + * [Under the hood](#under-the-hood) +* [Testing](#testing) + + +This directory contains scripts supporting dynamic, cross-product matrix generation for azure pipeline jobs. +It aims to replicate the [cross-product matrix functionality in github actions](https://docs.github.com/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-running-with-more-than-one-version-of-nodejs), +but also adds some additional features like sparse matrix generation, cross-product includes and excludes, and programmable matrix filters. + +This functionality is made possible by the ability for the azure pipelines yaml to take a [dynamic variable as an input +for a job matrix definition](https://docs.microsoft.com/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration) (see the code sample at the bottom of the linked section). + +## Usage in a pipeline + +In order to use these scripts in a pipeline, you must provide a config file and call the matrix creation script within a powershell job. + +For a single matrix, you can include the `/eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml` template in a pipeline (see /eng/common/scripts/job-matrix/samples/matrix-test.yml for a full working example): + +``` +jobs: + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + parameters: + MatrixConfigs: + - Name: base_product_matrix + Path: eng/scripts/job-matrix/samples/matrix.json + Selection: all + NonSparseParameters: + - framework + GenerateVMJobs: true + - Name: sparse_product_matrix + Path: eng/scripts/job-matrix/samples/matrix.json + Selection: sparse + GenerateVMJobs: true + JobTemplatePath: /eng/common/scripts/job-matrix/samples/matrix-job-sample.yml + AdditionalParameters: [] + CloudConfig: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + Location: eastus2 + Cloud: Public + MatrixFilters: [] +``` + +## Matrix config file syntax + +Matrix parameters can either be a list of strings, or a set of grouped strings (represented as a hash). The latter parameter +type is useful for when 2 or more parameters need to be grouped together, but without generating more than one matrix permutation. + +``` +"matrix": { + "": [ ], + "": [ ], + "": { + "": { + "", + }, + "": { + "", + } + } +} +"include": [ , , ... ], +"exclude": [ , , ... ], +"displayNames": { : } +``` + +See `samples/matrix.json` for a full sample. + +### Fields + +#### matrix + +The `matrix` field defines the base cross-product matrix. The generated matrix can be full or sparse. + +Example: +``` +"matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1", + "net50" + ], + "additionalTestArguments": [ + "", + "/p:UseProjectReferenceToAzureClients=true", + ] +} +``` + +#### include + +The `include` field defines any number of matrices to be appended to the base matrix after processing exclusions. + +#### exclude + +The `include` field defines any number of matrices to be removed from the base matrix. Exclude parameters can be a partial +set, meaning as long as all exclude parameters match against a matrix entry (even if the matrix entry has additional parameters), +then it will be excluded from the matrix. For example, the below entry will match the exclusion and be removed: + +``` +matrix entry: +{ + "a": 1, + "b": 2, + "c": 3, +} + +"exclude": [ + { + "a": 1, + "b": 2 + } +] +``` + +#### displayNames + +Specify any overrides for the azure pipelines definition and UI that determines the matrix job name. If some parameter +values are too long or unreadable for this purpose (e.g. a command line argument), then you can replace them with a more +readable value here. For example: + +``` +"displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" +}, +"matrix": { + "additionalTestArguments": [ + "/p:UseProjectReferenceToAzureClients=true" + ] +} +``` + +#### $IMPORT + +Matrix configs can also import another matrix config. The effect of this is the imported matrix will be generated, +and then the importing config will be combined with that matrix (as if each entry of the imported matrix was a parameter). +To import a matrix, add a parameter with the key `$IMPORT`: + +``` +"matrix": { + "$IMPORT": "path/to/matrix.json", + "JavaVersion": [ "1.8", "1.11" ] +} +``` + +Importing can be useful, for example, in cases where there is a shared base matrix, but there is a need to run it +once for each instance of a language version. + +The processing order is as follows: + +Given a matrix and import matrix like below: +``` +{ + "matrix": { + "$IMPORT": "example-matrix.json", + "endpointType": [ "storage", "cosmos" ], + "JavaVersion": [ "1.8", "1.11" ] + }, + "include": [ + { + "operatingSystem": "windows", + "mode": "TestFromSource", + "JavaVersion": "1.8" + } + ] +} + +### example-matrix.json to import +{ + "matrix": { + "operatingSystem": [ "windows", "linux" ], + "client": [ "netty", "okhttp" ] + }, + "include": [ + { + "operatingSystem": "mac", + "client": "netty" + } + ] +} +``` + +1. The base matrix is generated (sparse in this example): + ``` + { + "storage_18": { + "endpointType": "storage", + "JavaVersion": "1.8" + }, + "cosmos_111": { + "endpointType": "cosmos", + "JavaVersion": "1.11" + } + } + ``` +1. The imported base matrix is generated (sparse in this example): + ``` + { + "windows_netty": { + "operatingSystem": "windows", + "client": "netty" + }, + "linux_okhttp": { + "operatingSystem": "linux", + "client": "okhttp" + } + } + ``` +1. Includes/excludes from the imported matrix get applied to the imported matrix + ``` + { + "windows_netty": { + "operatingSystem": "windows", + "client": "netty" + }, + "linux_okhttp": { + "operatingSystem": "linux", + "client": "okhttp" + }, + "mac_netty": { + "operatingSystem": "mac", + "client": "netty" + } + } + ``` +1. The base matrix is multipled by the imported matrix (in this case, the base matrix has 2 elements, and the imported + matrix has 3 elements, so the product is a matrix with 6 elements: + ``` + "storage_18_windows_netty": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "windows", + "client": "netty" + }, + "storage_18_linux_okhttp": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "linux", + "client": "okhttp" + }, + "storage_18_mac_netty": { + "endpointType": "storage", + "JavaVersion": "1.8", + "operatingSystem": "mac", + "client": "netty" + }, + "cosmos_111_windows_netty": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "windows", + "client": "netty" + }, + "cosmos_111_linux_okhttp": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "linux", + "client": "okhttp" + }, + "cosmos_111_mac_netty": { + "endpointType": "cosmos", + "JavaVersion": "1.11", + "operatingSystem": "mac", + "client": "netty" + } + } + ``` +1. Includes/excludes from the top-level matrix get applied to the multiplied matrix, so the below element will be added + to the above matrix, for an output matrix with 7 elements: + ``` + "windows_TestFromSource_18": { + "operatingSystem": "windows", + "mode": "TestFromSource", + "JavaVersion": "1.8" + } + ``` + +## Matrix Generation behavior + +#### all + +`all` will output the full matrix, i.e. every possible permutation of all parameters given (p1.Length * p2.Length * ...). + +#### sparse + +`sparse` outputs the minimum number of parameter combinations while ensuring that all parameter values are present in at least one matrix job. +Effectively this means the total length of a sparse matrix will be equal to the largest matrix dimension, i.e. `max(p1.Length, p2.Length, ...)`. + +To build a sparse matrix, a full matrix is generated, and then walked diagonally N times where N is the largest matrix dimension. +This pattern works for any N-dimensional matrix, via an incrementing index (n, n, n, ...), (n+1, n+1, n+1, ...), etc. +Index lookups against matrix dimensions are calculated modulus the dimension size, so a two-dimensional matrix of 4x2 might be walked like this: + +``` +index: 0, 0: +o . . . +. . . . + +index: 1, 1: +. . . . +. o . . + +index: 2, 2 (modded to 2, 0): +. . o . +. . . . + +index: 3, 3 (modded to 3, 1): +. . . . +. . . o +``` + +#### include/exclude + +Include and exclude support additions and subtractions off the base matrix. Both include and exclude take an array of matrix values. +Typically these values will be a single entry, but they also support the cross-product matrix definition syntax of the base matrix. + +Include and exclude are parsed fully. So if a sparse matrix is called for, a sparse version of the base matrix will be generated, but +the full matrix of both include and exclude will be processed. + +Excludes are processed first, so includes can be used to add back any specific jobs to the matrix. + +#### displayNames + +In the matrix job output that azure pipelines consumes, the format is a dictionary of dictionaries. For example: + +``` +{ + "net461_macOS1015": { + "framework": "net461", + "operatingSystem": "macOS-10.15" + }, + "net50_ubuntu1804": { + "framework": "net50", + "operatingSystem": "ubuntu-18.04" + }, + "netcoreapp21_windows2019": { + "framework": "netcoreapp2.1", + "operatingSystem": "windows-2019" + }, + "UseProjectRef_net461_windows2019": { + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true", + "framework": "net461", + "operatingSystem": "windows-2019" + } +} +``` + +The top level keys are used as job names, meaning they get displayed in the azure pipelines UI when running the pipeline. + +The logic for generating display names works like this: + +- Join parameter values by "_" + a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value. + b. For each name value, strip all non-alphanumeric characters (excluding "_"). + c. If the name is greater than 100 characters, truncate it. + +#### Filters + +Filters can be passed to the matrix as an array of strings, each matching the format of =. When a matrix entry +does not contain the specified key, it will default to a value of empty string for regex parsing. This can be used to specify +filters for keys that don't exist or keys that optionally exist and match a regex, as seen in the below example. + +Display name filters can also be passed as a single regex string that runs against the [generated display name](#displaynames) of the matrix job. +The intent of display name filters is to be defined primarily as a top level variable at template queue time in the azure pipelines UI. + +For example, the below command will filter for matrix entries with "windows" in the job display name, no matrix variable +named "ExcludedKey", a framework variable containing either "461" or "5.0", and an optional key "SupportedClouds" that, if exists, must contain "Public": + +``` +./Create-JobMatrix.ps1 ` + -ConfigPath samples/matrix.json ` + -Selection all ` + -DisplayNameFilter ".*windows.*" ` + -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*") +``` + +#### NonSparseParameters + +Sometimes it may be necessary to generate a sparse matrix, but keep the full combination of a few parameters. The +NonSparseParameters argument allows for more fine-grained control of matrix generation. For example: + +``` +./Create-JobMatrix.ps1 ` + -ConfigPath /path/to/matrix.json ` + -Selection sparse ` + -NonSparseParameters @("JavaTestVersion") +``` + +Given a matrix like below with `JavaTestVersion` marked as a non-sparse parameter: + +``` +{ + "matrix": { + "Agent": { + "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }, + "ubuntu-1804": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }, + "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + }, + "JavaTestVersion": [ "1.8", "1.11" ], + "AZURE_TEST_HTTP_CLIENTS": "netty", + "ArmTemplateParameters": [ "@{endpointType='storage'}", "@{endpointType='cosmos'}" ] + } +} +``` + +A matrix with 6 entries will be generated: A sparse matrix of Agent, AZURE_TEST_HTTP_CLIENTS and ArmTemplateParameters +(3 total entries) will be multipled by the two `JavaTestVersion` parameters `1.8` and `1.11`. + +#### Under the hood + +The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, +the below config would generate a 2x2x1x1x1 matrix (five-dimensional): + +``` +"matrix": { + "framework": [ "net461", "netcoreapp2.1" ], + "additionalTestArguments": [ "", "/p:SuperTest=true" ] + "pool": [ "ubuntu-18.04" ], + "container": [ "ubuntu-18.04" ], + "testMode": [ "Record" ] +} +``` + +The matrix is stored as a one-dimensional array, with a row-major indexing scheme (e.g. `(2, 1, 0, 1, 0)`). + +## Testing + +The matrix functions can be tested using [pester](https://pester.dev/). The test command must be run from within the tests directory. + +``` +$ cd tests +$ Invoke-Pester + +Starting discovery in 3 files. +Discovery finished in 75ms. +[+] /home/ben/sdk/azure-sdk-tools/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 750ms (309ms|428ms) +[+] /home/ben/sdk/azure-sdk-tools/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 867ms (250ms|608ms) +[+] /home/ben/sdk/azure-sdk-tools/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 2.71s (725ms|1.93s) +Tests completed in 4.33s +Tests Passed: 141, Failed: 0, Skipped: 4 NotRun: 0 +``` diff --git a/eng/common/scripts/job-matrix/job-matrix-functions.ps1 b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 new file mode 100644 index 00000000000..07b2363765c --- /dev/null +++ b/eng/common/scripts/job-matrix/job-matrix-functions.ps1 @@ -0,0 +1,553 @@ +Set-StrictMode -Version "4.0" + +class MatrixConfig { + [PSCustomObject]$displayNames + [Hashtable]$displayNamesLookup + [PSCustomObject]$matrix + [System.Collections.Specialized.OrderedDictionary]$orderedMatrix + [Array]$include + [Array]$exclude +} + +$IMPORT_KEYWORD = '$IMPORT' + +function CreateDisplayName([string]$parameter, [Hashtable]$displayNamesLookup) +{ + $name = $parameter.ToString() + + if ($displayNamesLookup.ContainsKey($parameter)) { + $name = $displayNamesLookup[$parameter] + } + + # Matrix naming restrictions: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration + $name = $name -replace "[^A-Za-z0-9_]", "" + return $name +} + +function GenerateMatrix( + [MatrixConfig]$config, + [String]$selectFromMatrixType, + [String]$displayNameFilter = ".*", + [Array]$filters = @(), + [Array]$nonSparseParameters = @() +) { + $orderedMatrix, $importedMatrix = ProcessImport $config.orderedMatrix $selectFromMatrixType + if ($selectFromMatrixType -eq "sparse") { + [Array]$matrix = GenerateSparseMatrix $orderedMatrix $config.displayNamesLookup $nonSparseParameters + } elseif ($selectFromMatrixType -eq "all") { + [Array]$matrix = GenerateFullMatrix $orderedMatrix $config.displayNamesLookup + } else { + throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)" + } + + # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the + # top level and imported sparse matrices (as opposed to a sparse selection of both matrices). + if ($importedMatrix) { + [Array]$matrix = CombineMatrices $matrix $importedMatrix + } + + if ($config.exclude) { + [Array]$matrix = ProcessExcludes $matrix $config.exclude + } + if ($config.include) { + [Array]$matrix = ProcessIncludes $config $matrix $selectFromMatrixType + } + + [Array]$matrix = FilterMatrixDisplayName $matrix $displayNameFilter + [Array]$matrix = FilterMatrix $matrix $filters + return $matrix +} + +function ProcessNonSparseParameters( + [System.Collections.Specialized.OrderedDictionary]$parameters, + [Array]$nonSparseParameters +) { + if (!$nonSparseParameters) { + return $parameters, $null + } + + $sparse = [ordered]@{} + $nonSparse = [ordered]@{} + + foreach ($param in $parameters.GetEnumerator()) { + if ($param.Name -in $nonSparseParameters) { + $nonSparse[$param.Name] = $param.Value + } else { + $sparse[$param.Name] = $param.Value + } + } + + return $sparse, $nonSparse +} + +function FilterMatrixDisplayName([array]$matrix, [string]$filter) { + return $matrix | ForEach-Object { + if ($_.Name -match $filter) { + return $_ + } + } +} + +# Filters take the format of key=valueregex,key2=valueregex2 +function FilterMatrix([array]$matrix, [array]$filters) { + $matrix = $matrix | ForEach-Object { + if (MatchesFilters $_ $filters) { + return $_ + } + } + return $matrix +} + +function MatchesFilters([hashtable]$entry, [array]$filters) { + foreach ($filter in $filters) { + $key, $regex = ParseFilter $filter + # Default all regex checks to go against empty string when keys are missing. + # This simplifies the filter syntax/interface to be regex only. + $value = "" + if ($null -ne $entry -and $entry.parameters.Contains($key)) { + $value = $entry.parameters[$key] + } + if ($value -notmatch $regex) { + return $false + } + } + + return $true +} + +function ParseFilter([string]$filter) { + # Lazy match key in case value contains '=' + if ($filter -match "(.+?)=(.+)") { + $key = $matches[1] + $regex = $matches[2] + return $key, $regex + } else { + throw "Invalid filter: `"${filter}`", expected = format" + } +} + +# Importing the JSON as PSCustomObject preserves key ordering, +# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfigFromJson([String]$jsonConfig) +{ + [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + $config.orderedMatrix = [ordered]@{} + $config.displayNamesLookup = @{} + + if ($null -ne $config.matrix) { + $config.matrix.PSObject.Properties | ForEach-Object { + $config.orderedMatrix.Add($_.Name, $_.Value) + } + } + if ($null -ne $config.displayNames) { + $config.displayNames.PSObject.Properties | ForEach-Object { + $config.displayNamesLookup.Add($_.Name, $_.Value) + } + } + $config.include = $config.include | Where-Object { $null -ne $_ } | ForEach-Object { + $ordered = [ordered]@{} + $_.PSObject.Properties | ForEach-Object { + $ordered.Add($_.Name, $_.Value) + } + return $ordered + } + $config.exclude = $config.exclude | Where-Object { $null -ne $_ } | ForEach-Object { + $ordered = [ordered]@{} + $_.PSObject.Properties | ForEach-Object { + $ordered.Add($_.Name, $_.Value) + } + return $ordered + } + + return $config +} + +function ProcessExcludes([Array]$matrix, [Array]$excludes) +{ + $deleteKey = "%DELETE%" + $exclusionMatrix = @() + + foreach ($exclusion in $excludes) { + $full = GenerateFullMatrix $exclusion + $exclusionMatrix += $full + } + + foreach ($element in $matrix) { + foreach ($exclusion in $exclusionMatrix) { + $match = MatrixElementMatch $element.parameters $exclusion.parameters + if ($match) { + $element.parameters[$deleteKey] = $true + } + } + } + + return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } +} + +function ProcessIncludes([MatrixConfig]$config, [Array]$matrix) +{ + $inclusionMatrix = @() + foreach ($inclusion in $config.include) { + $full = GenerateFullMatrix $inclusion $config.displayNamesLookup + $inclusionMatrix += $full + } + + return $matrix + $inclusionMatrix +} + +function ProcessImport([System.Collections.Specialized.OrderedDictionary]$matrix, [String]$selection) +{ + if (!$matrix -or !$matrix.Contains($IMPORT_KEYWORD)) { + return $matrix + } + + $importPath = $matrix[$IMPORT_KEYWORD] + $matrix.Remove($IMPORT_KEYWORD) + + $matrixConfig = GetMatrixConfigFromJson (Get-Content $importPath) + $importedMatrix = GenerateMatrix $matrixConfig $selection + + return $matrix, $importedMatrix +} + +function CombineMatrices([Array]$matrix1, [Array]$matrix2) +{ + $combined = @() + if (!$matrix1) { + return $matrix2 + } + if (!$matrix2) { + return $matrix1 + } + + foreach ($entry1 in $matrix1) { + foreach ($entry2 in $matrix2) { + $newEntry = @{ + name = $entry1.name + parameters = CloneOrderedDictionary $entry1.parameters + } + foreach($param in $entry2.parameters.GetEnumerator()) { + if (!$newEntry.Contains($param.Name)) { + $newEntry.parameters[$param.Name] = $param.Value + } else { + Write-Warning "Skipping duplicate parameter `"$($param.Name)`" when combining matrix." + } + } + + # The maximum allowed matrix name length is 100 characters + $entry2.name = $entry2.name.TrimStart("job_") + $newEntry.name = $newEntry.name, $entry2.name -join "_" + if ($newEntry.name.Length -gt 100) { + $newEntry.name = $newEntry.name[0..99] -join "" + } + + $combined += $newEntry + } + } + + return $combined +} + +function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) +{ + if ($target.Count -eq 0) { + return $false + } + + foreach ($key in $target.Keys) { + if (!$source.Contains($key) -or $source[$key] -ne $target[$key]) { + return $false + } + } + + return $true +} + +function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) { + $newDictionary = [Ordered]@{} + foreach ($element in $dictionary.GetEnumerator()) { + $newDictionary[$element.Name] = $element.Value + } + return $newDictionary +} + +function SerializePipelineMatrix([Array]$matrix) +{ + $pipelineMatrix = [Ordered]@{} + foreach ($entry in $matrix) { + $pipelineMatrix.Add($entry.name, [Ordered]@{}) + foreach ($key in $entry.parameters.Keys) { + $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) + } + } + + return @{ + compressed = $pipelineMatrix | ConvertTo-Json -Compress ; + pretty = $pipelineMatrix | ConvertTo-Json; + } +} + +function GenerateSparseMatrix( + [System.Collections.Specialized.OrderedDictionary]$parameters, + [Hashtable]$displayNamesLookup, + [Array]$nonSparseParameters = @() +) { + $parameters, $nonSparse = ProcessNonSparseParameters $parameters $nonSparseParameters + [Array]$dimensions = GetMatrixDimensions $parameters + [Array]$matrix = GenerateFullMatrix $parameters $displayNamesLookup + + $sparseMatrix = @() + $indexes = GetSparseMatrixIndexes $dimensions + foreach ($idx in $indexes) { + $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + } + + if ($nonSparse) { + [Array]$allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup + return CombineMatrices $allOfMatrix $sparseMatrix + } + + return $sparseMatrix +} + +function GetSparseMatrixIndexes([Array]$dimensions) +{ + $size = ($dimensions | Measure-Object -Maximum).Maximum + $indexes = @() + + # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. + # For example, given a matrix with dimensions 3, 2, 2: + # 0, 0, 0 + # 1, 1, 1 + # 2, 2, 2 + # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions + for ($i = 0; $i -lt $size; $i++) { + $idx = @() + for ($j = 0; $j -lt $dimensions.Length; $j++) { + $idx += $i % $dimensions[$j] + } + $indexes += ,$idx + } + + return $indexes +} + +function GenerateFullMatrix( + [System.Collections.Specialized.OrderedDictionary] $parameters, + [Hashtable]$displayNamesLookup = @{} +) { + # Handle when the config does not have a matrix specified (e.g. only the include field is specified) + if ($parameters.Count -eq 0) { + return @() + } + + $parameterArray = $parameters.GetEnumerator() | ForEach-Object { $_ } + + $matrix = [System.Collections.ArrayList]::new() + InitializeMatrix $parameterArray $displayNamesLookup $matrix + + return $matrix +} + +function CreateMatrixEntry([System.Collections.Specialized.OrderedDictionary]$permutation, [Hashtable]$displayNamesLookup = @{}) +{ + $names = @() + $splattedParameters = [Ordered]@{} + + foreach ($entry in $permutation.GetEnumerator()) { + $nameSegment = "" + + if ($entry.Value -is [PSCustomObject]) { + $nameSegment = CreateDisplayName $entry.Name $displayNamesLookup + foreach ($toSplat in $entry.Value.PSObject.Properties) { + $splattedParameters.Add($toSplat.Name, $toSplat.Value) + } + } else { + $nameSegment = CreateDisplayName $entry.Value $displayNamesLookup + $splattedParameters.Add($entry.Name, $entry.Value) + } + + if ($nameSegment) { + $names += $nameSegment + } + } + + # The maximum allowed matrix name length is 100 characters + $name = $names -join "_" + if ($name.Length -gt 100) { + $name = $name[0..99] -join "" + } + $stripped = $name -replace "^[^A-Za-z]*", "" # strip leading digits + if ($stripped -eq "") { + $name = "job_" + $name # Handle names that consist entirely of numbers + } else { + $name = $stripped + } + + return @{ + name = $name + parameters = $splattedParameters + } +} + +function InitializeMatrix +{ + param( + [Array]$parameters, + [Hashtable]$displayNamesLookup, + [System.Collections.ArrayList]$permutations, + $permutation = [Ordered]@{} + ) + $head, $tail = $parameters + + if (!$head) { + $entry = CreateMatrixEntry $permutation $displayNamesLookup + $permutations.Add($entry) | Out-Null + return + } + + # This behavior implicitly treats non-array values as single elements + foreach ($value in $head.Value) { + $newPermutation = CloneOrderedDictionary $permutation + if ($value -is [PSCustomObject]) { + foreach ($nestedParameter in $value.PSObject.Properties) { + $nestedPermutation = CloneOrderedDictionary $newPermutation + $nestedPermutation[$nestedParameter.Name] = $nestedParameter.Value + InitializeMatrix $tail $displayNamesLookup $permutations $nestedPermutation + } + } else { + $newPermutation[$head.Name] = $value + InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation + } + } +} + +function GetMatrixDimensions([System.Collections.Specialized.OrderedDictionary]$parameters) +{ + $dimensions = @() + foreach ($param in $parameters.GetEnumerator()) { + if ($param.Value -is [PSCustomObject]) { + $dimensions += ($param.Value.PSObject.Properties | Measure-Object).Count + } elseif ($param.Value -is [Array]) { + $dimensions += $param.Value.Length + } else { + $dimensions += 1 + } + } + + return $dimensions +} + +function SetNdMatrixElement +{ + param( + $element, + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" + } + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + $matrix[$arrayIndex] = $element +} + +function GetNdMatrixArrayIndex +{ + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" + } + + $stride = 1 + # Commented out does lookup with wrap handling + # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] + $index = $idx[$idx.Length-1] + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + # Commented out does lookup with wrap handling + # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride + $index += $idx[$i-1] * $stride + } + + return $index +} + +function GetNdMatrixElement +{ + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + return $matrix[$arrayIndex] +} + +function GetNdMatrixIndex +{ + param( + [int]$index, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $matrixIndex = @() + $stride = 1 + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + $page = [math]::floor($index / $stride) % $dimensions[$i-1] + $matrixIndex = ,$page + $matrixIndex + } + $col = $index % $dimensions[$dimensions.Length-1] + $matrixIndex += $col + + return $matrixIndex +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +# The below functions are non-dynamic examples that # +# help explain the above N-dimensional algorithm # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) +{ + $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] + $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] + $stride3 = $idx[2] * $dimensions[3] + $stride4 = $idx[3] + + return $matrix[$stride1 + $stride2 + $stride3 + $stride4] +} + +function Get4dMatrixIndex([int]$index, [Array]$dimensions) +{ + $stride1 = $dimensions[3] + $stride2 = $dimensions[2] + $stride3 = $dimensions[1] + $page1 = [math]::floor($index / $stride1) % $dimensions[2] + $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] + $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] + $remainder = $index % $dimensions[3] + + return @($page3, $page2, $page1, $remainder) +} + diff --git a/eng/common/scripts/job-matrix/samples/matrix-job-sample.yml b/eng/common/scripts/job-matrix/samples/matrix-job-sample.yml new file mode 100644 index 00000000000..9d7c8c9a789 --- /dev/null +++ b/eng/common/scripts/job-matrix/samples/matrix-job-sample.yml @@ -0,0 +1,42 @@ +parameters: +- name: CloudConfig + type: object +- name: Matrix + type: string +- name: DependsOn + type: string + default: '' +- name: UsePlatformContainer + type: boolean + default: false + +jobs: + - job: + dependsOn: ${{ parameters.DependsOn }} + condition: ne(${{ parameters.Matrix }}, '{}') + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + + pool: + name: $(Pool) + vmImage: $(OSVmImage) + + ${{ if eq(parameters.UsePlatformContainer, 'true') }}: + container: $[ variables['Container'] ] + + steps: + - pwsh: | + Write-Output "MATRIX JOB PARAMETERS" + Write-Output $(Agent.JobName) + Write-Output "-----------------" + Write-Output $(OSVmImage) + Write-Output $(TestTargetFramework) + try { + Write-Output $(additionalTestArguments) + } catch {} + displayName: Print matrix job variables + + - pwsh: | + Write-Output "Success" + displayName: linux OS condition example + condition: and(succeededOrFailed(), contains(variables['OSVmImage'], 'Ubuntu')) diff --git a/eng/common/scripts/job-matrix/samples/matrix-test.yml b/eng/common/scripts/job-matrix/samples/matrix-test.yml new file mode 100644 index 00000000000..be9d907e7c2 --- /dev/null +++ b/eng/common/scripts/job-matrix/samples/matrix-test.yml @@ -0,0 +1,26 @@ +trigger: none + +variables: + - template: /eng/pipelines/templates/variables/globals.yml + +jobs: + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + parameters: + JobTemplatePath: /eng/common/scripts/job-matrix/samples/matrix-job-sample.yml + AdditionalParameters: [] + CloudConfig: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + Location: eastus2 + Cloud: Public + MatrixFilters: [] + MatrixConfigs: + - Name: base_product_matrix + Path: eng/common/scripts/job-matrix/samples/matrix.json + Selection: all + GenerateVMJobs: true + - Name: sparse_product_matrix + Path: eng/common/scripts/job-matrix/samples/matrix.json + Selection: sparse + NonSparseParameters: + - framework + GenerateVMJobs: true diff --git a/eng/common/scripts/job-matrix/samples/matrix.json b/eng/common/scripts/job-matrix/samples/matrix.json new file mode 100644 index 00000000000..cf88ef02700 --- /dev/null +++ b/eng/common/scripts/job-matrix/samples/matrix.json @@ -0,0 +1,28 @@ +{ + "displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" + }, + "matrix": { + "Agent": { + "ubuntu-18.04": { "OSVmImage": "ubuntu-18.04", "Pool": "Azure Pipelines" }, + "windows-2019": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" }, + "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + }, + "TestTargetFramework": [ "netcoreapp2.1", "net461", "net5.0" ] + }, + "include": [ + { + "Agent": { + "windows-2019": { "OSVmImage": "windows-2019", "Pool": "Azure Pipelines" } + }, + "TestTargetFramework": [ "net461", "net5.0" ], + "AdditionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "OSVmImage": "MMS2019", + "framework": "netcoreapp2.1" + } + ] +} diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 new file mode 100644 index 00000000000..a25367bebff --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.filter.tests.ps1 @@ -0,0 +1,84 @@ +Import-Module Pester + +BeforeAll { + . $PSScriptRoot/../job-matrix-functions.ps1 + + $matrixConfig = @" +{ + "matrix": { + "operatingSystem": [ "windows-2019", "ubuntu-18.04", "macOS-10.15" ], + "framework": [ "net461", "netcoreapp2.1" ], + "additionalArguments": [ "", "mode=test" ] + } +} +"@ + $config = GetMatrixConfigFromJson $matrixConfig +} + +Describe "Matrix Filter" -Tag "filter" { + It "Should filter by matrix display name" -TestCases @( + @{ regex = "windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ regex = "windows2019_netcoreapp21_modetest"; expectedFirst = "windows2019_netcoreapp21_modetest"; length = 1 } + @{ regex = ".*ubuntu.*"; expectedFirst = "ubuntu1804_net461"; length = 4 } + ) { + [array]$matrix = GenerateMatrix $config "all" $regex + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should handle no display name filter matches" { + $matrix = GenerateMatrix $config "all" + [array]$filtered = FilterMatrixDisplayName $matrix "doesnotexist" + $filtered | Should -BeNullOrEmpty + } + + It "Should filter by matrix key/value" -TestCases @( + @{ filterString = "operatingSystem=windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "operatingSystem=windows-2019"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "framework=.*"; expectedFirst = "windows2019_net461"; length = 12 } + @{ filterString = "additionalArguments=mode=test"; expectedFirst = "windows2019_net461_modetest"; length = 6 } + @{ filterString = "additionalArguments=^$"; expectedFirst = "windows2019_net461"; length = 6 } + ) { + [array]$matrix = GenerateMatrix $config "all" -filters @($filterString) + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should filter by optional matrix key/value" -TestCases @( + @{ filterString = "operatingSystem=^$|windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "doesnotexist=^$|.*"; expectedFirst = "windows2019_net461"; length = 12 } + ) { + [array]$matrix = GenerateMatrix $config "all" -filters @($filterString) + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should handle multiple matrix key/value filters " { + [array]$matrix = GenerateMatrix $config "all" -filters "operatingSystem=windows.*","framework=.*","additionalArguments=mode=test" + $matrix.Length | Should -Be 2 + $matrix[0].Name | Should -Be "windows2019_net461_modetest" + } + + It "Should handle no matrix key/value filter matches" { + [array]$matrix = GenerateMatrix $config "all" -filters @("doesnot=exist") + $matrix | Should -BeNullOrEmpty + } + + It "Should handle invalid matrix key/value filter syntax" { + { GenerateMatrix $config "all" -filters @("invalid") } | Should -Throw + { GenerateMatrix $config "all" -filters @("emptyvalue=") } | Should -Throw + { GenerateMatrix $config "all" -filters @("=emptykey") } | Should -Throw + { GenerateMatrix $config "all" -filters @("=") } | Should -Throw + } + + It "Should filter by key exclude" { + [array]$matrix = GenerateMatrix $config "all" -filters @("operatingSystem=^$") + $matrix | Should -BeNullOrEmpty + + [array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 12 + $matrix += @{ Name = "excludeme"; Parameters = [Ordered]@{ "foo" = 1 } } + [array]$matrix = FilterMatrix $matrix @("foo=^$") + $matrix.Length | Should -Be 12 + } +} diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 new file mode 100644 index 00000000000..748c6632d0f --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1 @@ -0,0 +1,219 @@ +Import-Module Pester + + +BeforeAll { + . $PSScriptRoot/../job-matrix-functions.ps1 + + function CompareMatrices([Array]$matrix, [Array]$expected) { + $matrix.Length | Should -Be $expected.Length + + for ($i = 0; $i -lt $matrix.Length; $i++) { + foreach ($entry in $matrix[$i]) { + $expected[$i].name | Should -Be $entry.name + foreach ($param in $entry.parameters.GetEnumerator()) { + $expected[$i].parameters[$param.Name] | Should -Be $param.Value + } + } + } + } +} + +Describe "Platform Matrix nonSparse" -Tag "nonsparse" { + BeforeEach { + $matrixJson = @' +{ + "matrix": { + "testField1": [ 1, 2 ], + "testField2": [ 1, 2, 3 ], + "testField3": [ 1, 2, 3, 4 ], + } +} +'@ + $config = GetMatrixConfigFromJson $matrixJson + } + + It "Should process nonSparse parameters" { + $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField1","testField3" + $parameters.Count | Should -Be 1 + $parameters["testField2"] | Should -Be 1,2,3 + $nonSparse.Count | Should -Be 2 + $nonSparse["testField1"] | Should -Be 1,2 + $nonSparse["testField3"] | Should -Be 1,2,3,4 + + $parameters, $nonSparse = ProcessNonSparseParameters $config.orderedMatrix "testField3" + $parameters.Count | Should -Be 2 + $parameters.Contains("testField3") | Should -Be $false + $nonSparse.Count | Should -Be 1 + $nonSparse["testField3"] | Should -Be 1,2,3,4 + } + + It "Should ignore nonSparse with all selection" { + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3" + $matrix.Length | Should -Be 24 + } + + It "Should combine sparse matrix with nonSparse parameters" { + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3" + $matrix.Length | Should -Be 12 + } + + It "Should combine with multiple nonSparse fields" { + $matrixJson = @' +{ + "matrix": { + "testField1": [ 1, 2 ], + "testField2": [ 1, 2 ], + "testField3": [ 31, 32 ], + "testField4": [ 41, 42 ] + } +} +'@ + $config = GetMatrixConfigFromJson $matrixJson + + $matrix = GenerateMatrix $config "all" -nonSparseParameters "testField3","testField4" + $matrix.Length | Should -Be 16 + + $matrix = GenerateMatrix $config "sparse" -nonSparseParameters "testField3","testField4" + $matrix.Length | Should -Be 8 + } +} + +Describe "Platform Matrix Import" -Tag "import" { + It "Should generate a matrix with nonSparseParameters and an imported sparse matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2" ] + } +} +'@ + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" -nonSparseParameters "testField" + + $matrix.Length | Should -Be 6 + + $matrix[0].name | Should -Be test1_foo1_bar1 + $matrix[0].parameters.testField | Should -Be "test1" + $matrix[0].parameters.Foo | Should -Be "foo1" + $matrix[2].name | Should -Be test1_importedBaz + $matrix[2].parameters.testField | Should -Be "test1" + $matrix[2].parameters.Baz | Should -Be "importedBaz" + $matrix[4].name | Should -Be test2_foo2_bar2 + $matrix[4].parameters.testField | Should -Be "test2" + $matrix[4].parameters.Foo | Should -Be "foo2" + } + + It "Should generate a sparse matrix with an imported a sparse matrix" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField1": [ "test11", "test12" ], + "testField2": [ "test21", "test22" ] + } +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "testField1": "test11", "testField2": "test21", "Foo": "foo1", "Bar": "bar1" }, + "name": "test11_test21_foo1_bar1" + }, + { + "parameters": { "testField1": "test11", "testField2": "test21", "Foo": "foo2", "Bar": "bar2" }, + "name": "test11_test21_foo2_bar2" + }, + { + "parameters": { "testField1": "test11", "testField2": "test21", "Baz": "importedBaz" }, + "name": "test11_test21_importedBaz" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo1", "Bar": "bar1" }, + "name": "test12_test22_foo1_bar1" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Foo": "foo2", "Bar": "bar2" }, + "name": "test12_test22_foo2_bar2" + }, + { + "parameters": { "testField1": "test12", "testField2": "test22", "Baz": "importedBaz" }, + "name": "test12_test22_importedBaz" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 6 + CompareMatrices $matrix $expected + } + + It "Should import a sparse matrix with import, include, and exclude" { + $matrixJson = @' +{ + "matrix": { + "$IMPORT": "./test-import-matrix.json", + "testField": [ "test1", "test2", "test3" ], + }, + "include": [ + { + "testImportIncludeName": [ "testInclude1", "testInclude2" ] + } + ], + "exclude": [ + { + "testField": "test1" + }, + { + "testField": "test3", + "Baz": "importedBaz" + } + ] +} +'@ + + $expectedMatrix = @' +[ + { + "parameters": { "testField": "test2", "Foo": "foo1", "Bar": "bar1" }, + "name": "test2_foo1_bar1" + }, + { + "parameters": { "testField": "test2", "Foo": "foo2", "Bar": "bar2" }, + "name": "test2_foo2_bar2" + }, + { + "parameters": { "testField": "test2", "Baz": "importedBaz" }, + "name": "test2_importedBaz" + }, + { + "parameters": { "testField": "test3", "Foo": "foo1", "Bar": "bar1" }, + "name": "test3_foo1_bar1" + }, + { + "parameters": { "testField": "test3", "Foo": "foo2", "Bar": "bar2" }, + "name": "test3_foo2_bar2" + }, + { + "parameters": { "testImportIncludeName": "testInclude1" }, + "name": "testInclude1" + }, + { + "parameters": { "testImportIncludeName": "testInclude2" }, + "name": "testInclude2" + } +] +'@ + + $importConfig = GetMatrixConfigFromJson $matrixJson + $matrix = GenerateMatrix $importConfig "sparse" + $expected = $expectedMatrix | ConvertFrom-Json -AsHashtable + + $matrix.Length | Should -Be 7 + CompareMatrices $matrix $expected + } +} diff --git a/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 b/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 new file mode 100644 index 00000000000..ab9ce82a00f --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/job-matrix-functions.tests.ps1 @@ -0,0 +1,562 @@ +Import-Module Pester + +BeforeAll { + . $PSScriptRoot/../job-matrix-functions.ps1 + + $matrixConfig = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": ["net461", "netcoreapp2.1", "net50"], + "additionalArguments": "--enableWindowsFoo" + } + ], + "exclude": [ + { + "operatingSystem": "windows-2019", + "framework": "net461" + }, + { + "operatingSystem": "macOS-10.15", + "framework": "netcoreapp2.1" + }, + { + "operatingSystem": ["macOS-10.15", "ubuntu-18.04"], + "additionalArguments": "--enableFoo" + } + ] +} +"@ +} + +Describe "Matrix-Lookup" -Tag "lookup" { + It "Should navigate a 2d matrix: " -TestCases @( + @{ row = 0; col = 0; expected = 1 }, + @{ row = 0; col = 1; expected = 2 }, + @{ row = 1; col = 0; expected = 3 }, + @{ row = 1; col = 1; expected = 4 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 3d matrix: " -TestCases @( + @{ z = 0; row = 0; col = 0; expected = 1 } + @{ z = 0; row = 0; col = 1; expected = 2 } + @{ z = 0; row = 1; col = 0; expected = 3 } + @{ z = 0; row = 1; col = 1; expected = 4 } + @{ z = 1; row = 0; col = 0; expected = 5 } + @{ z = 1; row = 0; col = 1; expected = 6 } + @{ z = 1; row = 1; col = 0; expected = 7 } + @{ z = 1; row = 1; col = 1; expected = 8 } + ) { + $dimensions = @(2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8 + ) + GetNdMatrixElement @($z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 1; col = 0; expected = 3 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 4 } + @{ t = 0; z = 1; row = 0; col = 0; expected = 5 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 6 } + @{ t = 0; z = 1; row = 1; col = 0; expected = 7 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 8 } + @{ t = 1; z = 0; row = 0; col = 0; expected = 9 } + @{ t = 1; z = 0; row = 0; col = 1; expected = 10 } + @{ t = 1; z = 0; row = 1; col = 0; expected = 11 } + @{ t = 1; z = 0; row = 1; col = 1; expected = 12 } + @{ t = 1; z = 1; row = 0; col = 0; expected = 13 } + @{ t = 1; z = 1; row = 0; col = 1; expected = 14 } + @{ t = 1; z = 1; row = 1; col = 0; expected = 15 } + @{ t = 1; z = 1; row = 1; col = 1; expected = 16 } + ) { + $dimensions = @(2, 2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 0; col = 2; expected = 3 } + @{ t = 0; z = 0; row = 0; col = 3; expected = 4 } + + @{ t = 0; z = 0; row = 1; col = 0; expected = 5 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 6 } + @{ t = 0; z = 0; row = 1; col = 2; expected = 7 } + @{ t = 0; z = 0; row = 1; col = 3; expected = 8 } + + @{ t = 0; z = 1; row = 0; col = 0; expected = 9 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 10 } + @{ t = 0; z = 1; row = 0; col = 2; expected = 11 } + @{ t = 0; z = 1; row = 0; col = 3; expected = 12 } + + @{ t = 0; z = 1; row = 1; col = 0; expected = 13 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 14 } + @{ t = 0; z = 1; row = 1; col = 2; expected = 15 } + @{ t = 0; z = 1; row = 1; col = 3; expected = 16 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + # Skipping since by default wrapping behavior on indexing is disabled. + # Keeping here in case we want to enable it. + It -Skip "Should handle index wrapping: " -TestCases @( + @{ row = 2; col = 2; expected = 1 } + @{ row = 2; col = 3; expected = 2 } + @{ row = 4; col = 4; expected = 1 } + @{ row = 4; col = 5; expected = 2 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } +} + +Describe "Matrix-Reverse-Lookup" -Tag "lookup" { + It "Should lookup a 2d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0) } + @{ index = 1; expected = @(0,1) } + @{ index = 2; expected = @(1,0) } + @{ index = 3; expected = @(1,1) } + ) { + $dimensions = @(2, 2) + $matrix = @(1, 2, 3, 4) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + ) { + $dimensions = @(2, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,0,2) } + + @{ index = 3; expected = @(0,1,0) } + @{ index = 4; expected = @(0,1,1) } + @{ index = 5; expected = @(0,1,2) } + + @{ index = 6; expected = @(1,0,0) } + @{ index = 7; expected = @(1,0,1) } + @{ index = 8; expected = @(1,0,2) } + + @{ index = 9; expected = @(1,1,0) } + @{ index = 10; expected = @(1,1,1) } + @{ index = 11; expected = @(1,1,2) } + ) { + $dimensions = @(2, 2, 3) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + + @{ index = 8; expected = @(2,0,0) } + @{ index = 9; expected = @(2,0,1) } + @{ index = 10; expected = @(2,1,0) } + @{ index = 11; expected = @(2,1,1) } + ) { + $dimensions = @(3, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 4d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0,0) } + @{ index = 1; expected = @(0,0,0,1) } + @{ index = 2; expected = @(0,0,0,2) } + @{ index = 3; expected = @(0,0,0,3) } + + @{ index = 4; expected = @(0,0,1,0) } + @{ index = 5; expected = @(0,0,1,1) } + @{ index = 6; expected = @(0,0,1,2) } + @{ index = 7; expected = @(0,0,1,3) } + + @{ index = 8; expected = @(0,1,0,0) } + @{ index = 9; expected = @(0,1,0,1) } + @{ index = 10; expected = @(0,1,0,2) } + @{ index = 11; expected = @(0,1,0,3) } + + @{ index = 12; expected = @(0,1,1,0) } + @{ index = 13; expected = @(0,1,1,1) } + @{ index = 14; expected = @(0,1,1,2) } + @{ index = 15; expected = @(0,1,1,3) } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } +} + +Describe 'Matrix-Set' -Tag "set" { + It "Should set a matrix element" -TestCases @( + @{ value = "set"; index = @(0,0,0,0); arrayIndex = 0 } + @{ value = "ones"; index = @(0,1,1,1); arrayIndex = 13 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + + SetNdMatrixElement $value $index $matrix $dimensions + $matrix[$arrayIndex] | Should -Be $value + } +} + +Describe "Platform Matrix Generation" -Tag "generate" { + BeforeEach { + $matrixConfigForGenerate = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": "net461", + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "foo": "bar" + }, + { + "foo2": "bar2" + } + ] +} +"@ + $generateConfig = GetMatrixConfigFromJson $matrixConfigForGenerate + } + + It "Should get matrix dimensions from Nd parameters" { + GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2 + + $generateConfig.orderedMatrix.Add("testStringParameter", "test") + GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2, 1 + } + + It "Should use name overrides from displayNames" { + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateconfig.displayNamesLookup + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "windows2019_net461" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name | Should -Be "ubuntu1804_netcoreapp21_withFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.name | Should -Be "macOS1015_netcoreapp21_withFoo" + } + + It "Should enforce valid display name format" { + $generateconfig.displayNamesLookup["net461"] = '123.Some.456.Invalid_format-name$(foo)' + $generateconfig.displayNamesLookup["netcoreapp2.1"] = (New-Object string[] 150) -join "a" + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateconfig.orderedMatrix $generateconfig.displayNamesLookup + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "windows2019_123some456invalid_formatnamefoo" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name.Length | Should -Be 100 + # The withfoo part of the argument gets cut off at the character limit + $element.name | Should -BeLike "ubuntu1804_aaaaaaaaaaaaaaaaa*" + } + + + It "Should initialize an N-dimensional matrix from all parameter permutations" { + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $matrix.Count | Should -Be 12 + + $element = $matrix[0].parameters + $element.operatingSystem | Should -Be "windows-2019" + $element.framework | Should -Be "net461" + $element.additionalArguments | Should -Be "" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "ubuntu-18.04" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "macOS-10.15" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + } + + It "Should initialize a sparse matrix from an N-dimensional matrix" -TestCases @( + @{ i = 0; name = "windows2019_net461"; operatingSystem = "windows-2019"; framework = "net461"; additionalArguments = ""; } + @{ i = 1; name = "ubuntu1804_netcoreapp21_withfoo"; operatingSystem = "ubuntu-18.04"; framework = "netcoreapp2.1"; additionalArguments = "--enableFoo"; } + @{ i = 2; name = "macOS1015_net461"; operatingSystem = "macOS-10.15"; framework = "net461"; additionalArguments = ""; } + ) { + $sparseMatrix = GenerateSparseMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $size = ($dimensions | Measure-Object -Maximum).Maximum + $sparseMatrix.Count | Should -Be $size + + $sparseMatrix[$i].name | Should -Be $name + $element = $sparseMatrix[$i].parameters + $element.operatingSystem | Should -Be $operatingSystem + $element.framework | Should -Be $framework + $element.additionalArguments | Should -Be $additionalArguments + } + + It "Should generate a sparse matrix from an N-dimensional matrix config" { + $sparseMatrix = GenerateMatrix $generateConfig "sparse" + $sparseMatrix.Length | Should -Be 4 + } + + It "Should initialize a full matrix from an N-dimensional matrix config" { + $matrix = GenerateMatrix $generateConfig "all" + $matrix.Length | Should -Be 13 + } +} + +Describe "Config File Object Conversion" -Tag "convert" { + BeforeEach { + $config = GetMatrixConfigFromJson $matrixConfig + } + + It "Should convert a matrix config" { + $config.orderedMatrix | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + $config.orderedMatrix.operatingSystem[0] | Should -Be "windows-2019" + + $config.displayNamesLookup | Should -BeOfType [Hashtable] + $config.displayNamesLookup["--enableFoo"] | Should -Be "withFoo" + + $config.include | ForEach-Object { + $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + $config.exclude | ForEach-Object { + $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + } +} + +Describe "Platform Matrix Post Transformation" -Tag "transform" { + BeforeEach { + $config = GetMatrixConfigFromJson $matrixConfig + } + + It "Should match partial matrix elements" -TestCases @( + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1 }; expected = $true } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2 }; expected = $true } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2; c = 3 }; expected = $false } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ }; expected = $false } + @{ source = [Ordered]@{ }; target = [Ordered]@{ a = 1; b = 2; }; expected = $false } + ) { + MatrixElementMatch $source $target | Should -Be $expected + } + + It "Should remove matrix elements based on exclude filters" { + $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 5 + + $matrix = GenerateSparseMatrix $config.orderedMatrix $config.displayNamesLookup + [array]$withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 1 + } + + It "Should add matrix elements based on include elements" { + $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $withInclusion = ProcessIncludes $config $matrix "all" + $withInclusion.Length | Should -Be 15 + } + + It "Should include and exclude values with a matrix" { + [Array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 8 + + $matrix[0].name | Should -Be "windows2019_netcoreapp21" + $matrix[0].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[0].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[0].parameters.additionalArguments | Should -Be "" + + $matrix[1].name | Should -Be "windows2019_netcoreapp21_withfoo" + $matrix[1].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[1].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[1].parameters.additionalArguments | Should -Be "--enableFoo" + + $matrix[2].name | Should -Be "ubuntu1804_net461" + $matrix[2].parameters.framework | Should -Be "net461" + $matrix[2].parameters.operatingSystem | Should -Be "ubuntu-18.04" + $matrix[2].parameters.additionalArguments | Should -Be "" + + $matrix[4].name | Should -Be "macOS1015_net461" + $matrix[4].parameters.framework | Should -Be "net461" + $matrix[4].parameters.operatingSystem | Should -Be "macOS-10.15" + $matrix[4].parameters.additionalArguments | Should -Be "" + + $matrix[7].name | Should -Be "windows2019_net50_enableWindowsFoo" + $matrix[7].parameters.framework | Should -Be "net50" + $matrix[7].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[7].parameters.additionalArguments | Should -Be "--enableWindowsFoo" + } +} + +Describe "Platform Matrix Generation With Object Fields" -Tag "objectfields" { + BeforeEach { + $matrixConfigForObject = @" +{ + "matrix": { + "testObject": { + "testObjectName1": { "testObject1Value1": "1", "testObject1Value2": "2" }, + "testObjectName2": { "testObject2Value1": "1", "testObject2Value2": "2" } + }, + "secondTestObject": { + "secondTestObjectName1": { "secondTestObject1Value1": "1", "secondTestObject1Value2": "2" } + }, + "testField": [ "footest", "bartest" ] + }, + "include": [ + { + "testObjectInclude": { + "testObjectIncludeName": { "testObjectValue1": "1", "testObjectValue2": "2" } + }, + "testField": "footest" + } + ] +} +"@ + $objectFieldConfig = GetMatrixConfigFromJson $matrixConfigForObject + } + + It "Should parse dimensions properly" { + [Array]$dimensions = GetMatrixDimensions $objectFieldConfig.orderedMatrix + $dimensions.Length | Should -Be 3 + $dimensions[0] | Should -Be 2 + $dimensions[1] | Should -Be 1 + $dimensions[2] | Should -Be 2 + } + + It "Should populate a sparse matrix dimensions properly" { + [Array]$matrix = GenerateMatrix $objectFieldConfig "sparse" + $matrix.Length | Should -Be 3 + + $matrix[0].name | Should -Be "testObjectName1_secondTestObjectName1_footest" + $matrix[0].parameters.testField | Should -Be "footest" + $matrix[0].parameters.testObject1Value1 | Should -Be "1" + $matrix[0].parameters.testObject1Value2 | Should -Be "2" + $matrix[0].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[0].parameters.Count | Should -Be 5 + + $matrix[1].name | Should -Be "testObjectName2_secondTestObjectName1_bartest" + $matrix[1].parameters.testField | Should -Be "bartest" + $matrix[1].parameters.testObject2Value1 | Should -Be "1" + $matrix[1].parameters.testObject2Value2 | Should -Be "2" + $matrix[1].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[1].parameters.Count | Should -Be 5 + + $matrix[2].name | Should -Be "testObjectIncludeName_footest" + $matrix[2].parameters.testField | Should -Be "footest" + $matrix[2].parameters.testObjectValue1 | Should -Be "1" + $matrix[2].parameters.testObjectValue2 | Should -Be "2" + $matrix[2].parameters.Count | Should -Be 3 + } + + It "Should splat matrix entries that are objects into key/values" { + [Array]$matrix = GenerateMatrix $objectFieldConfig "all" + $matrix.Length | Should -Be 5 + + $matrix[0].name | Should -Be "testObjectName1_secondTestObjectName1_footest" + $matrix[0].parameters.testField | Should -Be "footest" + $matrix[0].parameters.testObject1Value1 | Should -Be "1" + $matrix[0].parameters.testObject1Value2 | Should -Be "2" + $matrix[0].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[0].parameters.Count | Should -Be 5 + + $matrix[3].name | Should -Be "testObjectName2_secondTestObjectName1_bartest" + $matrix[3].parameters.testField | Should -Be "bartest" + $matrix[3].parameters.testObject2Value1 | Should -Be "1" + $matrix[3].parameters.testObject2Value2 | Should -Be "2" + $matrix[3].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[3].parameters.Count | Should -Be 5 + + $matrix[4].name | Should -Be "testObjectIncludeName_footest" + $matrix[4].parameters.testField | Should -Be "footest" + $matrix[4].parameters.testObjectValue1 | Should -Be "1" + $matrix[4].parameters.testObjectValue2 | Should -Be "2" + $matrix[4].parameters.Count | Should -Be 3 + } +} diff --git a/eng/common/scripts/job-matrix/tests/test-import-matrix.json b/eng/common/scripts/job-matrix/tests/test-import-matrix.json new file mode 100644 index 00000000000..2048bd390c5 --- /dev/null +++ b/eng/common/scripts/job-matrix/tests/test-import-matrix.json @@ -0,0 +1,11 @@ +{ + "matrix": { + "Foo": [ "foo1", "foo2" ], + "Bar": [ "bar1", "bar2" ] + }, + "include": [ + { + "Baz": "importedBaz" + } + ] +}