From 02362ab9cb462739f8b2ab89e7c4f763db2372b9 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 9 Nov 2020 17:49:43 -0800 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 1163 (#13338) * Update subscription configuration schema to include new parameters * Support platform specific arm template parameters and legacy hashtable format * Update arm template parameter comment to include top level key * Restore AdditionalParameters. Merge ArmTemplateParameters from stringified hash literal * Handle duplicate keys more explicitly for arm and env vars * Regenerate New-TestResources.ps1 markdown * revert variable name to environmentVariables to fix post-scripts * Handle empty arm template parameters better * Remove arm template parameter merge logic from deploy template * Add merge hashes function to New-TestResources.ps1 * Add merge hashes function to New-TestResources.ps1 * Add env variable overwrite warning. Use ContainsKey checks * Temporarily manually fix invalid generated markdown links Co-authored-by: Ben Broderick Phillips --- .../TestResources/New-TestResources.ps1 | 52 ++++++++++++++----- .../TestResources/New-TestResources.ps1.md | 43 ++++++++++++--- .../TestResources/Remove-TestResources.ps1 | 2 - .../TestResources/deploy-test-resources.yml | 30 ++++++++--- 4 files changed, 101 insertions(+), 26 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 8de70b50dce7..4c03816c1284 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -10,7 +10,7 @@ [CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')] param ( - # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming. + # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming [Parameter()] [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')] [string] $BaseName, @@ -58,9 +58,16 @@ param ( [ValidateSet('AzureCloud', 'AzureUSGovernment', 'AzureChinaCloud', 'Dogfood')] [string] $Environment = 'AzureCloud', + [Parameter()] + [hashtable] $ArmTemplateParameters, + [Parameter()] [hashtable] $AdditionalParameters, + [Parameter()] + [ValidateNotNull()] + [hashtable] $EnvironmentVariables = @{}, + [Parameter()] [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID), @@ -101,6 +108,16 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) { } } +function MergeHashes([hashtable] $source, [psvariable] $dest) { + foreach ($key in $source.Keys) { + if ($dest.Value.ContainsKey($key) -and $dest.Value[$key] -ne $source[$key]) { + Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + + "to new value '$($source[$key])'") + } + $dest.Value[$key] = $source[$key] + } +} + # Support actions to invoke on exit. $exitActions = @({ if ($exitActions.Count -gt 1) { @@ -118,7 +135,6 @@ $repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path $templateFileName = 'test-resources.json' $templateFiles = @() -$environmentVariables = @{} # Azure SDK Developer Playground $defaultSubscription = "faa080af-c1d8-40ad-9cce-e1a450ca5b57" @@ -252,7 +268,7 @@ $serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { $ServiceDirectory } -if ($CI) { +if ($CI) { $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16) Write-Verbose "Generated base name '$BaseName' for CI build" } @@ -289,7 +305,13 @@ if ($CI) { # Set the resource group name variable. Write-Host "Setting variable 'AZURE_RESOURCEGROUP_NAME': $ResourceGroupName" Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME;]$ResourceGroupName" - $environmentVariables['AZURE_RESOURCEGROUP_NAME'] = $ResourceGroupName + if ($EnvironmentVariables.ContainsKey('AZURE_RESOURCEGROUP_NAME') -and ` + $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] -ne $ResourceGroupName) + { + Write-Warning ("Overwriting 'EnvironmentVariables.AZURE_RESOURCEGROUP_NAME' with value " + + "'$($EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'])' " + "to new value '$($ResourceGroupName)'") + } + $EnvironmentVariables['AZURE_RESOURCEGROUP_NAME'] = $ResourceGroupName } Log "Creating resource group '$ResourceGroupName' in location '$Location'" @@ -322,11 +344,11 @@ if ($TenantId) { if ($TestApplicationSecret) { $templateParameters.Add('testApplicationSecret', $TestApplicationSecret) } -if ($AdditionalParameters) { - $templateParameters += $AdditionalParameters -} -# Include environment-specific parameters only if not already provided as part of the "AdditionalParameters" +MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters) +MergeHashes $AdditionalParameters $(Get-Variable templateParameters) + +# Include environment-specific parameters only if not already provided as part of the "ArmTemplateParameters" if (($context.Environment.StorageEndpointSuffix) -and (-not ($templateParameters.ContainsKey('storageEndpointSuffix')))) { $templateParameters.Add('storageEndpointSuffix', $context.Environment.StorageEndpointSuffix) } @@ -388,6 +410,8 @@ foreach ($templateFile in $templateFiles) { "$($serviceDirectoryPrefix)STORAGE_ENDPOINT_SUFFIX" = $context.Environment.StorageEndpointSuffix; } + MergeHashes $EnvironmentVariables $(Get-Variable deploymentOutputs) + foreach ($key in $deployment.Outputs.Keys) { $variable = $deployment.Outputs[$key] @@ -422,7 +446,7 @@ foreach ($templateFile in $templateFiles) { foreach ($key in $deploymentOutputs.Keys) { $value = $deploymentOutputs[$key] - $environmentVariables[$key] = $value + $EnvironmentVariables[$key] = $value if ($CI) { # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. @@ -453,7 +477,7 @@ $exitActions.Invoke() # Suppress output locally if ($CI) { - return $environmentVariables + return $EnvironmentVariables } <# @@ -571,8 +595,14 @@ Name of the cloud environment. The default is the Azure Public Cloud ('AzureCloud') .PARAMETER AdditionalParameters +Optional key-value pairs of parameters to pass to the ARM template(s) and pre-post scripts. + +.PARAMETER ArmTemplateParameters Optional key-value pairs of parameters to pass to the ARM template(s). +.PARAMETER EnvironmentVariables +Optional key-value pairs of parameters to set as environment variables to the shell. + .PARAMETER CI Indicates the script is run as part of a Continuous Integration / Continuous Deployment (CI/CD) build (only Azure Pipelines is currently supported). @@ -617,6 +647,4 @@ Run this in an Azure DevOps CI (with approrpiate variables configured) before executing live tests. The script will output variables as secrets (to enable log redaction). -.LINK -Remove-TestResources.ps1 #> diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md index c41693c87666..515a51bd373c 100644 --- a/eng/common/TestResources/New-TestResources.ps1.md +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -16,8 +16,9 @@ Deploys live test resources defined for a service directory to Azure. ``` New-TestResources.ps1 [-BaseName ] [-ResourceGroupName ] [-ServiceDirectory] [-TestApplicationId ] [-TestApplicationSecret ] [-TestApplicationOid ] - [-DeleteAfterHours ] [-Location ] [-Environment ] [-AdditionalParameters ] - [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm] [] + [-DeleteAfterHours ] [-Location ] [-Environment ] [-ArmTemplateParameters ] + [-AdditionalParameters ] [-EnvironmentVariables ] [-CI] [-Force] [-OutFile] [-WhatIf] + [-Confirm] [] ``` ### Provisioner @@ -26,8 +27,8 @@ New-TestResources.ps1 [-BaseName ] [-ResourceGroupName ] [-Servi [-TestApplicationId ] [-TestApplicationSecret ] [-TestApplicationOid ] -TenantId [-SubscriptionId ] -ProvisionerApplicationId -ProvisionerApplicationSecret [-DeleteAfterHours ] [-Location ] - [-Environment ] [-AdditionalParameters ] [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm] - [] + [-Environment ] [-ArmTemplateParameters ] [-AdditionalParameters ] + [-EnvironmentVariables ] [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm] [] ``` ## DESCRIPTION @@ -339,7 +340,7 @@ Accept wildcard characters: False ### -Environment Name of the cloud environment. The default is the Azure Public Cloud -('PublicCloud') +('AzureCloud') ```yaml Type: String @@ -353,7 +354,7 @@ Accept pipeline input: False Accept wildcard characters: False ``` -### -AdditionalParameters +### -ArmTemplateParameters Optional key-value pairs of parameters to pass to the ARM template(s). ```yaml @@ -368,6 +369,36 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -AdditionalParameters +Optional key-value pairs of parameters to pass to the ARM template(s) and pre-post scripts. + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -EnvironmentVariables +Optional key-value pairs of parameters to set as environment variables to the shell. + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: @{} +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -CI Indicates the script is run as part of a Continuous Integration / Continuous Deployment (CI/CD) build (only Azure Pipelines is currently supported). diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 83b1a58c347f..b74ddb42ac23 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -214,6 +214,4 @@ Remove-TestResources.ps1 ` When run in the context of an Azure DevOps pipeline, this script removes the resource group whose name is stored in the environment variable AZURE_RESOURCEGROUP_NAME. -.LINK -New-TestResources.ps1 #> diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml index 6f0db25d61a8..031c4b37cccd 100644 --- a/eng/common/TestResources/deploy-test-resources.yml +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -5,7 +5,7 @@ parameters: Location: '' SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) -# SubscriptionConfiguration will be splat into the parameters of the test +# SubscriptionConfiguration will be splatted into the parameters of the test # resources script. It should be JSON in the form: # { # "SubscriptionId": "", @@ -15,25 +15,43 @@ parameters: # "ProvisionerApplicationId": "", # "ProvisionerApplicationSecret": "", # "Environment": "AzureCloud | AzureGov | AzureChina | " +# "EnvironmentVariables": { +# "SERVICE_MANAGEMENT_URL": "", +# "STORAGE_ENDPOINT_SUFFIX": "", +# "RESOURCE_MANAGER_URL": "", +# "SEARCH_ENDPOINT_SUFFIX": "", +# "COSMOS_TABLES_ENDPOINT_SUFFIX": "" +# }, +# "ArmTemplateParameters": { +# "keyVaultDomainSuffix": "", +# "storageEndpointSuffix": "", +# "endpointSuffix": "", +# "azureAuthorityHost": "", +# "keyVaultEndpointSuffix": "" +# } # } + steps: - template: /eng/common/TestResources/setup-az-modules.yml - pwsh: | eng/common/TestResources/Import-AzModules.ps1 - $subscriptionConfiguration = @" + $subscriptionConfiguration = @' ${{ parameters.SubscriptionConfiguration }} - "@ | ConvertFrom-Json -AsHashtable; + '@ | ConvertFrom-Json -AsHashtable; + # The subscriptionConfiguration may have ArmTemplateParameters defined, so + # pass those in via the ArmTemplateParameters flag, and handle any + # additional parameters from the pipelines via AdditionalParameters eng/common/TestResources/New-TestResources.ps1 ` -BaseName 'Generated' ` - -ServiceDirectory ${{ parameters.ServiceDirectory }} ` + -ServiceDirectory '${{ parameters.ServiceDirectory }}' ` -Location '${{ parameters.Location }}' ` - -DeleteAfterHours ${{ parameters.DeleteAfterHours }} ` - -AdditionalParameters ${{ parameters.ArmTemplateParameters }} ` + -DeleteAfterHours '${{ parameters.DeleteAfterHours }}' ` @subscriptionConfiguration ` + -AdditionalParameters ${{ parameters.ArmTemplateParameters }} ` -CI ` -Force ` -Verbose | Out-Null