From c64609a6786ed0e9fc3f6626757134edb3730a73 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:53:40 -0400 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools for PR 8558 (#44863) * Support storage network access and worm removal in remove test resources script * Move storage network access script to common resource helpers file * Improve storage container deletion resilience * Plumb through pool variable to live test cleanup template * Add sleep for network rule application --------- Co-authored-by: Ben Broderick Phillips --- .../TestResources/New-TestResources.ps1 | 294 +----------------- .../TestResources/Remove-TestResources.ps1 | 19 ++ .../TestResources/TestResources-Helpers.ps1 | 267 ++++++++++++++++ .../TestResources/remove-test-resources.yml | 10 +- .../scripts/Helpers/Resource-Helpers.ps1 | 232 ++++++++++++-- 5 files changed, 501 insertions(+), 321 deletions(-) create mode 100644 eng/common/TestResources/TestResources-Helpers.ps1 diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 6ee09ff3b23ec..6ccf55a781c1e 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -117,6 +117,8 @@ param ( $NewTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) +. $PSScriptRoot/TestResources-Helpers.ps1 . $PSScriptRoot/SubConfig-Helpers.ps1 if (!$ServicePrincipalAuth) { @@ -131,272 +133,6 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' } -function Log($Message) -{ - Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) -} - -# vso commands are specially formatted log lines that are parsed by Azure Pipelines -# to perform additional actions, most commonly marking values as secrets. -# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands -function LogVsoCommand([string]$message) -{ - if (!$CI -or $SuppressVsoCommands) { - return - } - Write-Host $message -} - -function Retry([scriptblock] $Action, [int] $Attempts = 5) -{ - $attempt = 0 - $sleep = 5 - - while ($attempt -lt $Attempts) { - try { - $attempt++ - return $Action.Invoke() - } catch { - if ($attempt -lt $Attempts) { - $sleep *= 2 - - Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." - Start-Sleep -Seconds $sleep - } else { - throw - } - } - } -} - -# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. -# This is necessary to work around breaking changes introduced in Az version 7.0.0: -# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ -function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) -{ - if ((Get-Module Az.Resources).Version -eq "5.3.0") { - # https://github.com/Azure/azure-powershell/issues/17040 - # New-AzAdServicePrincipal calls will fail with: - # "You cannot call a method on a null-valued expression." - Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" - Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" - exit 1 - } - - try { - $servicePrincipal = Retry { - New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName - } - } catch { - # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying - # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: - # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." - # Provide a more helpful diagnostic prompt to the user if appropriate: - $totalApps = (Get-AzADApplication -OwnedApplication).Length - $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` - "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` - " or by running the following command to remove apps created by this script:" + ` - "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` - "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." - Write-Warning $msg - throw - } - - $spPassword = "" - $appId = "" - if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { - Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" - # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 - $spPassword = $servicePrincipal.Secret - $appId = $servicePrincipal.ApplicationId - } else { - if ((Get-Module Az.Resources).Version -eq "5.1.0") { - Write-Verbose "Creating password and credential for service principal via MS Graph API" - Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" - # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. - # Submitting a password credential object without specifying a password will result in one being generated on the server side. - $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" - $password.DisplayName = "Password for $displayName" - $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } - $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force - $appId = $servicePrincipal.AppId - } else { - Write-Verbose "Creating service principal credential via MS Graph API" - # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the - # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. - $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } - $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force - $appId = $servicePrincipal.AppId - } - } - - return @{ - AppId = $appId - ApplicationId = $appId - # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion - Id = $servicePrincipal.Id - DisplayName = $servicePrincipal.DisplayName - Secret = $spPassword - } -} - -function LoadCloudConfig([string] $env) -{ - $configPath = "$PSScriptRoot/clouds/$env.json" - if (!(Test-Path $configPath)) { - Write-Warning "Could not find cloud configuration for environment '$env'" - return @{} - } - - $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable - return $config -} - -function MergeHashes([hashtable] $source, [psvariable] $dest) -{ - foreach ($key in $source.Keys) { - if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { - Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + - "to new value '$($source[$key])'") - } - $dest.Value[$key] = $source[$key] - } -} - -function BuildBicepFile([System.IO.FileSystemInfo] $file) -{ - if (!(Get-Command bicep -ErrorAction Ignore)) { - Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install" - throw - } - - $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() - $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" - - # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the - # outputted json for mismatched parameter declarations. - bicep build $file.FullName --outfile $templateFilePath - if ($LASTEXITCODE) { - Write-Error "Failure building bicep file '$($file.FullName)'" - throw - } - - return $templateFilePath -} - -function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { - $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName - # Add default values - $deploymentOutputs = [Ordered]@{ - "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; - "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; - "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; - "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; - "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; - "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; - "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; - "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); - } - - if ($ServicePrincipalAuth) { - $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId; - $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret; - $deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id; - } - - MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) - - foreach ($key in $deployment.Outputs.Keys) { - $variable = $deployment.Outputs[$key] - - # Work around bug that makes the first few characters of environment variables be lowercase. - $key = $key.ToUpperInvariant() - - if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { - $deploymentOutputs[$key] = $variable.Value - } - } - - # Force capitalization of all keys to avoid Azure Pipelines confusion with - # variable auto-capitalization and OS env var capitalization differences - $capitalized = @{} - foreach ($item in $deploymentOutputs.GetEnumerator()) { - $capitalized[$item.Name.ToUpperInvariant()] = $item.Value - } - - return $capitalized -} - -function SetDeploymentOutputs( - [string]$serviceName, - [object]$azContext, - [object]$deployment, - [object]$templateFile, - [hashtable]$environmentVariables = @{} -) { - $deploymentEnvironmentVariables = $environmentVariables.Clone() - $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables - - if ($OutFile) { - if (!$IsWindows) { - Write-Host 'File option is supported only on Windows' - } - - $outputFile = "$($templateFile.originalFilePath).env" - - $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) - $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) - - Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force - - Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" - } else { - if (!$CI) { - # Write an extra new line to isolate the environment variables for easy reading. - Log "Persist the following environment variables based on your detected shell ($shell):`n" - } - - # Write overwrite warnings first, since local execution prints a runnable command to export variables - foreach ($key in $deploymentOutputs.Keys) { - if ([Environment]::GetEnvironmentVariable($key)) { - Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" - } - } - - # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep - # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). - # This variable supports a second check on not marking previously allowed keys/values as secret. - $notSecretValues = @() - foreach ($key in $deploymentOutputs.Keys) { - $value = $deploymentOutputs[$key] - $deploymentEnvironmentVariables[$key] = $value - - if ($CI) { - if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { - # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. - # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. - LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" - Write-Host "Setting variable as secret '$key'" - } else { - Write-Host "Setting variable '$key': $value" - $notSecretValues += $value - } - LogVsoCommand "##vso[task.setvariable variable=$key;]$value" - } else { - Write-Host ($shellExportFormat -f $key, $value) - } - } - - if ($key) { - # Isolate the environment variables for easy reading. - Write-Host "`n" - $key = $null - } - } - - return $deploymentEnvironmentVariables, $deploymentOutputs -} # Support actions to invoke on exit. $exitActions = @({ @@ -843,31 +579,7 @@ try { -templateFile $templateFile ` -environmentVariables $EnvironmentVariables - $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } - # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access - if ($storageAccounts) { - foreach ($account in $storageAccounts) { - $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name - if ($rules -and $rules.DefaultAction -eq "Allow") { - Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } - if ($CI -and $env:PoolSubnet) { - Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } - } elseif ($AllowIpRanges) { - Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" - $ipRanges = $AllowIpRanges | ForEach-Object { - @{ Action = 'allow'; IPAddressOrRange = $_ } - } - Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } - } elseif (!$CI) { - Write-Host "Enabling access to '$($account.Name)' from client IP" - $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site - Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } - } - } - } - } + SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI $postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath "$ResourceType-resources-post.ps1" if (Test-Path $postDeploymentScript) { diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 490b41b8ebe9b..08ca9d8f5a54d 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -61,6 +61,19 @@ param ( [Parameter()] [switch] $ServicePrincipalAuth, + # List of CIDR ranges to add to specific resource firewalls, e.g. @(10.100.0.0/16, 10.200.0.0/16) + [Parameter()] + [ValidateCount(0,399)] + [Validatescript({ + foreach ($range in $PSItem) { + if ($range -like '*/31' -or $range -like '*/32') { + throw "Firewall IP Ranges cannot contain a /31 or /32 CIDR" + } + } + return $true + })] + [array] $AllowIpRanges = @(), + [Parameter()] [switch] $Force, @@ -69,6 +82,9 @@ param ( $RemoveTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) +. (Join-Path $PSScriptRoot TestResources-Helpers.ps1) + # By default stop for any error. if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' @@ -241,6 +257,9 @@ $verifyDeleteScript = { # Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. $purgeableResources = Get-PurgeableGroupResources $ResourceGroupName +SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI +Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI + Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob diff --git a/eng/common/TestResources/TestResources-Helpers.ps1 b/eng/common/TestResources/TestResources-Helpers.ps1 new file mode 100644 index 0000000000000..6dee017aec9a7 --- /dev/null +++ b/eng/common/TestResources/TestResources-Helpers.ps1 @@ -0,0 +1,267 @@ +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} + +# vso commands are specially formatted log lines that are parsed by Azure Pipelines +# to perform additional actions, most commonly marking values as secrets. +# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands +function LogVsoCommand([string]$message) { + if (!$CI -or $SuppressVsoCommands) { + return + } + Write-Host $message +} + +function Retry([scriptblock] $Action, [int] $Attempts = 5) { + $attempt = 0 + $sleep = 5 + + while ($attempt -lt $Attempts) { + try { + $attempt++ + return $Action.Invoke() + } + catch { + if ($attempt -lt $Attempts) { + $sleep *= 2 + + Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..." + Start-Sleep -Seconds $sleep + } + else { + throw + } + } + } +} + +# NewServicePrincipalWrapper creates an object from an AAD graph or Microsoft Graph service principal object type. +# This is necessary to work around breaking changes introduced in Az version 7.0.0: +# https://azure.microsoft.com/en-us/updates/update-your-apps-to-use-microsoft-graph-before-30-june-2022/ +function NewServicePrincipalWrapper([string]$subscription, [string]$resourceGroup, [string]$displayName) { + if ((Get-Module Az.Resources).Version -eq "5.3.0") { + # https://github.com/Azure/azure-powershell/issues/17040 + # New-AzAdServicePrincipal calls will fail with: + # "You cannot call a method on a null-valued expression." + Write-Warning "Az.Resources version 5.3.0 is not supported. Please update to >= 5.3.1" + Write-Warning "Update-Module Az.Resources -RequiredVersion 5.3.1" + exit 1 + } + + try { + $servicePrincipal = Retry { + New-AzADServicePrincipal -Role "Owner" -Scope "/subscriptions/$SubscriptionId/resourceGroups/$ResourceGroupName" -DisplayName $displayName + } + } + catch { + # The underlying error "The directory object quota limit for the Principal has been exceeded" gets overwritten by the module trying + # to call New-AzADApplication with a null object instead of stopping execution, which makes this case hard to diagnose because it prints the following: + # "Cannot bind argument to parameter 'ObjectId' because it is an empty string." + # Provide a more helpful diagnostic prompt to the user if appropriate: + $totalApps = (Get-AzADApplication -OwnedApplication).Length + $msg = "App Registrations owned by you total $totalApps and may exceed the max quota (likely around 135)." + ` + "`nTry removing some at https://ms.portal.azure.com/#view/Microsoft_AAD_IAM/ActiveDirectoryMenuBlade/~/RegisteredApps" + ` + " or by running the following command to remove apps created by this script:" + ` + "`n Get-AzADApplication -DisplayNameStartsWith '$baseName' | Remove-AzADApplication" + ` + "`nNOTE: You may need to wait for the quota number to be updated after removing unused applications." + Write-Warning $msg + throw + } + + $spPassword = "" + $appId = "" + if (Get-Member -Name "Secret" -InputObject $servicePrincipal -MemberType property) { + Write-Verbose "Using legacy PSADServicePrincipal object type from AAD graph API" + # Secret property exists on PSADServicePrincipal type from AAD graph in Az # module versions < 7.0.0 + $spPassword = $servicePrincipal.Secret + $appId = $servicePrincipal.ApplicationId + } + else { + if ((Get-Module Az.Resources).Version -eq "5.1.0") { + Write-Verbose "Creating password and credential for service principal via MS Graph API" + Write-Warning "Please update Az.Resources to >= 5.2.0 by running 'Update-Module Az'" + # Microsoft graph objects (Az.Resources version == 5.1.0) do not provision a secret on creation so it must be added separately. + # Submitting a password credential object without specifying a password will result in one being generated on the server side. + $password = New-Object -TypeName "Microsoft.Azure.PowerShell.Cmdlets.Resources.MSGraph.Models.ApiV10.MicrosoftGraphPasswordCredential" + $password.DisplayName = "Password for $displayName" + $credential = Retry { New-AzADSpCredential -PasswordCredentials $password -ServicePrincipalObject $servicePrincipal -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + else { + Write-Verbose "Creating service principal credential via MS Graph API" + # In 5.2.0 the password credential issue was fixed (see https://github.com/Azure/azure-powershell/pull/16690) but the + # parameter set was changed making the above call fail due to a missing ServicePrincipalId parameter. + $credential = Retry { $servicePrincipal | New-AzADSpCredential -ErrorAction 'Stop' } + $spPassword = ConvertTo-SecureString $credential.SecretText -AsPlainText -Force + $appId = $servicePrincipal.AppId + } + } + + return @{ + AppId = $appId + ApplicationId = $appId + # This is the ObjectId/OID but most return objects use .Id so keep it consistent to prevent confusion + Id = $servicePrincipal.Id + DisplayName = $servicePrincipal.DisplayName + Secret = $spPassword + } +} + +function LoadCloudConfig([string] $env) { + $configPath = "$PSScriptRoot/clouds/$env.json" + if (!(Test-Path $configPath)) { + Write-Warning "Could not find cloud configuration for environment '$env'" + return @{} + } + + $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable + return $config +} + +function MergeHashes([hashtable] $source, [psvariable] $dest) { + foreach ($key in $source.Keys) { + if ($dest.Value.Contains($key) -and $dest.Value[$key] -ne $source[$key]) { + Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " + + "to new value '$($source[$key])'") + } + $dest.Value[$key] = $source[$key] + } +} + +function BuildBicepFile([System.IO.FileSystemInfo] $file) { + if (!(Get-Command bicep -ErrorAction Ignore)) { + Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See aka.ms/bicep-install" + throw + } + + $tmp = $env:TEMP ? $env:TEMP : [System.IO.Path]::GetTempPath() + $templateFilePath = Join-Path $tmp "$ResourceType-resources.$(New-Guid).compiled.json" + + # Az can deploy bicep files natively, but by compiling here it becomes easier to parse the + # outputted json for mismatched parameter declarations. + bicep build $file.FullName --outfile $templateFilePath + if ($LASTEXITCODE) { + Write-Error "Failure building bicep file '$($file.FullName)'" + throw + } + + return $templateFilePath +} + +function BuildDeploymentOutputs([string]$serviceName, [object]$azContext, [object]$deployment, [hashtable]$environmentVariables) { + $serviceDirectoryPrefix = BuildServiceDirectoryPrefix $serviceName + # Add default values + $deploymentOutputs = [Ordered]@{ + "${serviceDirectoryPrefix}SUBSCRIPTION_ID" = $azContext.Subscription.Id; + "${serviceDirectoryPrefix}RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; + "${serviceDirectoryPrefix}LOCATION" = $resourceGroup.Location; + "${serviceDirectoryPrefix}ENVIRONMENT" = $azContext.Environment.Name; + "${serviceDirectoryPrefix}AZURE_AUTHORITY_HOST" = $azContext.Environment.ActiveDirectoryAuthority; + "${serviceDirectoryPrefix}RESOURCE_MANAGER_URL" = $azContext.Environment.ResourceManagerUrl; + "${serviceDirectoryPrefix}SERVICE_MANAGEMENT_URL" = $azContext.Environment.ServiceManagementUrl; + "AZURE_SERVICE_DIRECTORY" = $serviceName.ToUpperInvariant(); + } + + if ($ServicePrincipalAuth) { + $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_ID"] = $TestApplicationId; + $deploymentOutputs["${serviceDirectoryPrefix}CLIENT_SECRET"] = $TestApplicationSecret; + $deploymentOutputs["${serviceDirectoryPrefix}TENANT_ID"] = $azContext.Tenant.Id; + } + + MergeHashes $environmentVariables $(Get-Variable deploymentOutputs) + + foreach ($key in $deployment.Outputs.Keys) { + $variable = $deployment.Outputs[$key] + + # Work around bug that makes the first few characters of environment variables be lowercase. + $key = $key.ToUpperInvariant() + + if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') { + $deploymentOutputs[$key] = $variable.Value + } + } + + # Force capitalization of all keys to avoid Azure Pipelines confusion with + # variable auto-capitalization and OS env var capitalization differences + $capitalized = @{} + foreach ($item in $deploymentOutputs.GetEnumerator()) { + $capitalized[$item.Name.ToUpperInvariant()] = $item.Value + } + + return $capitalized +} + +function SetDeploymentOutputs( + [string]$serviceName, + [object]$azContext, + [object]$deployment, + [object]$templateFile, + [hashtable]$environmentVariables = @{} +) { + $deploymentEnvironmentVariables = $environmentVariables.Clone() + $deploymentOutputs = BuildDeploymentOutputs $serviceName $azContext $deployment $deploymentEnvironmentVariables + + if ($OutFile) { + if (!$IsWindows) { + Write-Host 'File option is supported only on Windows' + } + + $outputFile = "$($templateFile.originalFilePath).env" + + $environmentText = $deploymentOutputs | ConvertTo-Json; + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + + Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" + } + else { + if (!$CI) { + # Write an extra new line to isolate the environment variables for easy reading. + Log "Persist the following environment variables based on your detected shell ($shell):`n" + } + + # Write overwrite warnings first, since local execution prints a runnable command to export variables + foreach ($key in $deploymentOutputs.Keys) { + if ([Environment]::GetEnvironmentVariable($key)) { + Write-Warning "Deployment outputs will overwrite pre-existing environment variable '$key'" + } + } + + # Marking values as secret by allowed keys below is not sufficient, as there may be outputs set in the ARM/bicep + # file that re-mark those values as secret (since all user-provided deployment outputs are treated as secret by default). + # This variable supports a second check on not marking previously allowed keys/values as secret. + $notSecretValues = @() + foreach ($key in $deploymentOutputs.Keys) { + $value = $deploymentOutputs[$key] + $deploymentEnvironmentVariables[$key] = $value + + if ($CI) { + if (ShouldMarkValueAsSecret $serviceName $key $value $notSecretValues) { + # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. + # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. + LogVsoCommand "##vso[task.setvariable variable=_$key;issecret=true;]$value" + Write-Host "Setting variable as secret '$key'" + } + else { + Write-Host "Setting variable '$key': $value" + $notSecretValues += $value + } + LogVsoCommand "##vso[task.setvariable variable=$key;]$value" + } + else { + Write-Host ($shellExportFormat -f $key, $value) + } + } + + if ($key) { + # Isolate the environment variables for easy reading. + Write-Host "`n" + $key = $null + } + } + + return $deploymentEnvironmentVariables, $deploymentOutputs +} diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index b877d72139a20..025e90dd4c297 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -29,7 +29,9 @@ steps: displayName: Remove test resources condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true - env: ${{ parameters.EnvVars }} + env: + PoolSubnet: $(PoolSubnet) + ${{ insert }}: ${{ parameters.EnvVars }} inputs: azureSubscription: ${{ parameters.ServiceConnection }} azurePowerShellVersion: LatestVersion @@ -46,6 +48,7 @@ steps: @subscriptionConfiguration ` -ResourceType '${{ parameters.ResourceType }}' ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` + -AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') ` -CI ` -Force ` -Verbose @@ -63,10 +66,13 @@ steps: -ResourceType '${{ parameters.ResourceType }}' ` -ServiceDirectory "${{ parameters.ServiceDirectory }}" ` -ServicePrincipalAuth ` + -AllowIpRanges ('$(azsdk-corp-net-ip-ranges)' -split ',') ` -CI ` -Force ` -Verbose displayName: Remove test resources condition: and(eq(variables['CI_HAS_DEPLOYED_RESOURCES'], 'true'), ne(variables['Skip.RemoveTestResources'], 'true')) continueOnError: true - env: ${{ parameters.EnvVars }} + env: + PoolSubnet: $(PoolSubnet) + ${{ insert }}: ${{ parameters.EnvVars }} diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 6c02e9150e24c..938ccfa4b55f8 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -4,7 +4,7 @@ function Get-PurgeableGroupResources { param ( - [Parameter(Mandatory=$true, Position=0)] + [Parameter(Mandatory = $true, Position = 0)] [string] $ResourceGroupName ) @@ -27,8 +27,8 @@ function Get-PurgeableGroupResources { # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled. $deletedKeyVaults = @(Get-AzKeyVault -ResourceGroupName $ResourceGroupName -ErrorAction Ignore | ForEach-Object { - # Enumerating vaults from a resource group does not return all properties we required. - Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } ` + # Enumerating vaults from a resource group does not return all properties we required. + Get-AzKeyVault -VaultName $_.VaultName -ErrorAction Ignore | Where-Object { $_.EnableSoftDelete } ` | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru ` | Add-Member -MemberType AliasProperty -Name AzsdkName -Value VaultName -PassThru }) @@ -56,13 +56,13 @@ function Get-PurgeableResources { $deletedHsms = @() foreach ($r in $content.value) { $deletedHsms += [pscustomobject] @{ - AzsdkResourceType = 'Managed HSM' - AzsdkName = $r.name - Id = $r.id - Name = $r.name - Location = $r.properties.location - DeletionDate = $r.properties.deletionDate -as [DateTime] - ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] + AzsdkResourceType = 'Managed HSM' + AzsdkName = $r.name + Id = $r.id + Name = $r.name + Location = $r.properties.location + DeletionDate = $r.properties.deletionDate -as [DateTime] + ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] EnablePurgeProtection = $r.properties.purgeProtectionEnabled } } @@ -91,7 +91,8 @@ function Get-PurgeableResources { Write-Verbose "Found $($deletedKeyVaults.Count) deleted Key Vaults to potentially purge." $purgeableResources += $deletedKeyVaults } - } catch { } + } + catch { } return $purgeableResources } @@ -100,7 +101,7 @@ function Get-PurgeableResources { # This allows you to pipe a collection and process each item in the collection. filter Remove-PurgeableResources { param ( - [Parameter(Position=0, ValueFromPipeline=$true)] + [Parameter(Position = 0, ValueFromPipeline = $true)] [object[]] $Resource, [Parameter()] @@ -128,7 +129,7 @@ filter Remove-PurgeableResources { # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior. Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue -AsJob ` - | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru + | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru } 'Managed HSM' { @@ -139,18 +140,19 @@ filter Remove-PurgeableResources { # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured. Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2023-02-01" -ErrorAction Ignore -AsJob ` - | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action { - param ( $response ) - if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { - Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." - } elseif ($response.Content) { - $content = $response.Content | ConvertFrom-Json - if ($content.error) { - $err = $content.error - Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" - } - } - }.GetNewClosure() + | Wait-PurgeableResourceJob -Resource $r -Timeout $Timeout -PassThru:$PassThru -Action { + param ( $response ) + if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { + Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." + } + elseif ($response.Content) { + $content = $response.Content | ConvertFrom-Json + if ($content.error) { + $err = $content.error + Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" + } + } + }.GetNewClosure() } default { @@ -167,12 +169,12 @@ function Log($Message) { function Wait-PurgeableResourceJob { param ( - [Parameter(Mandatory=$true, ValueFromPipeline=$true)] + [Parameter(Mandatory = $true, ValueFromPipeline = $true)] $Job, # The resource is used for logging and to return if `-PassThru` is specified # so we can easily see all resources that may be in a bad state when the script has completed. - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] $Resource, # Optional ScriptBlock should define params corresponding to the associated job's `Output` property. @@ -195,7 +197,8 @@ function Wait-PurgeableResourceJob { if ($Action) { $null = $Action.Invoke($result) } - } else { + } + else { Write-Warning "Timed out waiting to purge $($Resource.AzsdkResourceType) '$($Resource.AzsdkName)'. Cancelling job." $Job.Cancel() @@ -204,3 +207,176 @@ function Wait-PurgeableResourceJob { } } } + +# Helper function for removing storage accounts with WORM that sometimes get leaked from live tests not set up to clean +# up their resource policies +function Remove-WormStorageAccounts() { + [CmdletBinding(SupportsShouldProcess = $True)] + param( + [string]$GroupPrefix, + [switch]$CI + ) + + $ErrorActionPreference = 'Stop' + + # Be a little defensive so we don't delete non-live test groups via naming convention + # DO NOT REMOVE THIS + # We call this script from live test pipelines as well, and a string mismatch/error could blow away + # some static storage accounts we rely on + if (!$groupPrefix -or ($CI -and !$GroupPrefix.StartsWith('rg-'))) { + throw "The -GroupPrefix parameter must not be empty, or must start with 'rg-' in CI contexts" + } + + $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' } + + foreach ($group in $groups) { + Write-Host "=========================================" + $accounts = Get-AzStorageAccount -ResourceGroupName $group.ResourceGroupName + if ($accounts) { + foreach ($account in $accounts) { + if ($WhatIfPreference) { + Write-Host "What if: Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + else { + Write-Host "Removing $($account.StorageAccountName) in $($account.ResourceGroupName)" + } + + $hasContainers = ($account.Kind -ne "FileStorage") + + # If it doesn't have containers then we can skip the explicit clean-up of this storage account + if (!$hasContainers) { continue } + + $ctx = New-AzStorageContext -StorageAccountName $account.StorageAccountName + + $immutableBlobs = $ctx ` + | Get-AzStorageContainer ` + | Where-Object { $_.BlobContainerProperties.HasImmutableStorageWithVersioning } ` + | Get-AzStorageBlob + try { + foreach ($blob in $immutableBlobs) { + Write-Host "Removing legal hold - blob: $($blob.Name), account: $($account.StorageAccountName), group: $($group.ResourceGroupName)" + $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null + } + } + catch { + Write-Warning "User must have 'Storage Blob Data Owner' RBAC permission on subscription or resource group" + Write-Error $_ + throw + } + # Sometimes we get a 404 blob not found but can still delete containers, + # and sometimes we must delete the blob if there's a legal hold. + # Try to remove the blob, but keep running regardless. + $succeeded = $false + for ($attempt = 0; $attempt -lt 2; $attempt++) { + if ($succeeded) { + break + } + + try { + Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy + } + catch {} + + try { + $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force + $succeeded = $true + } + catch { + Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" + Write-Warning $_ + } + } + + try { + # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs + $ctx | Get-AzStorageContainer | ForEach-Object { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force } + } + catch { + Write-Warning "Container removal failed. Ignoring the error and trying to delete the storage account." + Write-Warning $_ + } + Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force + } + } + if ($WhatIfPreference) { + Write-Host "What if: Removing resource group $($group.ResourceGroupName)" + } + else { + Remove-AzResourceGroup -ResourceGroupName $group.ResourceGroupName -Force -AsJob + } + } +} + +function SetResourceNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI) { + SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -CI:$CI +} + +function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowIpRanges, [switch]$CI, [switch]$Override) { + $clientIp = $null + $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } + # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access + if ($storageAccounts) { + $appliedRule = $false + foreach ($account in $storageAccounts) { + $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name + if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { + Write-Host "Restricting network rules in storage account '$($account.Name)' to deny access by default" + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -DefaultAction Deny } + if ($CI -and $env:PoolSubnet) { + Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + $appliedRule = $true + } + elseif ($AllowIpRanges) { + Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" + $ipRanges = $AllowIpRanges | ForEach-Object { + @{ Action = 'allow'; IPAddressOrRange = $_ } + } + Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + $appliedRule = $true + } + elseif (!$CI) { + Write-Host "Enabling access to '$($account.Name)' from client IP" + $clientIp ??= Retry { Invoke-RestMethod -Uri 'https://icanhazip.com/' } # cloudflare owned ip site + $clientIp = $clientIp.Trim() + $ipRanges = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name + if ($ipRanges) { + foreach ($range in $ipRanges.IpRules) { + if (DoesSubnetOverlap $range.IPAddressOrRange $clientIp) { + return + } + } + } + Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } + $appliedRule = $true + } + } + } + if ($appliedRule) { + Write-Host "Sleeping for 15 seconds to allow network rules to take effect" + Start-Sleep 15 + } + } +} + +function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { + [System.Net.IPAddress]$overlapIpAddress = $overlapIp + $parsed = $ipOrCidr -split '/' + [System.Net.IPAddress]$baseIp = $parsed[0] + if ($parsed.Length -eq 1) { + return $baseIp -eq $overlapIpAddress + } + + $subnet = $parsed[1] + $subnetNum = [int]$subnet + + $baseMask = [math]::pow(2, 31) + $mask = 0 + for ($i = 0; $i -lt $subnetNum; $i++) { + $mask = $mask + $baseMask; + $baseMask = $baseMask / 2 + } + + return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) +}