From efa223b53adb6062f8d22b2dd56402b902cad9b2 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 3 Jul 2024 19:37:07 -0400 Subject: [PATCH] Add sleep for network rule application --- .../TestResources/New-TestResources.ps1 | 1 + .../TestResources/Remove-TestResources.ps1 | 4 ++-- .../scripts/Helpers/Resource-Helpers.ps1 | 21 ++++++++++++++----- eng/scripts/live-test-resource-cleanup.ps1 | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 1af87d50de1..6ccf55a781c 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -117,6 +117,7 @@ param ( $NewTestResourcesRemainingArguments ) +. (Join-Path $PSScriptRoot .. scripts Helpers Resource-Helpers.ps1) . $PSScriptRoot/TestResources-Helpers.ps1 . $PSScriptRoot/SubConfig-Helpers.ps1 diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index b5d10fb11f5..08ca9d8f5a5 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -257,8 +257,8 @@ $verifyDeleteScript = { # Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. $purgeableResources = Get-PurgeableGroupResources $ResourceGroupName -SetStorageNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI -Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName +SetResourceNetworkAccessRules -ResourceGroupName $ResourceGroupName -AllowIpRanges $AllowIpRanges -Override -CI:$CI +Remove-WormStorageAccounts -GroupPrefix $ResourceGroupName -CI:$CI Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index a60a17df5af..938ccfa4b55 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -213,7 +213,8 @@ function Wait-PurgeableResourceJob { function Remove-WormStorageAccounts() { [CmdletBinding(SupportsShouldProcess = $True)] param( - [string]$GroupPrefix + [string]$GroupPrefix, + [switch]$CI ) $ErrorActionPreference = 'Stop' @@ -222,8 +223,8 @@ function Remove-WormStorageAccounts() { # DO NOT REMOVE THIS # We call this script from live test pipelines as well, and a string mismatch/error could blow away # some static storage accounts we rely on - if (!$groupPrefix -or !$GroupPrefix.StartsWith('rg-')) { - throw "The -GroupPrefix parameter must start with 'rg-'" + if (!$groupPrefix -or ($CI -and !$GroupPrefix.StartsWith('rg-'))) { + throw "The -GroupPrefix parameter must not be empty, or must start with 'rg-' in CI contexts" } $groups = Get-AzResourceGroup | Where-Object { $_.ResourceGroupName.StartsWith($GroupPrefix) } | Where-Object { $_.ProvisioningState -ne 'Deleting' } @@ -274,12 +275,14 @@ function Remove-WormStorageAccounts() { try { Write-Host "Removing immutability policies - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" $null = $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlobImmutabilityPolicy - } catch {} + } + catch {} try { $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force $succeeded = $true - } catch { + } + catch { Write-Warning "Failed to remove blobs - account: $($ctx.StorageAccountName), group: $($group.ResourceGroupName)" Write-Warning $_ } @@ -314,6 +317,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI $storageAccounts = Retry { Get-AzResource -ResourceGroupName $ResourceGroupName -ResourceType "Microsoft.Storage/storageAccounts" } # Add client IP to storage account when running as local user. Pipeline's have their own vnet with access if ($storageAccounts) { + $appliedRule = $false foreach ($account in $storageAccounts) { $rules = Get-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -AccountName $account.Name if ($rules -and ($Override -or $rules.DefaultAction -eq "Allow")) { @@ -322,6 +326,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI if ($CI -and $env:PoolSubnet) { Write-Host "Enabling access to '$($account.Name)' from pipeline subnet $($env:PoolSubnet)" Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -VirtualNetworkResourceId $env:PoolSubnet } + $appliedRule = $true } elseif ($AllowIpRanges) { Write-Host "Enabling access to '$($account.Name)' to $($AllowIpRanges.Length) IP ranges" @@ -329,6 +334,7 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI @{ Action = 'allow'; IPAddressOrRange = $_ } } Retry { Update-AzStorageAccountNetworkRuleSet -ResourceGroupName $ResourceGroupName -Name $account.Name -IPRule $ipRanges | Out-Null } + $appliedRule = $true } elseif (!$CI) { Write-Host "Enabling access to '$($account.Name)' from client IP" @@ -343,9 +349,14 @@ function SetStorageNetworkAccessRules([string]$ResourceGroupName, [array]$AllowI } } Retry { Add-AzStorageAccountNetworkRule -ResourceGroupName $ResourceGroupName -Name $account.Name -IPAddressOrRange $clientIp | Out-Null } + $appliedRule = $true } } } + if ($appliedRule) { + Write-Host "Sleeping for 15 seconds to allow network rules to take effect" + Start-Sleep 15 + } } } diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1 index b5ec2add5d1..7d5ee72be31 100644 --- a/eng/scripts/live-test-resource-cleanup.ps1 +++ b/eng/scripts/live-test-resource-cleanup.ps1 @@ -436,7 +436,7 @@ function DeleteAndPurgeGroups([array]$toDelete) { # can be left around which prevent deletion. if ($rg.Tags?.ContainsKey('ServiceDirectory') -and $rg.Tags.ServiceDirectory -like '*storage*') { SetStorageNetworkAccessRules -ResourceGroupName $rg.ResourceGroupName -Override -CI:($null -ne $env:SYSTEM_TEAMPROJECTID) - Remove-WormStorageAccounts -GroupPrefix $rg.ResourceGroupName + Remove-WormStorageAccounts -GroupPrefix $rg.ResourceGroupName -CI:($null -ne $env:SYSTEM_TEAMPROJECTID) } else { Write-Host ($rg | Remove-AzResourceGroup -Force -AsJob).Name }