From 6246c9680f87959b071e8d46d15160f289351d28 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 16:10:53 -0500 Subject: [PATCH 1/5] Retry container deletion --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index ce8e40d15f48..f5e9d29b4db8 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -345,7 +345,9 @@ function RemoveStorageAccount($Account) { if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs - Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force + # If blob deletion takes longer then the container delete + # will fail, so retry a couple times + Retry -Action { Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force } -Attempts 2 #$container | Remove-AzStorageContainer } catch { Write-Host "Container removal failed: $($container.Name), account: $($Account.storageAccountName), group: $($Account.ResourceGroupName)" From 8286e057b6d88ab3742ad0eb68bd183c1ea2bdec Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 12:37:54 -0500 Subject: [PATCH 2/5] Do not try to purge keyvaults with purge protection --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index f5e9d29b4db8..de843a559fe9 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -123,8 +123,8 @@ filter Remove-PurgeableResources { switch ($r.AzsdkResourceType) { 'Key Vault' { if ($r.EnablePurgeProtection) { - # We will try anyway but will ignore errors. Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged until $($r.ScheduledPurgeDate)" + continue } # Use `-AsJob` to start a lightweight, cancellable job and pass to `Wait-PurgeableResoruceJob` for consistent behavior. @@ -134,8 +134,8 @@ filter Remove-PurgeableResources { 'Managed HSM' { if ($r.EnablePurgeProtection) { - # We will try anyway but will ignore errors. Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged until $($r.ScheduledPurgeDate)" + continue } # Use `GetNewClosure()` on the `-Action` ScriptBlock to make sure variables are captured. From acb5854d54c355c08e16145eecdf13cc9441b5d6 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 16:43:51 -0500 Subject: [PATCH 3/5] Delete all blobs when container has immutability --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index de843a559fe9..5c30d3b9a729 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -345,10 +345,9 @@ function RemoveStorageAccount($Account) { if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs - # If blob deletion takes longer then the container delete - # will fail, so retry a couple times - Retry -Action { Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force } -Attempts 2 - #$container | Remove-AzStorageContainer + # Add a retry in case blob deletion has not finished in time for container deletion, but not too many that we end up + # getting throttled by ARM/SRP if things are actually in a stuck state + Retry -Attempts 1 -Action { Remove-AzRmStorageContainer -Name $container.Name -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName -Force } } catch { Write-Host "Container removal failed: $($container.Name), account: $($Account.storageAccountName), group: $($Account.ResourceGroupName)" Write-Warning "Ignoring the error and trying to delete the storage account" @@ -362,7 +361,7 @@ function RemoveStorageAccount($Account) { } } -function EnableBlobDeletion($Blob, $StorageAccountName, $ResourceGroupName) { +function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGroupName) { # Some properties like immutability policies require the blob to be # deleted before the container can be deleted $forceBlobDeletion = $false @@ -396,6 +395,10 @@ function EnableBlobDeletion($Blob, $StorageAccountName, $ResourceGroupName) { $Blob.ICloudBlob.BreakLease() } + if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + $forceBlobDeletion = $true + } + return $forceBlobDeletion } From 0ef259a0f78ee0f56e6cd93bace12eb2c5f40df6 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Tue, 5 Nov 2024 18:10:01 -0500 Subject: [PATCH 4/5] Skip missing blob container properties --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 5c30d3b9a729..42f5b1bc1174 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -342,6 +342,9 @@ function RemoveStorageAccount($Account) { } foreach ($container in $containers) { + if (!($container | Get-Member 'BlobContainerProperties')) { + continue + } if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { try { # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs @@ -395,7 +398,7 @@ function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGro $Blob.ICloudBlob.BreakLease() } - if ($container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + if (($container | Get-Member 'BlobContainerProperties') -and $container.BlobContainerProperties.HasImmutableStorageWithVersioning) { $forceBlobDeletion = $true } From e1acf7315b4fa01209f9cdb6836201c1d5d40ead Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Wed, 6 Nov 2024 16:18:40 -0500 Subject: [PATCH 5/5] Fix null container --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 42f5b1bc1174..b152a3f21889 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -313,14 +313,16 @@ function RemoveStorageAccount($Account) { if ($Account.Kind -eq "FileStorage") { return } $containers = New-AzStorageContext -StorageAccountName $Account.StorageAccountName | Get-AzStorageContainer - $blobs = $containers | Get-AzStorageBlob $deleteNow = @() try { - foreach ($blob in $blobs) { - $shouldDelete = EnableBlobDeletion -Blob $blob -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName - if ($shouldDelete) { - $deleteNow += $blob + foreach ($container in $containers) { + $blobs = $container | Get-AzStorageBlob + foreach ($blob in $blobs) { + $shouldDelete = EnableBlobDeletion -Blob $blob -Container $container -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName + if ($shouldDelete) { + $deleteNow += $blob + } } } } catch { @@ -398,7 +400,7 @@ function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGro $Blob.ICloudBlob.BreakLease() } - if (($container | Get-Member 'BlobContainerProperties') -and $container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + if (($Container | Get-Member 'BlobContainerProperties') -and $Container.BlobContainerProperties.HasImmutableStorageWithVersioning) { $forceBlobDeletion = $true }