Retry network operations in resources scripts (#7510)

* Retry network operations in resources scripts * Pass -CI switch parameter
Azure · Jan 17, 2020 · a18cd1a · a18cd1a
1 parent fc90101
commit a18cd1a
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 10 deletions.
diff --git a/eng/New-TestResources.ps1 b/eng/New-TestResources.ps1
@@ -71,6 +71,27 @@ function Log($Message) {
     Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
 
+function Retry([scriptblock] $Action, [int] $Attempts = 5) {
+    $attempt = 0
+    $sleep = 5
+
+    while ($attempt -lt $Attempts) {
+        try {
+            $attempt++
+            return $Action.Invoke()
+        } catch {
+            if ($attempt -lt $Attempts) {
+                $sleep *= 2
+
+                Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..."
+                Start-Sleep -Seconds $sleep
+            } else {
+                Write-Error -ErrorRecord $_
+            }
+        }
+    }
+}
+
 # Support actions to invoke on exit.
 $exitActions = @({
     if ($exitActions.Count -gt 1) {
@@ -108,7 +129,10 @@ if ($ProvisionerApplicationId) {
     Log "Logging into service principal '$ProvisionerApplicationId'"
     $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force
     $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret)
-    $provisionerAccount = Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal
+
+    $provisionerAccount = Retry {
+        Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal
+    }
 
     $exitActions += {
         Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'"
@@ -118,7 +142,10 @@ if ($ProvisionerApplicationId) {
 
 # Get test application OID from ID if not already provided.
 if ($TestApplicationId -and !$TestApplicationOid) {
-    $testServicePrincipal = Get-AzADServicePrincipal -ApplicationId $TestApplicationId
+    $testServicePrincipal = Retry {
+        Get-AzADServicePrincipal -ApplicationId $TestApplicationId
+    }
+
     if ($testServicePrincipal -and $testServicePrincipal.Id) {
         $script:TestApplicationOid = $testServicePrincipal.Id
     }
@@ -160,7 +187,10 @@ if ($CI) {
 }
 
 Log "Creating resource group '$resourceGroupName' in location '$Location'"
-$resourceGroup = New-AzResourceGroup -Name "$resourceGroupName" -Location $Location -Tag $tags -Force:$Force
+$resourceGroup = Retry {
+    New-AzResourceGroup -Name "$resourceGroupName" -Location $Location -Tag $tags -Force:$Force
+}
+
 if ($resourceGroup.ProvisioningState -eq 'Succeeded') {
     # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway.
     Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'"
@@ -213,7 +243,10 @@ foreach ($templateFile in $templateFiles) {
     }
 
     Log "Deploying template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'"
-    $deployment = New-AzResourceGroupDeployment -Name $BaseName -ResourceGroupName $resourceGroup.ResourceGroupName -TemplateFile $templateFile -TemplateParameterObject $templateFileParameters
+    $deployment = Retry {
+        New-AzResourceGroupDeployment -Name $BaseName -ResourceGroupName $resourceGroup.ResourceGroupName -TemplateFile $templateFile -TemplateParameterObject $templateFileParameters
+    }
+
     if ($deployment.ProvisioningState -eq 'Succeeded') {
         # New-AzResourceGroupDeployment would've written an error and stopped the pipeline by default anyway.
         Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'"
@@ -320,4 +353,4 @@ To create a service principal in your current subscription, run: New-AzADService
 
 .LINK
 Remove-TestResources.ps1
-#>
+#>
diff --git a/eng/Remove-TestResources.ps1 b/eng/Remove-TestResources.ps1
@@ -47,6 +47,27 @@ function Log($Message) {
     Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
 
+function Retry([scriptblock] $Action, [int] $Attempts = 5) {
+    $attempt = 0
+    $sleep = 5
+
+    while ($attempt -lt $Attempts) {
+        try {
+            $attempt++
+            return $Action.Invoke()
+        } catch {
+            if ($attempt -lt $Attempts) {
+                $sleep *= 2
+
+                Write-Warning "Attempt $attempt failed: $_. Trying again in $sleep seconds..."
+                Start-Sleep -Seconds $sleep
+            } else {
+                Write-Error -ErrorRecord $_
+            }
+        }
+    }
+}
+
 # Support actions to invoke on exit.
 $exitActions = @({
     if ($exitActions.Count -gt 1) {
@@ -65,7 +86,9 @@ if ($ProvisionerApplicationId) {
     Log "Logging into service principal '$ProvisionerApplicationId'"
     $provisionerSecret = ConvertTo-SecureString -String $ProvisionerApplicationSecret -AsPlainText -Force
     $provisionerCredential = [System.Management.Automation.PSCredential]::new($ProvisionerApplicationId, $provisionerSecret)
-    $provisionerAccount = Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal
+    $provisionerAccount = Retry {
+        Connect-AzAccount -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal
+    }
 
     $exitActions += {
         Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'"
@@ -79,7 +102,7 @@ if (!$ResourceGroupName) {
 }
 
 Log "Deleting resource group '$ResourceGroupName'"
-if (Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force) {
+if (Retry { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force }) {
     Write-Verbose "Successfully deleted resource group '$ResourceGroupName'"
 }
 
@@ -122,4 +145,4 @@ Use the currently logged-in account to delete the resource group provisioned by
 
 .LINK
 New-TestResources.ps1
-#>
+#>
diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml
@@ -82,7 +82,7 @@ jobs:
             -ProvisionerApplicationId '$(provisioner-aad-id)'
             -ProvisionerApplicationSecret '$(provisioner-aad-secret)'
             -DeleteAfterHours 24
-            -Location westus2
+            -CI
             -Force
             -Verbose
           displayName: Provision Test Resources
@@ -107,7 +107,7 @@ jobs:
             -ProvisionerApplicationSecret '$(provisioner-aad-secret)'
             -Force
             -Verbose
-          condition: and(always(), ne(variables['AZURE_RESOURCEGROUP_NAME'], ''))
+          condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '')
           continueOnError: true
           displayName: Remove Test Resources