Add retries to Invoke-WebRequest, update to PowerShell Core 6 (Azure#…

…7030) - Downloads may occasionally fail due to server or network issues
szimmer-dev · Jul 26, 2019 · a1d20dd · a1d20dd
1 parent 8130715
commit a1d20dd
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 50 deletions.
diff --git a/eng/pipelines/.prettierrc.yml b/eng/pipelines/.prettierrc.yml
@@ -0,0 +1 @@
+singleQuote: false
diff --git a/eng/pipelines/docs.yml b/eng/pipelines/docs.yml
@@ -7,8 +7,8 @@ jobs:
     pool:
       vmImage: vs2017-win2016
     steps:
-      - powershell: |
-          Invoke-WebRequest -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" `
+      - pwsh: |
+          Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" `
           -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/"
         workingDirectory: $(Build.BinariesDirectory)
         displayName: Download and Extract DocFX
@@ -18,9 +18,9 @@ jobs:
           script: $(Build.BinariesDirectory)/docfx/docfx.exe init -q
           workingDirectory: $(Build.SourcesDirectory)
           failOnStderr: true
-      - powershell: ls
+      - pwsh: ls
         workingDirectory: $(Build.SourcesDirectory)/docfx_project/
-      - powershell: |
+      - pwsh: |
           Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/docfx.json" -Destination "$(Build.SourcesDirectory)/docfx_project/" -Force
         displayName: Copy over docfx.json
       - task: CmdLine@2
@@ -40,7 +40,7 @@ jobs:
         inputs:
           scriptPath: "$(Build.SourcesDirectory)/eng/docgeneration/script/CreateTocYml.py"
           arguments: $(Build.SourcesDirectory)
-      - powershell: |
+      - pwsh: |
           Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/api/*" -Destination "$(Build.SourcesDirectory)/docfx_project/api/" -Force
           Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/api/index.md" -Destination "$(Build.SourcesDirectory)/docfx_project/articles/intro.md" -Force
           Copy-Item "$(Build.SourcesDirectory)/Documentation/*" -Destination "$(Build.SourcesDirectory)/docfx_project/articles/"
@@ -61,15 +61,15 @@ jobs:
           scriptPath: "$(Build.SourcesDirectory)/eng/docgeneration/script/CreateTocYmlForArticles.py"
           arguments: $(Build.SourcesDirectory)
           failOnStderr: true
-      - powershell: ls
+      - pwsh: ls
         workingDirectory: $(Build.SourcesDirectory)/docfx_project/articles
       - task: CmdLine@2
         displayName: Build Doc Content
         inputs:
           script: $(Build.BinariesDirectory)/docfx/docfx.exe build
           workingDirectory: $(Build.SourcesDirectory)/docfx_project/
           failOnStderr: true
-      - powershell: |
+      - pwsh: |
           Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/assets/*" -Destination "$(Build.SourcesDirectory)/docfx_project/_site/" -Force
         displayName: Replace site assets
       - task: CopyFiles@2
@@ -83,4 +83,4 @@ jobs:
         condition: succeeded()
         inputs:
           artifactName: docfx_project
-          targetPath: $(Build.ArtifactStagingDirectory)/docfx_project/_site
+          targetPath: $(Build.ArtifactStagingDirectory)/docfx_project/_site
diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml
@@ -5,9 +5,9 @@ jobs:
     pool:
       vmImage: vs2017-win2016
     steps:
-      - powershell: |
+      - pwsh: |
           echo "##vso[build.addbuildtag]Scheduled"
-        displayName: 'Tag scheduled builds'
+        displayName: "Tag scheduled builds"
         condition: and(eq(variables['Build.SourceBranchName'],'master'),eq(variables['Build.Reason'],'Schedule'))
       - task: DotNetCoreInstaller@2
         displayName: "Use .NET Core sdk $(DotNetCoreSDKVersion)"
@@ -49,9 +49,9 @@ jobs:
           path: $(Pipeline.Workspace)/packages
           patterns: "*.nupkg"
       - task: PowerShell@2
-        displayName: 'Analyze dependencies'
+        displayName: "Analyze dependencies"
         inputs:
-          filePath: 'tools/AnalyzeDeps/AnalyzeDeps.ps1'
+          filePath: "tools/AnalyzeDeps/AnalyzeDeps.ps1"
           arguments: '-PackagesPath "$(Pipeline.Workspace)/packages/*.nupkg" -LockfilePath "$(Build.SourcesDirectory)/eng/Packages.Data.props" -OutPath "$(Build.ArtifactStagingDirectory)/dependencies.html"'
           pwsh: true
       - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
@@ -63,7 +63,7 @@ jobs:
         condition: and(succeededOrFailed(), ne(variables['Build.Reason'],'PullRequest'))
       - task: PublishPipelineArtifact@1
         condition: succeededOrFailed()
-        displayName: 'Publish Report Artifacts'
+        displayName: "Publish Report Artifacts"
         inputs:
           artifactName: reports
           path: $(Build.ArtifactStagingDirectory)/dependencies.html
@@ -92,8 +92,8 @@ jobs:
     pool:
       vmImage: "$(OSVmImage)"
     steps:
-      - powershell: |
-          Invoke-WebRequest -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" `
+      - pwsh: |
+          Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" `
           -OutFile "sdk-tools.zip" | Wait-Process; Expand-Archive -Path "sdk-tools.zip" -DestinationPath "./sdk-tools/"
         workingDirectory: "$(Build.BinariesDirectory)"
         displayName: "Download Tools Archive From Github Release and Extract it"
@@ -125,4 +125,4 @@ jobs:
           testResultsFiles: "**/$(TestTargetFramework)*.trx"
           testRunTitle: "$(OSName) $(TestTargetFramework)"
           testResultsFormat: "VSTest"
-          mergeTestResults: true
+          mergeTestResults: true
diff --git a/eng/pipelines/templates/jobs/archetype-sdk-mgmt.yml b/eng/pipelines/templates/jobs/archetype-sdk-mgmt.yml
@@ -5,9 +5,9 @@ jobs:
       vmImage: windows-2019
     steps:
       - task: DotNetCoreInstaller@0
-        displayName: 'Use .NET Core sdk $(DotNetCoreSDKVersion)'
+        displayName: "Use .NET Core sdk $(DotNetCoreSDKVersion)"
         inputs:
-         version: '$(DotNetCoreSDKVersion)'
+          version: "$(DotNetCoreSDKVersion)"
       #- script: "echo $(system.pullrequest.pullrequestnumber), https://github.com/$(build.repository.id), https://github.com/$(build.repository.ID)"
       - script: "dotnet msbuild mgmt.proj /v:m /t:CreateNugetPackage /p:PackageOutputPath=$(Build.ArtifactStagingDirectory) /p:Scope=${{parameters.Scope}} /p:ForPublishing=$(ShouldPublish) $(loggingArgs) $(RPScopeArgs)"
         displayName: "Build & Package"
@@ -18,19 +18,19 @@ jobs:
       - task: PublishBuildArtifacts@1
         condition: succeededOrFailed()
         inputs:
-            pathtoPublish: $(msBuildLogDir)
-            artifactName: MsBuildLogs
+          pathtoPublish: $(msBuildLogDir)
+          artifactName: MsBuildLogs
       - task: PublishPipelineArtifact@0
         condition: succeededOrFailed()
         inputs:
           artifactName: packages
           targetPath: $(Build.ArtifactStagingDirectory)
-      - powershell: |
+      - pwsh: |
           $releaseTag = "FOR_SIGNING"
           if ("$(ShouldPublish)" -eq "true") { $releaseTag = "FOR_PUBLISHING" }
           echo "Tagging build with $releaseTag"
           echo "##vso[build.addbuildtag]$releaseTag"
-        displayName: 'Tagging the build'
+        displayName: "Tagging the build"
         condition: ne(variables['System.TeamProject'], 'public')
 
   - job: Test
@@ -51,9 +51,9 @@ jobs:
       vmImage: "$(OSVmImage)"
     steps:
       - task: DotNetCoreInstaller@0
-        displayName: 'Use .NET Core sdk $(DotNetCoreSDKVersion)'
+        displayName: "Use .NET Core sdk $(DotNetCoreSDKVersion)"
         inputs:
-          version: '$(DotNetCoreSDKVersion)'
+          version: "$(DotNetCoreSDKVersion)"
       - script: "echo $(system.pullrequest.pullrequestnumber), http://github.com/$(build.repository.id), http://github.com/$(build.repository.ID)"
       - script: "dotnet msbuild mgmt.proj /v:n /t:RunTests /p:Scope=${{parameters.Scope}} /p:ForPublishing=$(ShouldPublish) /clp:ShowtimeStamp $(RPScopeArgs)"
         displayName: "Build & Run Tests"

diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml
@@ -4,10 +4,9 @@ parameters:
   MaxParallel: 0
 
 jobs:
-
-  - job: 'Test'
+  - job: "Test"
     variables:
-    - template: ../variables/globals.yml
+      - template: ../variables/globals.yml
 
     # Increase timeout so Event Hubs tests do not timeout (Windows runs for ~2h 35m, on average due to having two target platforms)
     # https://github.com/Azure/azure-sdk-for-net/issues/5982
@@ -17,8 +16,8 @@ jobs:
       maxParallel: ${{ parameters.MaxParallel }}
       matrix:
         Linux:
-          OSName: 'Linux'
-          OSVmImage: 'ubuntu-16.04'
+          OSName: "Linux"
+          OSVmImage: "ubuntu-16.04"
           TestTargetFramework: netcoreapp2.1
         Windows_NetCoreApp:
           OSName: "Windows"
@@ -33,21 +32,21 @@ jobs:
           OSVmImage: "macOS-10.13"
           TestTargetFramework: netcoreapp2.1
     pool:
-      vmImage: '$(OSVmImage)'
+      vmImage: "$(OSVmImage)"
 
     steps:
       - ${{ parameters.PreSteps }}
 
-      - powershell: |
-          Invoke-WebRequest -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" `
+      - pwsh: |
+          Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" `
           -OutFile "sdk-tools.zip" | Wait-Process; Expand-Archive -Path "sdk-tools.zip" -DestinationPath "./sdk-tools/"
-        workingDirectory: '$(Build.BinariesDirectory)'
-        displayName: 'Download Tools Archive From Github Release and Extract it'
+        workingDirectory: "$(Build.BinariesDirectory)"
+        displayName: "Download Tools Archive From Github Release and Extract it"
 
       - task: PythonScript@0
-        displayName: 'Run VerifyAgentOS script'
+        displayName: "Run VerifyAgentOS script"
         inputs:
-          scriptPath: '$(Build.BinariesDirectory)/sdk-tools/scripts/python/verify_agent_os.py'
+          scriptPath: "$(Build.BinariesDirectory)/sdk-tools/scripts/python/verify_agent_os.py"
           arguments: $(OSName)
 
       - task: DotNetCoreInstaller@2
@@ -63,7 +62,7 @@ jobs:
           version: "$(DotNetCoreSDKVersion)"
 
       - script: dotnet test eng/service.proj --framework $(TestTargetFramework) --logger "trx;LogFileName=$(TestTargetFramework).trx" /p:ServiceDirectory=${{ parameters.ServiceDirectory }} /p:IncludeSrc=false
-        displayName: 'Build & Test (all tests for $(TestTargetFramework))'
+        displayName: "Build & Test (all tests for $(TestTargetFramework))"
         env:
           DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
           DOTNET_CLI_TELEMETRY_OPTOUT: 1
@@ -75,7 +74,7 @@ jobs:
         condition: succeededOrFailed()
         displayName: "Publish Results ($(TestTargetFramework))"
         inputs:
-          testResultsFiles: '**/$(TestTargetFramework)*.trx'
-          testRunTitle: '$(OSName) $(TestTargetFramework)'
-          testResultsFormat: 'VSTest'
+          testResultsFiles: "**/$(TestTargetFramework)*.trx"
+          testRunTitle: "$(OSName) $(TestTargetFramework)"
+          testResultsFormat: "VSTest"
           mergeTestResults: true
diff --git a/sdk/storage/tests.yml b/sdk/storage/tests.yml
@@ -1,14 +1,14 @@
 trigger: none
 
 jobs:
-- template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
-  parameters:
-    ServiceDirectory: storage
-    PreSteps:
-      - powershell: |
-          $TestConfigurationPath = "$(Build.ArtifactStagingDirectory)/TestConfiguration.xml"
-          '$(net-storage-test-configuration)' | Out-File -Encoding Utf8 $TestConfigurationPath
-          Write-Host "##vso[task.setvariable variable=TestConfigurationPath]$TestConfigurationPath"
+  - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
+    parameters:
+      ServiceDirectory: storage
+      PreSteps:
+        - pwsh: |
+            $TestConfigurationPath = "$(Build.ArtifactStagingDirectory)/TestConfiguration.xml"
+            '$(net-storage-test-configuration)' | Out-File -Encoding Utf8 $TestConfigurationPath
+            Write-Host "##vso[task.setvariable variable=TestConfigurationPath]$TestConfigurationPath"
 
-    EnvVars:
-      AZ_STORAGE_CONFIG_PATH: $(TestConfigurationPath)
+      EnvVars:
+        AZ_STORAGE_CONFIG_PATH: $(TestConfigurationPath)