From a1d20dd22bed19f9c2d769cf37abc51372531a54 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Thu, 25 Jul 2019 17:19:22 -0700 Subject: [PATCH] Add retries to Invoke-WebRequest, update to PowerShell Core 6 (#7030) - Downloads may occasionally fail due to server or network issues --- eng/pipelines/.prettierrc.yml | 1 + eng/pipelines/docs.yml | 16 +++++----- .../templates/jobs/archetype-sdk-client.yml | 16 +++++----- .../templates/jobs/archetype-sdk-mgmt.yml | 16 +++++----- .../templates/jobs/archetype-sdk-tests.yml | 31 +++++++++---------- sdk/storage/tests.yml | 20 ++++++------ 6 files changed, 50 insertions(+), 50 deletions(-) create mode 100644 eng/pipelines/.prettierrc.yml diff --git a/eng/pipelines/.prettierrc.yml b/eng/pipelines/.prettierrc.yml new file mode 100644 index 0000000000000..abeaf738769f4 --- /dev/null +++ b/eng/pipelines/.prettierrc.yml @@ -0,0 +1 @@ +singleQuote: false diff --git a/eng/pipelines/docs.yml b/eng/pipelines/docs.yml index d8907df095c4a..6173083669e00 100644 --- a/eng/pipelines/docs.yml +++ b/eng/pipelines/docs.yml @@ -7,8 +7,8 @@ jobs: pool: vmImage: vs2017-win2016 steps: - - powershell: | - Invoke-WebRequest -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" ` + - pwsh: | + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/dotnet/docfx/releases/download/v2.43.2/docfx.zip" ` -OutFile "docfx.zip" | Wait-Process; Expand-Archive -Path "docfx.zip" -DestinationPath "./docfx/" workingDirectory: $(Build.BinariesDirectory) displayName: Download and Extract DocFX @@ -18,9 +18,9 @@ jobs: script: $(Build.BinariesDirectory)/docfx/docfx.exe init -q workingDirectory: $(Build.SourcesDirectory) failOnStderr: true - - powershell: ls + - pwsh: ls workingDirectory: $(Build.SourcesDirectory)/docfx_project/ - - powershell: | + - pwsh: | Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/docfx.json" -Destination "$(Build.SourcesDirectory)/docfx_project/" -Force displayName: Copy over docfx.json - task: CmdLine@2 @@ -40,7 +40,7 @@ jobs: inputs: scriptPath: "$(Build.SourcesDirectory)/eng/docgeneration/script/CreateTocYml.py" arguments: $(Build.SourcesDirectory) - - powershell: | + - pwsh: | Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/api/*" -Destination "$(Build.SourcesDirectory)/docfx_project/api/" -Force Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/api/index.md" -Destination "$(Build.SourcesDirectory)/docfx_project/articles/intro.md" -Force Copy-Item "$(Build.SourcesDirectory)/Documentation/*" -Destination "$(Build.SourcesDirectory)/docfx_project/articles/" @@ -61,7 +61,7 @@ jobs: scriptPath: "$(Build.SourcesDirectory)/eng/docgeneration/script/CreateTocYmlForArticles.py" arguments: $(Build.SourcesDirectory) failOnStderr: true - - powershell: ls + - pwsh: ls workingDirectory: $(Build.SourcesDirectory)/docfx_project/articles - task: CmdLine@2 displayName: Build Doc Content @@ -69,7 +69,7 @@ jobs: script: $(Build.BinariesDirectory)/docfx/docfx.exe build workingDirectory: $(Build.SourcesDirectory)/docfx_project/ failOnStderr: true - - powershell: | + - pwsh: | Copy-Item "$(Build.SourcesDirectory)/eng/docgeneration/assets/*" -Destination "$(Build.SourcesDirectory)/docfx_project/_site/" -Force displayName: Replace site assets - task: CopyFiles@2 @@ -83,4 +83,4 @@ jobs: condition: succeeded() inputs: artifactName: docfx_project - targetPath: $(Build.ArtifactStagingDirectory)/docfx_project/_site \ No newline at end of file + targetPath: $(Build.ArtifactStagingDirectory)/docfx_project/_site diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 7cb41aa5b04e3..bf25818fa394d 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -5,9 +5,9 @@ jobs: pool: vmImage: vs2017-win2016 steps: - - powershell: | + - pwsh: | echo "##vso[build.addbuildtag]Scheduled" - displayName: 'Tag scheduled builds' + displayName: "Tag scheduled builds" condition: and(eq(variables['Build.SourceBranchName'],'master'),eq(variables['Build.Reason'],'Schedule')) - task: DotNetCoreInstaller@2 displayName: "Use .NET Core sdk $(DotNetCoreSDKVersion)" @@ -49,9 +49,9 @@ jobs: path: $(Pipeline.Workspace)/packages patterns: "*.nupkg" - task: PowerShell@2 - displayName: 'Analyze dependencies' + displayName: "Analyze dependencies" inputs: - filePath: 'tools/AnalyzeDeps/AnalyzeDeps.ps1' + filePath: "tools/AnalyzeDeps/AnalyzeDeps.ps1" arguments: '-PackagesPath "$(Pipeline.Workspace)/packages/*.nupkg" -LockfilePath "$(Build.SourcesDirectory)/eng/Packages.Data.props" -OutPath "$(Build.ArtifactStagingDirectory)/dependencies.html"' pwsh: true - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 @@ -63,7 +63,7 @@ jobs: condition: and(succeededOrFailed(), ne(variables['Build.Reason'],'PullRequest')) - task: PublishPipelineArtifact@1 condition: succeededOrFailed() - displayName: 'Publish Report Artifacts' + displayName: "Publish Report Artifacts" inputs: artifactName: reports path: $(Build.ArtifactStagingDirectory)/dependencies.html @@ -92,8 +92,8 @@ jobs: pool: vmImage: "$(OSVmImage)" steps: - - powershell: | - Invoke-WebRequest -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" ` + - pwsh: | + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" ` -OutFile "sdk-tools.zip" | Wait-Process; Expand-Archive -Path "sdk-tools.zip" -DestinationPath "./sdk-tools/" workingDirectory: "$(Build.BinariesDirectory)" displayName: "Download Tools Archive From Github Release and Extract it" @@ -125,4 +125,4 @@ jobs: testResultsFiles: "**/$(TestTargetFramework)*.trx" testRunTitle: "$(OSName) $(TestTargetFramework)" testResultsFormat: "VSTest" - mergeTestResults: true \ No newline at end of file + mergeTestResults: true diff --git a/eng/pipelines/templates/jobs/archetype-sdk-mgmt.yml b/eng/pipelines/templates/jobs/archetype-sdk-mgmt.yml index d0284c12d1de0..34c09e77e2f70 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-mgmt.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-mgmt.yml @@ -5,9 +5,9 @@ jobs: vmImage: windows-2019 steps: - task: DotNetCoreInstaller@0 - displayName: 'Use .NET Core sdk $(DotNetCoreSDKVersion)' + displayName: "Use .NET Core sdk $(DotNetCoreSDKVersion)" inputs: - version: '$(DotNetCoreSDKVersion)' + version: "$(DotNetCoreSDKVersion)" #- script: "echo $(system.pullrequest.pullrequestnumber), https://github.com/$(build.repository.id), https://github.com/$(build.repository.ID)" - script: "dotnet msbuild mgmt.proj /v:m /t:CreateNugetPackage /p:PackageOutputPath=$(Build.ArtifactStagingDirectory) /p:Scope=${{parameters.Scope}} /p:ForPublishing=$(ShouldPublish) $(loggingArgs) $(RPScopeArgs)" displayName: "Build & Package" @@ -18,19 +18,19 @@ jobs: - task: PublishBuildArtifacts@1 condition: succeededOrFailed() inputs: - pathtoPublish: $(msBuildLogDir) - artifactName: MsBuildLogs + pathtoPublish: $(msBuildLogDir) + artifactName: MsBuildLogs - task: PublishPipelineArtifact@0 condition: succeededOrFailed() inputs: artifactName: packages targetPath: $(Build.ArtifactStagingDirectory) - - powershell: | + - pwsh: | $releaseTag = "FOR_SIGNING" if ("$(ShouldPublish)" -eq "true") { $releaseTag = "FOR_PUBLISHING" } echo "Tagging build with $releaseTag" echo "##vso[build.addbuildtag]$releaseTag" - displayName: 'Tagging the build' + displayName: "Tagging the build" condition: ne(variables['System.TeamProject'], 'public') - job: Test @@ -51,9 +51,9 @@ jobs: vmImage: "$(OSVmImage)" steps: - task: DotNetCoreInstaller@0 - displayName: 'Use .NET Core sdk $(DotNetCoreSDKVersion)' + displayName: "Use .NET Core sdk $(DotNetCoreSDKVersion)" inputs: - version: '$(DotNetCoreSDKVersion)' + version: "$(DotNetCoreSDKVersion)" - script: "echo $(system.pullrequest.pullrequestnumber), http://github.com/$(build.repository.id), http://github.com/$(build.repository.ID)" - script: "dotnet msbuild mgmt.proj /v:n /t:RunTests /p:Scope=${{parameters.Scope}} /p:ForPublishing=$(ShouldPublish) /clp:ShowtimeStamp $(RPScopeArgs)" displayName: "Build & Run Tests" diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml index ecd6789a3dba8..578c50b9678a1 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml @@ -4,10 +4,9 @@ parameters: MaxParallel: 0 jobs: - - - job: 'Test' + - job: "Test" variables: - - template: ../variables/globals.yml + - template: ../variables/globals.yml # Increase timeout so Event Hubs tests do not timeout (Windows runs for ~2h 35m, on average due to having two target platforms) # https://github.com/Azure/azure-sdk-for-net/issues/5982 @@ -17,8 +16,8 @@ jobs: maxParallel: ${{ parameters.MaxParallel }} matrix: Linux: - OSName: 'Linux' - OSVmImage: 'ubuntu-16.04' + OSName: "Linux" + OSVmImage: "ubuntu-16.04" TestTargetFramework: netcoreapp2.1 Windows_NetCoreApp: OSName: "Windows" @@ -33,21 +32,21 @@ jobs: OSVmImage: "macOS-10.13" TestTargetFramework: netcoreapp2.1 pool: - vmImage: '$(OSVmImage)' + vmImage: "$(OSVmImage)" steps: - ${{ parameters.PreSteps }} - - powershell: | - Invoke-WebRequest -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" ` + - pwsh: | + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://github.com/Azure/azure-sdk-tools/releases/download/sdk-tools_14793/sdk-tools.zip" ` -OutFile "sdk-tools.zip" | Wait-Process; Expand-Archive -Path "sdk-tools.zip" -DestinationPath "./sdk-tools/" - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Download Tools Archive From Github Release and Extract it' + workingDirectory: "$(Build.BinariesDirectory)" + displayName: "Download Tools Archive From Github Release and Extract it" - task: PythonScript@0 - displayName: 'Run VerifyAgentOS script' + displayName: "Run VerifyAgentOS script" inputs: - scriptPath: '$(Build.BinariesDirectory)/sdk-tools/scripts/python/verify_agent_os.py' + scriptPath: "$(Build.BinariesDirectory)/sdk-tools/scripts/python/verify_agent_os.py" arguments: $(OSName) - task: DotNetCoreInstaller@2 @@ -63,7 +62,7 @@ jobs: version: "$(DotNetCoreSDKVersion)" - script: dotnet test eng/service.proj --framework $(TestTargetFramework) --logger "trx;LogFileName=$(TestTargetFramework).trx" /p:ServiceDirectory=${{ parameters.ServiceDirectory }} /p:IncludeSrc=false - displayName: 'Build & Test (all tests for $(TestTargetFramework))' + displayName: "Build & Test (all tests for $(TestTargetFramework))" env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_CLI_TELEMETRY_OPTOUT: 1 @@ -75,7 +74,7 @@ jobs: condition: succeededOrFailed() displayName: "Publish Results ($(TestTargetFramework))" inputs: - testResultsFiles: '**/$(TestTargetFramework)*.trx' - testRunTitle: '$(OSName) $(TestTargetFramework)' - testResultsFormat: 'VSTest' + testResultsFiles: "**/$(TestTargetFramework)*.trx" + testRunTitle: "$(OSName) $(TestTargetFramework)" + testResultsFormat: "VSTest" mergeTestResults: true diff --git a/sdk/storage/tests.yml b/sdk/storage/tests.yml index 6497299cb93c7..5316b4942a7da 100644 --- a/sdk/storage/tests.yml +++ b/sdk/storage/tests.yml @@ -1,14 +1,14 @@ trigger: none jobs: -- template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml - parameters: - ServiceDirectory: storage - PreSteps: - - powershell: | - $TestConfigurationPath = "$(Build.ArtifactStagingDirectory)/TestConfiguration.xml" - '$(net-storage-test-configuration)' | Out-File -Encoding Utf8 $TestConfigurationPath - Write-Host "##vso[task.setvariable variable=TestConfigurationPath]$TestConfigurationPath" + - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + parameters: + ServiceDirectory: storage + PreSteps: + - pwsh: | + $TestConfigurationPath = "$(Build.ArtifactStagingDirectory)/TestConfiguration.xml" + '$(net-storage-test-configuration)' | Out-File -Encoding Utf8 $TestConfigurationPath + Write-Host "##vso[task.setvariable variable=TestConfigurationPath]$TestConfigurationPath" - EnvVars: - AZ_STORAGE_CONFIG_PATH: $(TestConfigurationPath) + EnvVars: + AZ_STORAGE_CONFIG_PATH: $(TestConfigurationPath)