Sync eng/common directory with azure-sdk-tools repository

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -9,11 +9,14 @@ parameters:
   TargetDocRepoName: ''
   TargetDocRepoOwner: ''
   PRBranchName: 'smoke-test-rdme'
+  SourceBranchName: 'smoke-test'
   ArtifactName: ''
   Language: ''
   DocRepoDestinationPath: '' #usually docs-ref-services/
+  CIConfigs: '[]'
   GHReviewersVariable: '' 
   GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step
+  OnboardingBranch: ''
 
 steps:
 - pwsh: |
@@ -22,8 +25,8 @@ steps:
     try {
       Push-Location ${{ parameters.WorkingDirectory }}/repo
 
-      Write-Host "git checkout smoke-test"
-      git checkout smoke-test
+      Write-Host "git checkout ${{ parameters.SourceBranchName }}"
+      git checkout ${{ parameters.SourceBranchName }}
     } finally {
       Pop-Location
     }
@@ -48,15 +51,73 @@ steps:
   env:
     GH_TOKEN: $(azuresdk-github-pat)
 
+- task: PowerShell@2
+  displayName: 'Update Docs.MS CI Targeted Packages'
+  condition: and(succeededOrFailed(), eq('${{ parameters.OnboardingBranch }}',''))
+  inputs:
+    targetType: filePath
+    filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1
+    arguments: >
+      -ArtifactLocation ${{ parameters.ArtifactLocation }}
+      -WorkDirectory "${{ parameters.WorkingDirectory }}"
+      -RepoId ${{ parameters.RepoId }}
+      -Repository ${{ parameters.PackageRepository }}
+      -ReleaseSHA ${{ parameters.ReleaseSha }}
+      -CIRepository "${{ parameters.WorkingDirectory }}/repo"
+      -Configs "${{ parameters.CIConfigs }}"
+    pwsh: true
+  env:
+    GH_TOKEN: $(azuresdk-github-pat)
+
 - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
   parameters:
     RepoName: ${{ parameters.TargetDocRepoName }}
     RepoOwner: ${{ parameters.TargetDocRepoOwner }}
     PRBranchName: ${{ parameters.PRBranchName }}
-    CommitMsg: "Update readme content for ${{ parameters.ArtifactName }}"
-    PRTitle: "Docs.MS Readme Update."
-    BaseBranchName: smoke-test
+    CommitMsg: "Update docs metadata and targeting for release of ${{ parameters.ArtifactName }}"
+    PRTitle: "Docs.MS Release Updates for ${{ parameters.ArtifactName }}"
+    BaseBranchName: ${{ parameters.SourceBranchName }}
     WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
     ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
     GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
     GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} 
+
+- ${{if ne( parameters['OnboardingBranch'], '')}}:
+  - pwsh: |
+      Push-Location ${{ parameters.WorkingDirectory }}/repo
+
+      git reset --hard HEAD
+      git remote rm azure-sdk-fork
+      git checkout ${{ parameters.OnboardingBranch}}
+    displayName: Reset Docs Repo, Checkout Onboarding Branch
+    ignoreLASTEXITCODE: false
+
+  - task: PowerShell@2
+    displayName: 'Update Docs.MS CI Targeted Packages'
+    inputs:
+      targetType: filePath
+      filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1
+      arguments: >
+        -ArtifactLocation ${{ parameters.ArtifactLocation }}
+        -WorkDirectory "${{ parameters.WorkingDirectory }}"
+        -RepoId ${{ parameters.RepoId }}
+        -Repository ${{ parameters.PackageRepository }}
+        -ReleaseSHA ${{ parameters.ReleaseSha }}
+        -CIRepository "${{ parameters.WorkingDirectory }}/repo"
+        -Configs "${{ parameters.CIConfigs }}"
+      pwsh: true
+    env:
+      GH_TOKEN: $(azuresdk-github-pat)
+
+  - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+    parameters:
+      RepoName: ${{ parameters.TargetDocRepoName }}
+      RepoOwner: ${{ parameters.TargetDocRepoOwner }}
+      PRBranchName: ${{ parameters.PRBranchName }}-ci
+      CommitMsg: "CI Update for release of ${{ parameters.ArtifactName }}"
+      PRTitle: "Docs.MS CI Updates for ${{ parameters.ArtifactName }}"
+      BaseBranchName: ${{ parameters.OnboardingBranch }}
+      WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo
+      ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }}
+      GHReviewersVariable: ${{ parameters.GHReviewersVariable }}
+      GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} 

eng/common/pipelines/templates/steps/get-pr-owners.yml

@@ -39,7 +39,7 @@ steps:
 
   - pwsh: |
       $originalValue = "$(${{ parameters.TargetVariable }})"
-      $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory)
+      $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory)
       if ($result) {
         Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result"
       }

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -36,47 +36,7 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) {
       "Authorization" = "token $($env:GH_TOKEN)"
     }
 
-    Invoke-WebRequest-WithHandling -url $url -body $body -headers $headers -method "Post"
-  }
-}
-
-function Invoke-WebRequest-WithHandling($url, $method, $body = $null, $headers = $null) {
-  $attempts = 1
-
-  while ($attempts -le 3) {
-    try {
-      return Invoke-RestMethod -Method $method -Uri $url -Body $body -Headers $headers
-    }
-    catch {
-      $response = $_.Exception.Response
-
-      $statusCode = $response.StatusCode.value__
-      $statusDescription = $response.StatusDescription
-
-      if ($statusCode) {
-        Write-Host "API request attempt number $attempts to $url failed with statuscode $statusCode"
-        Write-Host $statusDescription
-
-        Write-Host "Rate Limit Details:"
-        Write-Host "Total: $($response.Headers.GetValues("X-RateLimit-Limit"))"
-        Write-Host "Remaining: $($response.Headers.GetValues("X-RateLimit-Remaining"))"
-        Write-Host "Reset Epoch: $($response.Headers.GetValues("X-RateLimit-Reset"))"
-      }
-      else {
-        Write-Host "API request attempt number $attempts to $url failed with no statuscode present, exception follows:"
-        Write-Host $_.Exception.Response
-        Write-Host $_.Exception
-      }
-
-      if ($attempts -ge 3) {
-        Write-Host "Abandoning Request $url after 3 attempts."
-        exit(1)
-      }
-
-      Start-Sleep -s 10
-    }
-
-    $attempts += 1
+    Invoke-RestMethod -Uri $url -Body $body -Headers $headers -Method "Post" -MaximumRetryCount 3 -RetryIntervalSec 10
   }
 }
 
@@ -107,6 +67,7 @@ function ParseMavenPackage($pkg, $workingDirectory) {
 
   return New-Object PSObject -Property @{
     PackageId      = $pkgId
+    GroupId        = $groupId
     PackageVersion = $pkgVersion
     Deployable     = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/"))
     ReleaseNotes   = $releaseNotes
@@ -119,7 +80,7 @@ function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) {
   try {
 
     $uri = "https://oss.sonatype.org/content/repositories/releases/$groupId/$pkgId/$pkgVersion/$pkgId-$pkgVersion.pom"
-    $pomContent = Invoke-RestMethod -MaximumRetryCount 3 -Method "GET" -uri $uri
+    $pomContent = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "GET" -uri $uri
 
     if ($pomContent -ne $null -or $pomContent.Length -eq 0) {
       return $true
@@ -259,7 +220,7 @@ function IsNugetPackageVersionPublished($pkgId, $pkgVersion) {
   $nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json"
 
   try {
-    $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -uri $nugetUri -Method "GET"
+    $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET"
 
     return $nugetVersions.versions.Contains($pkgVersion)
   }
@@ -382,7 +343,7 @@ function ParseCppArtifact($pkg, $workingDirectory) {
 # Returns the pypi publish status of a package id and version.
 function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
   try {
-    $existingVersion = (Invoke-RestMethod -MaximumRetryCount 3 -Method "Get" -uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version
+    $existingVersion = (Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "Get" -uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version
 
     # if existingVersion exists, then it's already been published
     return $True
@@ -406,7 +367,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) {
 # Retrieves the list of all tags that exist on the target repository
 function GetExistingTags($apiUrl) {
   try {
-    return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags"  -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
+    return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") }
   }
   catch {
     Write-Host $_
@@ -492,9 +453,11 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
       $pkgList += New-Object PSObject -Property @{
         PackageId      = $parsedPackage.PackageId
         PackageVersion = $parsedPackage.PackageVersion
+        GroupId        = $parsedPackage.GroupId
         Tag            = $tag
         ReleaseNotes   = $parsedPackage.ReleaseNotes
         ReadmeContent  = $parsedPackage.ReadmeContent
+        IsPrerelease   = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease
       }
     }
     catch {
@@ -531,7 +494,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap
   $unmatchedTags = @()
 
   foreach ($tag in $priorExistingTagList) {
-    $tagSha = (Invoke-WebRequest-WithHandling -Method "Get" -Url "$apiUrl/git/refs/tags/$tag" -Headers $headers)."object".sha
+    $tagSha = (Invoke-RestMethod -Method "Get" -Uri "$apiUrl/git/refs/tags/$tag" -Headers $headers -MaximumRetryCount 3 -RetryIntervalSec 10)."object".sha
 
     if ($tagSha -eq $releaseSha) {
       Write-Host "This package has already been released. The existing tag commit SHA $releaseSha matches the artifact SHA being processed. Skipping release step for this tag."