diff --git a/eng/common/pipelines/templates/steps/docs-metadata-release.yml b/eng/common/pipelines/templates/steps/docs-metadata-release.yml index a9ff8c4e17b4..d75fed40ae40 100644 --- a/eng/common/pipelines/templates/steps/docs-metadata-release.yml +++ b/eng/common/pipelines/templates/steps/docs-metadata-release.yml @@ -9,11 +9,14 @@ parameters: TargetDocRepoName: '' TargetDocRepoOwner: '' PRBranchName: 'smoke-test-rdme' + SourceBranchName: 'smoke-test' ArtifactName: '' Language: '' DocRepoDestinationPath: '' #usually docs-ref-services/ + CIConfigs: '[]' GHReviewersVariable: '' GHTeamReviewersVariable: '' # externally set, as eng-common does not have the identity-resolver. Run as pre-step + OnboardingBranch: '' steps: - pwsh: | @@ -22,8 +25,8 @@ steps: try { Push-Location ${{ parameters.WorkingDirectory }}/repo - Write-Host "git checkout smoke-test" - git checkout smoke-test + Write-Host "git checkout ${{ parameters.SourceBranchName }}" + git checkout ${{ parameters.SourceBranchName }} } finally { Pop-Location } @@ -48,15 +51,73 @@ steps: env: GH_TOKEN: $(azuresdk-github-pat) +- task: PowerShell@2 + displayName: 'Update Docs.MS CI Targeted Packages' + condition: and(succeededOrFailed(), eq('${{ parameters.OnboardingBranch }}','')) + inputs: + targetType: filePath + filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1 + arguments: > + -ArtifactLocation ${{ parameters.ArtifactLocation }} + -WorkDirectory "${{ parameters.WorkingDirectory }}" + -RepoId ${{ parameters.RepoId }} + -Repository ${{ parameters.PackageRepository }} + -ReleaseSHA ${{ parameters.ReleaseSha }} + -CIRepository "${{ parameters.WorkingDirectory }}/repo" + -Configs "${{ parameters.CIConfigs }}" + pwsh: true + env: + GH_TOKEN: $(azuresdk-github-pat) + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: RepoName: ${{ parameters.TargetDocRepoName }} RepoOwner: ${{ parameters.TargetDocRepoOwner }} PRBranchName: ${{ parameters.PRBranchName }} - CommitMsg: "Update readme content for ${{ parameters.ArtifactName }}" - PRTitle: "Docs.MS Readme Update." - BaseBranchName: smoke-test + CommitMsg: "Update docs metadata and targeting for release of ${{ parameters.ArtifactName }}" + PRTitle: "Docs.MS Release Updates for ${{ parameters.ArtifactName }}" + BaseBranchName: ${{ parameters.SourceBranchName }} WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} GHReviewersVariable: ${{ parameters.GHReviewersVariable }} GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} + +- ${{if ne( parameters['OnboardingBranch'], '')}}: + - pwsh: | + Push-Location ${{ parameters.WorkingDirectory }}/repo + + git reset --hard HEAD + git remote rm azure-sdk-fork + git checkout ${{ parameters.OnboardingBranch}} + displayName: Reset Docs Repo, Checkout Onboarding Branch + ignoreLASTEXITCODE: false + + - task: PowerShell@2 + displayName: 'Update Docs.MS CI Targeted Packages' + inputs: + targetType: filePath + filePath: ${{ parameters.ScriptDirectory }}/update-docs-ci.ps1 + arguments: > + -ArtifactLocation ${{ parameters.ArtifactLocation }} + -WorkDirectory "${{ parameters.WorkingDirectory }}" + -RepoId ${{ parameters.RepoId }} + -Repository ${{ parameters.PackageRepository }} + -ReleaseSHA ${{ parameters.ReleaseSha }} + -CIRepository "${{ parameters.WorkingDirectory }}/repo" + -Configs "${{ parameters.CIConfigs }}" + pwsh: true + env: + GH_TOKEN: $(azuresdk-github-pat) + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: ${{ parameters.TargetDocRepoName }} + RepoOwner: ${{ parameters.TargetDocRepoOwner }} + PRBranchName: ${{ parameters.PRBranchName }}-ci + CommitMsg: "CI Update for release of ${{ parameters.ArtifactName }}" + PRTitle: "Docs.MS CI Updates for ${{ parameters.ArtifactName }}" + BaseBranchName: ${{ parameters.OnboardingBranch }} + WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo + ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} + GHReviewersVariable: ${{ parameters.GHReviewersVariable }} + GHTeamReviewersVariable: ${{ parameters.GHTeamReviewersVariable }} \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/get-pr-owners.yml b/eng/common/pipelines/templates/steps/get-pr-owners.yml index a80d5b83b2de..2abe10b81a52 100644 --- a/eng/common/pipelines/templates/steps/get-pr-owners.yml +++ b/eng/common/pipelines/templates/steps/get-pr-owners.yml @@ -39,7 +39,7 @@ steps: - pwsh: | $originalValue = "$(${{ parameters.TargetVariable }})" - $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory /sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory) + $result = $(Build.SourcesDirectory)/eng/common/scripts/get-codeowners.ps1 -TargetDirectory sdk/${{ parameters.ServiceDirectory }}/ -RootDirectory $(Build.SourcesDirectory) if ($result) { Write-Output "##vso[task.setvariable variable=${{ parameters.TargetVariable }}]$originalValue,$result" } diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index af7c9401e644..93bd6ac5f2db 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -36,47 +36,7 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { "Authorization" = "token $($env:GH_TOKEN)" } - Invoke-WebRequest-WithHandling -url $url -body $body -headers $headers -method "Post" - } -} - -function Invoke-WebRequest-WithHandling($url, $method, $body = $null, $headers = $null) { - $attempts = 1 - - while ($attempts -le 3) { - try { - return Invoke-RestMethod -Method $method -Uri $url -Body $body -Headers $headers - } - catch { - $response = $_.Exception.Response - - $statusCode = $response.StatusCode.value__ - $statusDescription = $response.StatusDescription - - if ($statusCode) { - Write-Host "API request attempt number $attempts to $url failed with statuscode $statusCode" - Write-Host $statusDescription - - Write-Host "Rate Limit Details:" - Write-Host "Total: $($response.Headers.GetValues("X-RateLimit-Limit"))" - Write-Host "Remaining: $($response.Headers.GetValues("X-RateLimit-Remaining"))" - Write-Host "Reset Epoch: $($response.Headers.GetValues("X-RateLimit-Reset"))" - } - else { - Write-Host "API request attempt number $attempts to $url failed with no statuscode present, exception follows:" - Write-Host $_.Exception.Response - Write-Host $_.Exception - } - - if ($attempts -ge 3) { - Write-Host "Abandoning Request $url after 3 attempts." - exit(1) - } - - Start-Sleep -s 10 - } - - $attempts += 1 + Invoke-RestMethod -Uri $url -Body $body -Headers $headers -Method "Post" -MaximumRetryCount 3 -RetryIntervalSec 10 } } @@ -107,6 +67,7 @@ function ParseMavenPackage($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = $pkgId + GroupId = $groupId PackageVersion = $pkgVersion Deployable = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/")) ReleaseNotes = $releaseNotes @@ -119,7 +80,7 @@ function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) { try { $uri = "https://oss.sonatype.org/content/repositories/releases/$groupId/$pkgId/$pkgVersion/$pkgId-$pkgVersion.pom" - $pomContent = Invoke-RestMethod -MaximumRetryCount 3 -Method "GET" -uri $uri + $pomContent = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "GET" -uri $uri if ($pomContent -ne $null -or $pomContent.Length -eq 0) { return $true @@ -259,7 +220,7 @@ function IsNugetPackageVersionPublished($pkgId, $pkgVersion) { $nugetUri = "https://api.nuget.org/v3-flatcontainer/$($pkgId.ToLowerInvariant())/index.json" try { - $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -uri $nugetUri -Method "GET" + $nugetVersions = Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -uri $nugetUri -Method "GET" return $nugetVersions.versions.Contains($pkgVersion) } @@ -382,7 +343,7 @@ function ParseCppArtifact($pkg, $workingDirectory) { # Returns the pypi publish status of a package id and version. function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { try { - $existingVersion = (Invoke-RestMethod -MaximumRetryCount 3 -Method "Get" -uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version + $existingVersion = (Invoke-RestMethod -MaximumRetryCount 3 -RetryIntervalSec 10 -Method "Get" -uri "https://pypi.org/pypi/$pkgId/$pkgVersion/json").info.version # if existingVersion exists, then it's already been published return $True @@ -406,7 +367,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { # Retrieves the list of all tags that exist on the target repository function GetExistingTags($apiUrl) { try { - return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } + return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } } catch { Write-Host $_ @@ -492,9 +453,11 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a $pkgList += New-Object PSObject -Property @{ PackageId = $parsedPackage.PackageId PackageVersion = $parsedPackage.PackageVersion + GroupId = $parsedPackage.GroupId Tag = $tag ReleaseNotes = $parsedPackage.ReleaseNotes ReadmeContent = $parsedPackage.ReadmeContent + IsPrerelease = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease } } catch { @@ -531,7 +494,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap $unmatchedTags = @() foreach ($tag in $priorExistingTagList) { - $tagSha = (Invoke-WebRequest-WithHandling -Method "Get" -Url "$apiUrl/git/refs/tags/$tag" -Headers $headers)."object".sha + $tagSha = (Invoke-RestMethod -Method "Get" -Uri "$apiUrl/git/refs/tags/$tag" -Headers $headers -MaximumRetryCount 3 -RetryIntervalSec 10)."object".sha if ($tagSha -eq $releaseSha) { Write-Host "This package has already been released. The existing tag commit SHA $releaseSha matches the artifact SHA being processed. Skipping release step for this tag." diff --git a/eng/common/scripts/update-docs-ci.ps1 b/eng/common/scripts/update-docs-ci.ps1 new file mode 100644 index 000000000000..98f993a17f17 --- /dev/null +++ b/eng/common/scripts/update-docs-ci.ps1 @@ -0,0 +1,275 @@ +#Requires -Version 6.0 +# This script is intended to update docs.ms CI configuration (currently supports Java, Python, C#, JS) +# as part of the azure-sdk release. For details on calling, check `archtype--release` in each azure-sdk +# repository. + +# Where possible, this script adds as few changes as possible to the target config. We only +# specifically mark a version for Python Preview and Java. This script is intended to be invoked +# multiple times. Once for each moniker. Currently only supports "latest" and "preview" artifact selection however. +param ( + [Parameter(Mandatory = $true)] + $ArtifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory) + + [Parameter(Mandatory = $true)] + $WorkDirectory, # a clean folder that we can work in + + [Parameter(Mandatory = $true)] + $ReleaseSHA, # the SHA for the artifacts. DevOps: $(Release.Artifacts..SourceVersion) or $(Build.SourceVersion) + + [Parameter(Mandatory = $true)] + $RepoId, # full repo id. EG azure/azure-sdk-for-net DevOps: $(Build.Repository.Id). Used as a part of VerifyPackages + + [Parameter(Mandatory = $true)] + [ValidateSet("Nuget","NPM","PyPI","Maven")] + $Repository, # EG: "Maven", "PyPI", "NPM" + + [Parameter(Mandatory = $true)] + $CIRepository, + + [Parameter(Mandatory = $true)] + $Configs +) + +# import artifact parsing and semver handling +. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) +. (Join-Path $PSScriptRoot SemVer.ps1) + +# Updates a python CI configuration json. +# For "latest", the version attribute is cleared, as default behavior is to pull latest "non-preview". +# For "preview", we update to >= the target releasing package version. +function UpdateParamsJsonPython($pkgs, $ciRepo, $locationInDocRepo){ + $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $pkgJsonLoc)) { + Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." + exit(1) + } + + $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json + $visibleInCI = @{} + + for ($i=0; $i -lt $allJson.packages.Length; $i++) { + $pkgDef = $allJson.packages[$i] + + if ($pkgDef.package_info.name) { + $visibleInCI[$pkgDef.package_info.name] = $i + } + } + + foreach ($releasingPkg in $pkgs) { + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $existingPackageDef = $targetData[$packagesIndex] + + if ($releasingPkg.IsPrerelease) { + if (-not $existingPackageDef.package_info.version) { + $existingPackageDef.package_info | Add-Member -NotePropertyName version -NotePropertyValue "" + } + + $existingPackageDef.package_info.version = ">=$($releasingPkg.PackageVersion)" + } + else { + if ($def.version) { + $def.PSObject.Properties.Remove('version') + } + } + } + else { + $newItem = New-Object PSObject -Property @{ + package_info = New-Object PSObject -Property @{ + prefer_source_distribution = "true" + install_type = "pypi" + name=$releasingPkg.PackageId + } + excludePath = @("test*","example*","sample*","doc*") + } + $allJson.packages += $newItem + } + } + + $jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } + + Set-Content -Path $pkgJsonLoc -Value $jsonContent +} + +# Updates a js CI configuration json. +# For "latest", we simply set a target package name +# For "preview", we add @next to the target package name +function UpdateParamsJsonJS($pkgs, $ciRepo, $locationInDocRepo){ + $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $pkgJsonLoc)) { + Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." + exit(1) + } + + $allJson = Get-Content $pkgJsonLoc | ConvertFrom-Json + + $visibleInCI = @{} + + for ($i=0; $i -lt $allJson.npm_package_sources.Length; $i++) { + $pkgDef = $allJson.npm_package_sources[$i] + $accessor = ($pkgDef.name).Replace("`@next", "") + $visibleInCI[$accessor] = $i + } + + foreach ($releasingPkg in $pkgs) { + $name = $releasingPkg.PackageId + + if ($releasingPkg.IsPrerelease) { + $name += "`@next" + } + + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $existingPackageDef = $allJson.npm_package_sources[$packagesIndex] + $existingPackageDef.name = $name + } + else { + $newItem = New-Object PSObject -Property @{ + name = $name + } + + if ($newItem) { $allJson.npm_package_sources += $newItem } + } + } + + $jsonContent = $allJson | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } + + Set-Content -Path $pkgJsonLoc -Value $jsonContent +} + +# details on CSV schema can be found here +# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job +function UpdateCSVBasedCI($pkgs, $ciRepo, $locationInDocRepo){ + $csvLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $csvLoc)) { + Write-Error "Unable to locate package csv at location $csvLoc, exiting." + exit(1) + } + + $allCSVRows = Get-Content $csvLoc + $visibleInCI = @{} + + # first pull what's already available + for ($i=0; $i -lt $allCSVRows.Length; $i++) { + $pkgDef = $allCSVRows[$i] + + # get rid of the modifiers to get just the package id + $id = $pkgDef.split(",")[1] -replace "\[.*?\]", "" + + $visibleInCI[$id] = $i + } + + foreach ($releasingPkg in $pkgs) { + $installModifiers = "tfm=netstandard2.0" + if ($releasingPkg.IsPrerelease) { + $installModifiers += ";isPrerelease=true" + } + $lineId = $releasingPkg.PackageId.Replace(".","").ToLower() + + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $allCSVRows[$packagesIndex] = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" + } + else { + $newItem = "$($lineId),[$installModifiers]$($releasingPkg.PackageId)" + $allCSVRows += ($newItem) + } + } + + Set-Content -Path $csvLoc -Value $allCSVRows +} + +# a "package.json configures target packages for all the monikers in a Repository, it also has a slightly different +# schema than the moniker-specific json config that is seen in python and js +function UpdatePackageJson($pkgs, $ciRepo, $locationInDocRepo, $monikerId){ + $pkgJsonLoc = (Join-Path -Path $ciRepo -ChildPath $locationInDocRepo) + + if (-not (Test-Path $pkgJsonLoc)) { + Write-Error "Unable to locate package json at location $pkgJsonLoc, exiting." + exit(1) + } + + $allJsonData = Get-Content $pkgJsonLoc | ConvertFrom-Json + + $visibleInCI = @{} + + for ($i=0; $i -lt $allJsonData[$monikerId].packages.Length; $i++) { + $pkgDef = $allJsonData[$monikerId].packages[$i] + $visibleInCI[$pkgDef.packageArtifactId] = $i + } + + foreach ($releasingPkg in $pkgs) { + if ($visibleInCI.ContainsKey($releasingPkg.PackageId)) { + $packagesIndex = $visibleInCI[$releasingPkg.PackageId] + $existingPackageDef = $allJsonData[$monikerId].packages[$packagesIndex] + $existingPackageDef.packageVersion = $releasingPkg.PackageVersion + } + else { + $newItem = New-Object PSObject -Property @{ + packageDownloadUrl = "https://repo1.maven.org/maven2" + packageGroupId = $releasingPkg.GroupId + packageArtifactId = $releasingPkg.PackageId + packageVersion = $releasingPkg.PackageVersion + inputPath = @() + excludePath = @() + } + + $allJsonData[$monikerId].packages += $newItem + } + } + + $jsonContent = $allJsonData | ConvertTo-Json -Depth 10 | % {$_ -replace "(?m) (?<=^(?: )*)", " " } + + Set-Content -Path $pkgJsonLoc -Value $jsonContent +} + +$targets = ($Configs | ConvertFrom-Json).targets + +#{ +# path_to_config: +# mode: +# monikerid +#} + +$apiUrl = "https://api.github.com/repos/$repoId" +$pkgs = VerifyPackages -pkgRepository $Repository ` + -artifactLocation $ArtifactLocation ` + -workingDirectory $WorkDirectory ` + -apiUrl $apiUrl ` + -continueOnError $True + +foreach ($config in $targets) { + if ($config.mode -eq "Preview") { $includePreview = $true } else { $includePreview = $false } + $pkgsFiltered = $pkgs | ? { $_.IsPrerelease -eq $includePreview} + + if ($pkgs) { + Write-Host "Given the visible artifacts, CI updates against $($config.path_to_config) will be processed for the following packages." + Write-Host ($pkgsFiltered | % { $_.PackageId + " " + $_.PackageVersion }) + + switch ($Repository) { + "Nuget" { + UpdateCSVBasedCI -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config + break + } + "NPM" { + UpdateParamsJsonJS -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config + break + } + "PyPI" { + UpdateParamsJsonPython -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config + break + } + "Maven" { + UpdatePackageJson -pkgs $pkgsFiltered -ciRepo $CIRepository -locationInDocRepo $config.path_to_config -monikerId $config.monikerid + break + } + default { + Write-Host "Unrecognized target: $Repository" + exit(1) + } + } + } +} \ No newline at end of file diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index a858078f5448..42ca30894fd6 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -14,8 +14,6 @@ param ( $DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go? ) - -# import artifact parsing and semver handling . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) . (Join-Path $PSScriptRoot SemVer.ps1) @@ -43,7 +41,7 @@ function GetMetaData($lang){ } } - $metadataResponse = Invoke-WebRequest-WithHandling -url $metadataUri -method "GET" | ConvertFrom-Csv + $metadataResponse = Invoke-RestMethod -Uri $metadataUri -method "GET" -MaximumRetryCount 3 -RetryIntervalSec 10 | ConvertFrom-Csv return $metadataResponse }