From e7746da91537d18f7801c0b8d141970e7fda345c Mon Sep 17 00:00:00 2001 From: azure-sdk Date: Fri, 4 Sep 2020 02:33:14 +0000 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository for Tools PR 926 --- .../templates/steps/publish-blobs.yml | 4 + .../scripts/artifact-metadata-parsing.ps1 | 74 +++++++++++++------ .../scripts/copy-docs-to-blobstorage.ps1 | 47 +++++++++--- eng/common/scripts/update-docs-metadata.ps1 | 6 ++ 4 files changed, 98 insertions(+), 33 deletions(-) diff --git a/eng/common/pipelines/templates/steps/publish-blobs.yml b/eng/common/pipelines/templates/steps/publish-blobs.yml index 682cc4d4f7c88..5888edff87d12 100644 --- a/eng/common/pipelines/templates/steps/publish-blobs.yml +++ b/eng/common/pipelines/templates/steps/publish-blobs.yml @@ -4,6 +4,8 @@ parameters: TargetLanguage: '' BlobName: '' ScriptPath: '' + ArtifactLocation: '' + RepoId: $(Build.Repository.Name) steps: - pwsh: | @@ -21,6 +23,8 @@ steps: -SASKey "${{ parameters.BlobSASKey }}" -Language "${{ parameters.TargetLanguage }}" -BlobName "${{ parameters.BlobName }}" + -PublicArtifactLocation "${{ parameters.ArtifactLocation }}" + -RepoReplaceRegex "(https://github.com/${{ parameters.RepoId }}/(?:blob|tree)/)master" pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Copy Docs to Blob diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index f17e59700b69a..94e95ca170d2b 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -69,6 +69,7 @@ function ParseMavenPackage($pkg, $workingDirectory) { PackageId = $pkgId GroupId = $groupId PackageVersion = $pkgVersion + ReleaseTag = "$($pkgId)_$($pkgVersion)" Deployable = $forceCreate -or !(IsMavenPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion -groupId $groupId.Replace(".", "/")) ReleaseNotes = $releaseNotes ReadmeContent = $readmeContent @@ -150,6 +151,7 @@ function ParseNPMPackage($pkg, $workingDirectory) { $resultObj = New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion + ReleaseTag = "$($pkgId)_$($pkgVersion)" Deployable = $forceCreate -or !(IsNPMPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) ReleaseNotes = $releaseNotes ReadmeContent = $readmeContent @@ -208,6 +210,7 @@ function ParseNugetPackage($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion + ReleaseTag = "$($pkgId)_$($pkgVersion)" Deployable = $forceCreate -or !(IsNugetPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) ReleaseNotes = $releaseNotes ReadmeContent = $readmeContent @@ -272,6 +275,7 @@ function ParsePyPIPackage($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion + ReleaseTag = "$($pkgId)_$($pkgVersion)" Deployable = $forceCreate -or !(IsPythonPackageVersionPublished -pkgId $pkgId -pkgVersion $pkgVersion) ReleaseNotes = $releaseNotes ReadmeContent = $readmeContent @@ -300,6 +304,7 @@ function ParseCArtifact($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = '' PackageVersion = $pkgVersion + ReleaseTag = $pkgVersion # Artifact info is always considered deployable for C becasue it is not # deployed anywhere. Dealing with duplicate tags happens downstream in # CheckArtifactShaAgainstTagsList @@ -331,6 +336,7 @@ function ParseCppArtifact($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = $pkgName PackageVersion = $pkgVersion + ReleaseTag = "$($pkgId)_$($pkgVersion)" # Artifact info is always considered deployable for now becasue it is not # deployed anywhere. Dealing with duplicate tags happens downstream in # CheckArtifactShaAgainstTagsList @@ -387,39 +393,65 @@ function GetExistingTags($apiUrl) { } } -# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases -function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) { - $pkgList = [array]@() - $ParsePkgInfoFn = "" +# Retrieve release tag for artiface package. If multiple packages, then output the first one. +function RetrieveReleaseTag($pkgRepository, $artifactLocation, $continueOnError = $true) { + if (!$artifactLocation) { + return "" + } + try { + $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation + if (!$pkgs -or !$pkgs[0]) { + Write-Host "No packages retrieved from artifact location." + return "" + } + if ($pkgs.Count -gt 1) { + Write-Host "There are more than 1 packages retieved from artifact location." + foreach ($pkg in $pkgs) { + Write-Host "The package name is $($pkg.BaseName)" + } + return "" + } + $parsedPackage = &$parsePkgInfoFn -pkg $pkgs[0] -workingDirectory $artifactLocation + return $parsedPackage.ReleaseTag + } + catch { + if ($continueOnError) { + return "" + } + Write-Error "No release tag retrieved from $artifactLocation" + } +} +function RetrievePackages($pkgRepository, $artifactLocation) { + $parsePkgInfoFn = "" $packagePattern = "" - + $pkgRepository = $pkgRepository.Trim() switch ($pkgRepository) { "Maven" { - $ParsePkgInfoFn = "ParseMavenPackage" + $parsePkgInfoFn = "ParseMavenPackage" $packagePattern = "*.pom" break } "Nuget" { - $ParsePkgInfoFn = "ParseNugetPackage" + $parsePkgInfoFn = "ParseNugetPackage" $packagePattern = "*.nupkg" break } "NPM" { - $ParsePkgInfoFn = "ParseNPMPackage" + $parsePkgInfoFn = "ParseNPMPackage" $packagePattern = "*.tgz" break } "PyPI" { - $ParsePkgInfoFn = "ParsePyPIPackage" + $parsePkgInfoFn = "ParsePyPIPackage" $packagePattern = "*.zip" break } "C" { - $ParsePkgInfoFn = "ParseCArtifact" + $parsePkgInfoFn = "ParseCArtifact" $packagePattern = "*.json" } "CPP" { - $ParsePkgInfoFn = "ParseCppArtifact" + $parsePkgInfoFn = "ParseCppArtifact" $packagePattern = "*.json" } default { @@ -427,12 +459,18 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a exit(1) } } + $pkgs = Get-ChildItem -Path $artifactLocation -Include $packagePattern -Recurse -File + return $pkgs, $parsePkgInfoFn +} - $pkgs = (Get-ChildItem -Path $artifactLocation -Include $packagePattern -Recurse -File) +# Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases +function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) { + $pkgList = [array]@() + $pkgs, $parsePkgInfoFn = RetrievePackages -pkgRepository $pkgRepository -artifactLocation $artifactLocation foreach ($pkg in $pkgs) { try { - $parsedPackage = &$ParsePkgInfoFn -pkg $pkg -workingDirectory $workingDirectory + $parsedPackage = &$parsePkgInfoFn -pkg $pkg -workingDirectory $workingDirectory if ($parsedPackage -eq $null) { continue @@ -444,17 +482,11 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a exit(1) } - $tag = if ($parsedPackage.packageId) { - "$($parsedPackage.packageId)_$($parsedPackage.PackageVersion)" - } else { - $parsedPackage.PackageVersion - } - $pkgList += New-Object PSObject -Property @{ PackageId = $parsedPackage.PackageId PackageVersion = $parsedPackage.PackageVersion GroupId = $parsedPackage.GroupId - Tag = $tag + Tag = $parsedPackage.ReleaseTag ReleaseNotes = $parsedPackage.ReleaseNotes ReadmeContent = $parsedPackage.ReadmeContent IsPrerelease = [AzureEngSemanticVersion]::ParseVersionString($parsedPackage.PackageVersion).IsPrerelease @@ -511,4 +543,4 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap Write-Host "Tags already existing with different SHA versions. Exiting." exit(1) } -} \ No newline at end of file +} diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index a0893099293a0..9b7dea48b1d5d 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -7,8 +7,11 @@ param ( $Language, $BlobName, $ExitOnError=1, - $UploadLatest=1 + $UploadLatest=1, + $PublicArtifactLocation = "", + $RepoReplaceRegex = "(https://github.com/.*/(?:blob|tree)/)master" ) +. (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) $Language = $Language.ToLower() @@ -186,7 +189,8 @@ function Upload-Blobs Param ( [Parameter(Mandatory=$true)] [String]$DocDir, [Parameter(Mandatory=$true)] [String]$PkgName, - [Parameter(Mandatory=$true)] [String]$DocVersion + [Parameter(Mandatory=$true)] [String]$DocVersion, + [Parameter(Mandatory=$false)] [String]$ReleaseTag ) #eg : $BlobName = "https://azuresdkdocs.blob.core.windows.net" $DocDest = "$($BlobName)/`$web/$($Language)" @@ -196,7 +200,23 @@ function Upload-Blobs Write-Host "DocVersion $($DocVersion)" Write-Host "DocDir $($DocDir)" Write-Host "Final Dest $($DocDest)/$($PkgName)/$($DocVersion)" + Write-Host "Release Tag $($ReleaseTag)" + # Use the step to replace master link to release tag link + if ($ReleaseTag) { + foreach ($htmlFile in (Get-ChildItem $DocDir -include *.html -r)) + { + $fileContent = Get-Content -Path $htmlFile + $updatedFileContent = $fileContent -replace $RepoReplaceRegex, "`${1}$ReleaseTag" + if ($updatedFileContent -ne $fileContent) { + Set-Content -Path $htmlFile -Value $updatedFileContent + } + } + } + else { + Write-Warning "Not able to do the master link replacement, since no release tag found for the release. Please manually check." + } + Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..." & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true @@ -213,7 +233,6 @@ function Upload-Blobs } } - if ($Language -eq "javascript") { $PublishedDocs = Get-ChildItem "$($DocLocation)/documentation" | Where-Object -FilterScript {$_.Name.EndsWith(".zip")} @@ -227,7 +246,8 @@ if ($Language -eq "javascript") if($dirList.Length -eq 1){ $DocVersion = $dirList[0].Name Write-Host "Uploading Doc for $($PkgName) Version:- $($DocVersion)..." - Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion + $releaseTag = RetrieveReleaseTag "NPM" $PublicArtifactLocation + Upload-Blobs -DocDir "$($DocLocation)/documentation/$($Item.BaseName)/$($Item.BaseName)/$($DocVersion)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag } else{ Write-Host "found more than 1 folder under the documentation for package - $($Item.Name)" @@ -252,7 +272,8 @@ if ($Language -eq "dotnet") Write-Host "DocDir $($Item)" Write-Host "PkgName $($PkgName)" Write-Host "DocVersion $($DocVersion)" - Upload-Blobs -DocDir "$($Item)" -PkgName $PkgName -DocVersion $DocVersion + $releaseTag = RetrieveReleaseTag "Nuget" $PublicArtifactLocation + Upload-Blobs -DocDir "$($Item)" -PkgName $PkgName -DocVersion $DocVersion -ReleaseTag $releaseTag } else { @@ -279,8 +300,8 @@ if ($Language -eq "python") Write-Host "Discovered Package Name: $PkgName" Write-Host "Discovered Package Version: $Version" Write-Host "Directory for Upload: $UnzippedDocumentationPath" - - Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version + $releaseTag = RetrieveReleaseTag "PyPI" $PublicArtifactLocation + Upload-Blobs -DocDir $UnzippedDocumentationPath -PkgName $PkgName -DocVersion $Version -ReleaseTag $releaseTag } } @@ -326,8 +347,8 @@ if ($Language -eq "java") Write-Host "DocDir $($UnjarredDocumentationPath)" Write-Host "PkgName $($ArtifactId)" Write-Host "DocVersion $($Version)" - - Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version + $releaseTag = RetrieveReleaseTag "Maven" $PublicArtifactLocation + Upload-Blobs -DocDir $UnjarredDocumentationPath -PkgName $ArtifactId -DocVersion $Version -ReleaseTag $releaseTag } Finally { if (![string]::IsNullOrEmpty($UnjarredDocumentationPath)) { @@ -349,11 +370,13 @@ if ($Language -eq "c") # Those loops are left over from previous versions of this script which were # used to publish multiple docs packages in a single invocation. $pkgInfo = Get-Content $DocLocation/package-info.json | ConvertFrom-Json - Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version + $releaseTag = RetrieveReleaseTag "C" $PublicArtifactLocation + Upload-Blobs -DocDir $DocLocation -PkgName 'docs' -DocVersion $pkgInfo.version -ReleaseTag $releaseTag } if ($Language -eq "cpp") { $packageInfo = (Get-Content (Join-Path $DocLocation 'package-info.json') | ConvertFrom-Json) - Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -} \ No newline at end of file + $releaseTag = RetrieveReleaseTag "CPP" $PublicArtifactLocation + Upload-Blobs -DocDir $DocLocation -PkgName $packageInfo.name -DocVersion $packageInfo.version -ReleaseTag $releaseTag +} diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index 42ca30894fd6d..7cf65fa7d2a4d 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -17,6 +17,8 @@ param ( . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) . (Join-Path $PSScriptRoot SemVer.ps1) +$releaseReplaceRegex = "(https://github.com/$RepoId/(?:blob|tree)/)master" + function GetMetaData($lang){ switch ($lang) { "java" { @@ -75,6 +77,10 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ if ($headerContentMatches) { $foundTitle = $headerContentMatches.Matches[0] $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n" + + # Replace github master link with release tag. + $ReplacementPattern = "`${1}$($pkgInfo.Tag)" + $fileContent = $fileContent -replace $releaseReplaceRegex, $ReplacementPattern } $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"