diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md index 39efc1446258..c9a462aae3a8 100644 --- a/eng/common/TestResources/New-TestResources.ps1.md +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -412,6 +412,23 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -OutFile +save test environment settings into a test-resources.json.env file next to test-resources.json. +The file is protected via DPAPI. The environment file would be scoped to the current repository directory. +Note: Supported only on Windows. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### CommonParameters This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md index 89e61a4cb07d..3c0cd7c2207e 100644 --- a/eng/common/TestResources/README.md +++ b/eng/common/TestResources/README.md @@ -35,6 +35,12 @@ eng\common\TestResources\New-TestResources.ps1 ` -TestApplicationSecret (ConvertFrom-SecureString $sp.Secret -AsPlainText) ``` +If you are running this for a .NET project on Windows, the recommended method is to +add the `-OutFile` switch to the above command. This will save test environment settings +into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI. +The environment file would be scoped to the current repository directory and avoids the need to +set environment variables or restart your IDE to recognize them. + Along with some log messages, this will output environment variables based on your current shell like in the following example: diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml index 46ff5a8f62f8..790de92bcbe5 100644 --- a/eng/common/pipelines/templates/steps/create-pull-request.yml +++ b/eng/common/pipelines/templates/steps/create-pull-request.yml @@ -35,27 +35,31 @@ steps: workingDirectory: ${{ parameters.WorkingDirectory }} ignoreLASTEXITCODE: true -- pwsh: | - ${{ parameters.ScriptDirectory }}/git-branch-push.ps1 ` - -PRBranchName "${{ parameters.PRBranchName }}" ` - -CommitMsg "${{ parameters.CommitMsg }}" ` - -GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/${{ parameters.RepoName }}.git" ` - -PushArgs "${{ parameters.PushArgs }}" - +- task: PowerShell@2 displayName: Push changes - workingDirectory: ${{ parameters.WorkingDirectory }} condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: ${{ parameters.WorkingDirectory }} + filePath: ${{ parameters.ScriptDirectory }}/git-branch-push.ps1 + arguments: > + -PRBranchName "${{ parameters.PRBranchName }}" + -CommitMsg "${{ parameters.CommitMsg }}" + -GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/${{ parameters.RepoName }}.git" + -PushArgs "${{ parameters.PushArgs }}" -- pwsh: | - ${{ parameters.ScriptDirectory }}/Submit-PullRequest.ps1 ` - -RepoOwner "${{ parameters.RepoOwner }}" ` - -RepoName "${{ parameters.RepoName }}" ` - -BaseBranch "${{ parameters.BaseBranchName }}" ` - -PROwner "${{ parameters.PROwner }}" ` - -PRBranch "${{ parameters.PRBranchName }}" ` - -AuthToken "$(azuresdk-github-pat)" ` - -PRTitle "${{ parameters.PRTitle }}" - +- task: PowerShell@2 displayName: Create pull request - workingDirectory: ${{ parameters.WorkingDirectory }} - condition: and(succeeded(), eq(variables['HasChanges'], 'true')) \ No newline at end of file + condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: ${{ parameters.WorkingDirectory }} + filePath: ${{ parameters.ScriptDirectory }}/Submit-PullRequest.ps1 + arguments: > + -RepoOwner "${{ parameters.RepoOwner }}" + -RepoName "${{ parameters.RepoName }}" + -BaseBranch "${{ parameters.BaseBranchName }}" + -PROwner "${{ parameters.PROwner }}" + -PRBranch "${{ parameters.PRBranchName }}" + -AuthToken "$(azuresdk-github-pat)" + -PRTitle "${{ parameters.PRTitle }}" diff --git a/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml b/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml index a1fa15069dbe..8fb57243b0fd 100644 --- a/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml +++ b/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml @@ -10,9 +10,13 @@ steps: - task: PowerShell@2 displayName: 'Verify Package Tags and Create Git Releases' inputs: - targetType: filePath filePath: ${{ parameters.ScriptDirectory }}/create-tags-and-git-release.ps1 - arguments: -artifactLocation ${{parameters.ArtifactLocation}} -packageRepository ${{parameters.PackageRepository}} -releaseSha ${{parameters.ReleaseSha}} -repoId ${{parameters.RepoId}} -workingDirectory '${{parameters.WorkingDirectory}}' + arguments: > + -artifactLocation ${{ parameters.ArtifactLocation }} + -packageRepository ${{ parameters.PackageRepository }} + -releaseSha ${{ parameters.ReleaseSha }} + -repoId ${{ parameters.RepoId }} + -workingDirectory '${{ parameters.WorkingDirectory }}' pwsh: true timeoutInMinutes: 5 env: diff --git a/eng/common/pipelines/templates/steps/docs-metadata-release.yml b/eng/common/pipelines/templates/steps/docs-metadata-release.yml index 89d12d4ac1d1..6ff9f84ff3ee 100644 --- a/eng/common/pipelines/templates/steps/docs-metadata-release.yml +++ b/eng/common/pipelines/templates/steps/docs-metadata-release.yml @@ -16,10 +16,10 @@ parameters: steps: - pwsh: | git clone https://github.com/${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }} ${{ parameters.WorkingDirectory }}/repo - + try { Push-Location ${{ parameters.WorkingDirectory }}/repo - + Write-Host "git checkout smoke-test" git checkout smoke-test } finally { @@ -33,14 +33,14 @@ steps: inputs: targetType: filePath filePath: ${{ parameters.ScriptDirectory }}/update-docs-metadata.ps1 - arguments: > - -ArtifactLocation ${{parameters.ArtifactLocation}} - -Repository ${{parameters.PackageRepository}} - -ReleaseSHA ${{parameters.ReleaseSha}} - -RepoId ${{parameters.RepoId}} - -WorkDirectory '${{parameters.WorkingDirectory}}' - -DocRepoLocation "${{parameters.WorkingDirectory}}/repo" - -Language "${{parameters.Language}}" + arguments: > + -ArtifactLocation ${{ parameters.ArtifactLocation }} + -Repository ${{ parameters.PackageRepository }} + -ReleaseSHA ${{ parameters.ReleaseSha }} + -RepoId ${{ parameters.RepoId }} + -WorkDirectory "${{ parameters.WorkingDirectory }}" + -DocRepoLocation "${{ parameters.WorkingDirectory }}/repo" + -Language "${{parameters.Language}}" -DocRepoContentLocation ${{ parameters.DocRepoDestinationPath }} pwsh: true env: @@ -54,5 +54,5 @@ steps: CommitMsg: "Update readme content for ${{ parameters.ArtifactName }}" PRTitle: "Docs.MS Readme Update." BaseBranchName: smoke-test - WorkingDirectory: ${{parameters.WorkingDirectory}}/repo - ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}} + WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo + ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} diff --git a/eng/common/pipelines/templates/steps/publish-blobs.yml b/eng/common/pipelines/templates/steps/publish-blobs.yml index ce0d9f708cca..682cc4d4f7c8 100644 --- a/eng/common/pipelines/templates/steps/publish-blobs.yml +++ b/eng/common/pipelines/templates/steps/publish-blobs.yml @@ -7,16 +7,20 @@ parameters: steps: - pwsh: | - Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" ` - -OutFile "azcopy.zip" | Wait-Process; Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" -OutFile "azcopy.zip" | Wait-Process; + Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" workingDirectory: $(Build.BinariesDirectory) displayName: Download and Extract azcopy Zip - task: Powershell@2 inputs: - targetType: 'filePath' filePath: ${{ parameters.ScriptPath }} - arguments: -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0] -DocLocation "${{ parameters.FolderForUpload }}" -SASKey "${{ parameters.BlobSASKey }}" -Language "${{ parameters.TargetLanguage }}" -BlobName "${{ parameters.BlobName }}" + arguments: > + -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0] + -DocLocation "${{ parameters.FolderForUpload }}" + -SASKey "${{ parameters.BlobSASKey }}" + -Language "${{ parameters.TargetLanguage }}" + -BlobName "${{ parameters.BlobName }}" pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Copy Docs to Blob diff --git a/eng/common/pipelines/templates/steps/verify-agent-os.yml b/eng/common/pipelines/templates/steps/verify-agent-os.yml index 1786612ba56c..b221583bc6da 100644 --- a/eng/common/pipelines/templates/steps/verify-agent-os.yml +++ b/eng/common/pipelines/templates/steps/verify-agent-os.yml @@ -1,4 +1,3 @@ -# Template for all Python Scripts in this repository parameters: OSVmImage: $(OSVmImage) diff --git a/eng/common/pipelines/templates/steps/verify-changelog.yml b/eng/common/pipelines/templates/steps/verify-changelog.yml new file mode 100644 index 000000000000..f90238471d95 --- /dev/null +++ b/eng/common/pipelines/templates/steps/verify-changelog.yml @@ -0,0 +1,25 @@ +parameters: +- name: PackageName + type: string + default: 'not-specified' +- name: ServiceName + type: string + default: 'not-specified' +- name: ForRelease + type: boolean + default: false + +steps: + - task: Powershell@2 + inputs: + filePath: /eng/common/scripts/Verify-ChangeLog.ps1 + arguments: > + -PackageName ${{ parameters.PackageName }} + -ServiceName ${{ parameters.ServiceName }} + -RepoRoot $(Build.SourcesDirectory) + -RepoName $(Build.Repository.Name) + -ForRelease ${{ parameters.ForRelease }} + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Verify ChangeLog / Release Notes + continueOnError: false \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/verify-links.yml b/eng/common/pipelines/templates/steps/verify-links.yml new file mode 100644 index 000000000000..1a99350f0161 --- /dev/null +++ b/eng/common/pipelines/templates/steps/verify-links.yml @@ -0,0 +1,12 @@ +parameters: + Directory: 'not-specified' + +steps: + - task: PowerShell@2 + displayName: Link verification check + inputs: + pwsh: true + workingDirectory: $(Build.SourcesDirectory)/${{ parameters.Directory }} + filePath: eng/common/scripts/Verify-Links.ps1 + arguments: > + -urls $(dir -r -i *.md) -rootUrl "file://$(Build.SourcesDirectory)/${{ parameters.Directory }}" diff --git a/eng/common/scripts/Submit-PullRequest.ps1 b/eng/common/scripts/Submit-PullRequest.ps1 index 2313a83c9f9a..ef2a6f545061 100644 --- a/eng/common/scripts/Submit-PullRequest.ps1 +++ b/eng/common/scripts/Submit-PullRequest.ps1 @@ -41,21 +41,25 @@ param( $PRBody = $PRTitle ) -Write-Host "> $PSCommandPath $args" +$headers = @{ + Authorization = "bearer $AuthToken" +} $query = "state=open&head=${PROwner}:${PRBranch}&base=${BaseBranch}" -$resp = Invoke-RestMethod "https://api.github.com/repos/$RepoOwner/$RepoName/pulls?$query" +try { + $resp = Invoke-RestMethod -Headers $headers "https://api.github.com/repos/$RepoOwner/$RepoName/pulls?$query" +} +catch { + Write-Error "Invoke-RestMethod [https://api.github.com/repos/$RepoOwner/$RepoName/pulls?$query] failed with exception:`n$_" + exit 1 +} $resp | Write-Verbose if ($resp.Count -gt 0) { Write-Host -f green "Pull request already exists $($resp[0].html_url)" } else { - $headers = @{ - Authorization = "bearer $AuthToken" - } - $data = @{ title = $PRTitle head = "${PROwner}:${PRBranch}" @@ -64,9 +68,16 @@ else { maintainer_can_modify = $true } - $resp = Invoke-RestMethod -Method POST -Headers $headers ` - https://api.github.com/repos/$RepoOwner/$RepoName/pulls ` - -Body ($data | ConvertTo-Json) + try { + $resp = Invoke-RestMethod -Method POST -Headers $headers ` + "https://api.github.com/repos/$RepoOwner/$RepoName/pulls" ` + -Body ($data | ConvertTo-Json) + } + catch { + Write-Error "Invoke-RestMethod [https://api.github.com/repos/$RepoOwner/$RepoName/pulls] failed with exception:`n$_" + exit 1 + } + $resp | Write-Verbose Write-Host -f green "Pull request created https://github.com/$RepoOwner/$RepoName/pull/$($resp.number)" } diff --git a/eng/common/scripts/Verify-ChangeLog.ps1 b/eng/common/scripts/Verify-ChangeLog.ps1 new file mode 100644 index 000000000000..1f9991bfd668 --- /dev/null +++ b/eng/common/scripts/Verify-ChangeLog.ps1 @@ -0,0 +1,22 @@ +# Wrapper Script for ChangeLog Verification +param ( + [Parameter(Mandatory=$true)] + [string]$PackageName, + [Parameter(Mandatory=$true)] + [string]$ServiceName, + [string]$RepoRoot, + [ValidateSet("net","java","js","python")] + [string]$Language, + [string]$RepoName, + [boolean]$ForRelease=$False +) + +Import-Module "${PSScriptRoot}/modules/common-manifest.psd1" + +if ([System.String]::IsNullOrEmpty($Language)) +{ + $Language = $RepoName.Substring($RepoName.LastIndexOf('-') + 1) +} + +$PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot +Confirm-ChangeLog -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgReadMePath -ForRelease $ForRelease \ No newline at end of file diff --git a/eng/common/scripts/Verify-Links.ps1 b/eng/common/scripts/Verify-Links.ps1 new file mode 100644 index 000000000000..c78ce9020ebb --- /dev/null +++ b/eng/common/scripts/Verify-Links.ps1 @@ -0,0 +1,244 @@ +param ( + # url list to verify links. Can either be a http address or a local file request. Local file paths support md and html files. + [string[]] $urls, + # file that contains a set of links to ignore when verifying + [string] $ignoreLinksFile = "$PSScriptRoot/ignore-links.txt", + # switch that will enable devops specific logging for warnings + [switch] $devOpsLogging = $false, + # check the links recurisvely based on recursivePattern + [switch] $recursive = $true, + # recusiving check links for all links verified that begin with this baseUrl, defaults to the folder the url is contained in + [string] $baseUrl = "", + # path to the root of the site for resolving rooted relative links, defaults to host root for http and file directory for local files + [string] $rootUrl = "", + # list of http status codes count as broken links. Defaults to 404. + [array] $errorStatusCodes = @(404), + # flag to allow resolving relative paths or not + [bool] $resolveRelativeLinks = $true +) + +$ProgressPreference = "SilentlyContinue"; # Disable invoke-webrequest progress dialog + +function NormalizeUrl([string]$url){ + if (Test-Path $url) { + $url = "file://" + (Resolve-Path $url).ToString(); + } + + $uri = [System.Uri]$url; + + if ($script:baseUrl -eq "") { + # for base url default to containing directory + $script:baseUrl = (new-object System.Uri($uri, ".")).ToString(); + } + + if ($script:rootUrl -eq "") { + if ($uri.IsFile) { + # for files default to the containing directory + $script:rootUrl = $script:baseUrl; + } + else { + # for http links default to the root path + $script:rootUrl = new-object System.Uri($uri, "/"); + } + } + return $uri +} + +function LogWarning +{ + if ($devOpsLogging) + { + Write-Host "##vso[task.LogIssue type=warning;]$args" + } + else + { + Write-Warning "$args" + } +} + +function ResolveUri ([System.Uri]$referralUri, [string]$link) +{ + # If the link is mailto, skip it. + if ($link.StartsWith("mailto:")) { + Write-Verbose "Skipping $link because it is a mailto link." + return $null + } + + $linkUri = [System.Uri]$link; + if($resolveRelativeLinks){ + if (!$linkUri.IsAbsoluteUri) { + # For rooted paths resolve from the baseUrl + if ($link.StartsWith("/")) { + echo "rooturl = $rootUrl" + $linkUri = new-object System.Uri([System.Uri]$rootUrl, ".$link"); + } + else { + $linkUri = new-object System.Uri($referralUri, $link); + } + } + } + + $linkUri = [System.Uri]$linkUri.GetComponents([System.UriComponents]::HttpRequestUrl, [System.UriFormat]::SafeUnescaped) + Write-Verbose "ResolvedUri $link to $linkUri" + + # If the link is not a web request, like mailto, skip it. + if (!$linkUri.Scheme.StartsWith("http") -and !$linkUri.IsFile) { + Write-Verbose "Skipping $linkUri because it is not http or file based." + return $null + } + + if ($null -ne $ignoreLinks -and $ignoreLinks.Contains($link)) { + Write-Verbose "Ignoring invalid link $linkUri because it is in the ignore file." + return $null + } + + return $linkUri; +} + +function ParseLinks([string]$baseUri, [string]$htmlContent) +{ + $hrefRegex = "]+href\s*=\s*[""']?(?[^""']*)[""']?" + $regexOptions = [System.Text.RegularExpressions.RegexOptions]"Singleline, IgnoreCase"; + + $hrefs = [RegEx]::Matches($htmlContent, $hrefRegex, $regexOptions); + + #$hrefs | Foreach-Object { Write-Host $_ } + + Write-Verbose "Found $($hrefs.Count) raw href's in page $baseUri"; + $links = $hrefs | ForEach-Object { ResolveUri $baseUri $_.Groups["href"].Value } | Sort-Object -Unique + + #$links | Foreach-Object { Write-Host $_ } + + return $links +} + +function CheckLink ([System.Uri]$linkUri) +{ + if ($checkedLinks.ContainsKey($linkUri)) { return } + + Write-Verbose "Checking link $linkUri..." + if ($linkUri.IsFile) { + if (!(Test-Path $linkUri.LocalPath)) { + LogWarning "Link to file does not exist $($linkUri.LocalPath)" + $script:badLinks += $linkUri + } + } + else { + try { + $response = Invoke-WebRequest -Uri $linkUri + $statusCode = $response.StatusCode + if ($statusCode -ne 200) { + Write-Host "[$statusCode] while requesting $linkUri" + } + } + catch { + $statusCode = $_.Exception.Response.StatusCode.value__ + + if ($statusCode -in $errorStatusCodes) { + LogWarning "[$statusCode] broken link $linkUri" + $script:badLinks += $linkUri + } + else { + if ($null -ne $statusCode) { + Write-Host "[$statusCode] while requesting $linkUri" + } + else { + Write-Host "Exception while requesting $linkUri" + Write-Host $_.Exception.ToString() + } + } + } + } + $checkedLinks[$linkUri] = $true; +} + +function GetLinks([System.Uri]$pageUri) +{ + if ($pageUri.Scheme.StartsWith("http")) { + try { + $response = Invoke-WebRequest -Uri $pageUri + $content = $response.Content + } + catch { + $statusCode = $_.Exception.Response.StatusCode.value__ + Write-Error "Invalid page [$statusCode] $pageUri" + } + } + elseif ($pageUri.IsFile -and (Test-Path $pageUri.LocalPath)) { + $file = $pageUri.LocalPath + if ($file.EndsWith(".md")) { + $content = (ConvertFrom-MarkDown $file).html + } + elseif ($file.EndsWith(".html")) { + $content = Get-Content $file + } + else { + if (Test-Path ($file + "index.html")) { + $content = Get-Content ($file + "index.html") + } + else { + # Fallback to just reading the content directly + $content = Get-Content $file + } + } + } + else { + Write-Error "Don't know how to process uri $pageUri" + } + + $links = ParseLinks $pageUri $content + + return $links; +} + +if ($urls) { + if ($urls.Count -eq 0) { + Write-Host "Usage $($MyInvocation.MyCommand.Name) "; + exit 1; + } +} + +if ($PSVersionTable.PSVersion.Major -lt 6) +{ + LogWarning "Some web requests will not work in versions of PS earlier then 6. You are running version $($PSVersionTable.PSVersion)." +} + +$badLinks = @(); +$ignoreLinks = @(); +if (Test-Path $ignoreLinksFile) +{ + $ignoreLinks = [Array](Get-Content $ignoreLinksFile | ForEach-Object { ($_ -replace "#.*", "").Trim() } | Where-Object { $_ -ne "" }) +} + +$checkedPages = @{}; +$checkedLinks = @{}; +$pageUrisToCheck = new-object System.Collections.Queue + +foreach ($url in $urls) { + $uri = NormalizeUrl $url + $pageUrisToCheck.Enqueue($uri); +} + +while ($pageUrisToCheck.Count -ne 0) +{ + $pageUri = $pageUrisToCheck.Dequeue(); + if ($checkedPages.ContainsKey($pageUri)) { continue } + $checkedPages[$pageUri] = $true; + + $linkUris = GetLinks $pageUri + Write-Host "Found $($linkUris.Count) links on page $pageUri"; + + foreach ($linkUri in $linkUris) { + CheckLink $linkUri + if ($recursive) { + if ($linkUri.ToString().StartsWith($baseUrl) -and !$checkedPages.ContainsKey($linkUri)) { + $pageUrisToCheck.Enqueue($linkUri); + } + } + } +} + +Write-Host "Found $($checkedLinks.Count) links with $($badLinks.Count) broken" +$badLinks | ForEach-Object { Write-Host " $_" } + +exit $badLinks.Count diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 2d2362f0d0d9..e48a526fe9e5 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -1,3 +1,4 @@ +Import-Module "${PSScriptRoot}/modules/ChangeLog-Operations.psm1" . (Join-Path $PSScriptRoot SemVer.ps1) $SDIST_PACKAGE_REGEX = "^(?.*)\-(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" @@ -8,8 +9,8 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { Write-Host "Creating release $($pkgInfo.Tag)" $releaseNotes = "" - if ($pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent -ne $null) { - $releaseNotes = $pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent + if ($pkgInfo.ReleaseNotes -ne $null) { + $releaseNotes = $pkgInfo.ReleaseNotes } $isPrerelease = $False @@ -96,7 +97,7 @@ function ParseMavenPackage($pkg, $workingDirectory) { $changeLogLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-readme.md")[0] @@ -169,13 +170,15 @@ function ParseNPMPackage($pkg, $workingDirectory) { tar -xzf $pkg $packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json + $pkgId = $packageJSON.name + $pkgVersion = $packageJSON.version $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] + $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1 if ($readmeContentLoc) { $readmeContent = Get-Content -Raw $readmeContentLoc } @@ -183,9 +186,6 @@ function ParseNPMPackage($pkg, $workingDirectory) { cd $origFolder Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - $pkgId = $packageJSON.name - $pkgVersion = $packageJSON.version - $resultObj = New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion @@ -229,10 +229,12 @@ function ParseNugetPackage($pkg, $workingDirectory) { Copy-Item -Path $pkg -Destination $zipFileLocation Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder [xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content + $pkgId = $packageXML.package.metadata.id + $pkgVersion = $packageXML.package.metadata.version $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] @@ -241,8 +243,6 @@ function ParseNugetPackage($pkg, $workingDirectory) { } Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - $pkgId = $packageXML.package.metadata.id - $pkgVersion = $packageXML.package.metadata.version return New-Object PSObject -Property @{ PackageId = $pkgId @@ -297,13 +297,15 @@ function ParsePyPIPackage($pkg, $workingDirectory) { $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] + $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1 + if ($readmeContentLoc) { $readmeContent = Get-Content -Raw $readmeContentLoc } + Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue return New-Object PSObject -Property @{ @@ -321,10 +323,12 @@ function ParseCArtifact($pkg, $workingDirectory) { $releaseNotes = "" $readmeContent = "" + $pkgVersion = $packageInfo.version + $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0] @@ -333,8 +337,8 @@ function ParseCArtifact($pkg, $workingDirectory) { } return New-Object PSObject -Property @{ - PackageId = '' - PackageVersion = $packageInfo.version + PackageId = 'azure-sdk-for-c' + PackageVersion = $pkgVersion # Artifact info is always considered deployable for C becasue it is not # deployed anywhere. Dealing with duplicate tags happens downstream in # CheckArtifactShaAgainstTagsList diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index 03e508c17cb5..0093c38e7f57 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -10,8 +10,6 @@ param ( $UploadLatest=1 ) -Write-Host "> $PSCommandPath $args" - $Language = $Language.ToLower() # Regex inspired but simplified from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 index f87c90997839..cec45536b704 100644 --- a/eng/common/scripts/create-tags-and-git-release.ps1 +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -6,6 +6,7 @@ param ( # used by VerifyPackages $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory) $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary) + [ValidateSet("Nuget","NPM","PyPI","Maven")] $packageRepository, # used to indicate destination against which we will check the existing version. # valid options: PyPI, Nuget, NPM, Maven, C # used by CreateTags @@ -18,8 +19,6 @@ param ( [switch]$continueOnError = $false ) -Write-Host "> $PSCommandPath $args" - . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) $apiUrl = "https://api.github.com/repos/$repoId" diff --git a/eng/common/scripts/git-branch-push.ps1 b/eng/common/scripts/git-branch-push.ps1 index 9ff45f87ad67..9b3d78345589 100644 --- a/eng/common/scripts/git-branch-push.ps1 +++ b/eng/common/scripts/git-branch-push.ps1 @@ -28,8 +28,6 @@ param( [string] $PushArgs = "" ) -Write-Host "> $PSCommandPath $args" - # This is necessay because of the janky git command output writing to stderr. # Without explicitly setting the ErrorActionPreference to continue the script # would fail the first time git wrote command output. @@ -118,11 +116,11 @@ do if ($LASTEXITCODE -ne 0) { Write-Error "Unable to apply diff file LASTEXITCODE=$($LASTEXITCODE), see command output above." - continue + exit $LASTEXITCODE } Write-Host "git add -A" - git add -A + git add -A if ($LASTEXITCODE -ne 0) { Write-Error "Unable to git add LASTEXITCODE=$($LASTEXITCODE), see command output above." diff --git a/eng/common/scripts/modules/ChangeLog-Operations.psm1 b/eng/common/scripts/modules/ChangeLog-Operations.psm1 new file mode 100644 index 000000000000..5279668412b7 --- /dev/null +++ b/eng/common/scripts/modules/ChangeLog-Operations.psm1 @@ -0,0 +1,116 @@ +# Common Changelog Operations + +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)" + +# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG +function Get-ChangeLogEntries { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation + ) + + $changeLogEntries = @{} + if (!(Test-Path $ChangeLogLocation)) { + Write-Host "ChangeLog '{0}' was not found" -f $ChangeLogLocation + exit 1 + } + + try { + $contents = Get-Content $ChangeLogLocation + # walk the document, finding where the version specifiers are and creating lists + $changeLogEntry = $null + foreach ($line in $contents) { + if ($line -match $RELEASE_TITLE_REGEX) { + $changeLogEntry = [pscustomobject]@{ + ReleaseVersion = $matches["version"] + ReleaseStatus = $matches["releaseStatus"] + ReleaseTitle = $line + ReleaseContent = @() # Release content without the version title + } + $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry + } + else { + if ($changeLogEntry) { + $changeLogEntry.ReleaseContent += $line + } + } + } + } + catch { + Write-Host "Error parsing $ChangeLogLocation." + Write-Host $_.Exception.Message + } + return $changeLogEntries +} + +# Returns single changeLogEntry object containing the ChangeLog for a particular version +function Get-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + + $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation + + if ($changeLogEntries.ContainsKey($VersionString)) { + return $changeLogEntries[$VersionString] + } + Write-Error "Release Notes for the Specified version ${VersionString} was not found" + exit 1 +} + +#Returns the changelog for a particular version as string +function Get-ChangeLogEntryAsString { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + + $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + [string]$releaseTitle = $changeLogEntries.ReleaseTitle + [string]$releaseContent = $changeLogEntries.ReleaseContent -Join [Environment]::NewLine + return $releaseTitle, $releaseContent -Join [Environment]::NewLine +} + +function Confirm-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString, + [boolean]$ForRelease = $false + ) + + $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + + if ([System.String]::IsNullOrEmpty($changeLogEntries.ReleaseStatus)) { + Write-Host ("##[error]Changelog '{0}' has wrong release note title" -f $ChangeLogLocation) + Write-Host "##[info]Ensure the release date is included i.e. (yyyy-MM-dd) or (Unreleased) if not yet released" + exit 1 + } + + if ($ForRelease -eq $True) { + $CurrentDate = Get-Date -Format "yyyy-MM-dd" + if ($changeLogEntries.ReleaseStatus -ne "($CurrentDate)") { + Write-Host ("##[warning]Incorrect Date: Please use the current date in the Changelog '{0}' before releasing the package" -f $ChangeLogLocation) + exit 1 + } + + if ([System.String]::IsNullOrWhiteSpace($changeLogEntries.ReleaseContent)) { + Write-Host ("##[error]Empty Release Notes for '{0}' in '{1}'" -f $VersionString, $ChangeLogLocation) + Write-Host "##[info]Please ensure there is a release notes entry before releasing the package." + exit 1 + } + } + + Write-Host ($changeLogEntries | Format-Table | Out-String) +} + +Export-ModuleMember -Function 'Get-ChangeLogEntries' +Export-ModuleMember -Function 'Get-ChangeLogEntry' +Export-ModuleMember -Function 'Get-ChangeLogEntryAsString' +Export-ModuleMember -Function 'Confirm-ChangeLogEntry' \ No newline at end of file diff --git a/eng/common/scripts/modules/common-manifest.psd1 b/eng/common/scripts/modules/common-manifest.psd1 index 419f046e6f67..43dee1c26e23 100644 --- a/eng/common/scripts/modules/common-manifest.psd1 +++ b/eng/common/scripts/modules/common-manifest.psd1 @@ -66,7 +66,7 @@ ScriptsToProcess = @("${PSScriptRoot}\..\SemVer.ps1") # FormatsToProcess = @() # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess -NestedModules = @("${PSScriptRoot}\Package-Properties.psm1") +NestedModules = @("${PSScriptRoot}\Package-Properties.psm1", "${PSScriptRoot}\ChangeLog-Operations.psm1") # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. # FunctionsToExport = @() diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index f88717c4f714..a858078f5448 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -14,8 +14,6 @@ param ( $DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go? ) -Write-Host "> $PSCommandPath $args" - # import artifact parsing and semver handling . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) @@ -24,19 +22,19 @@ Write-Host "> $PSCommandPath $args" function GetMetaData($lang){ switch ($lang) { "java" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/java-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv" break } ".net" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/dotnet-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv" break } "python" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/python-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv" break } "javascript" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/js-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv" break } default { @@ -58,12 +56,12 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ $pkgId = $pkgInfo.PackageId.Replace("@azure/", "") try { - $metadata = GetMetaData -lang $lang + $metadata = GetMetaData -lang $lang $service = $metadata | ? { $_.Package -eq $pkgId } if ($service) { - $service = "$($service.Service)" + $service = "$($service.ServiceName)".ToLower().Replace(" ", "") } } catch { @@ -72,15 +70,17 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ } $fileContent = $pkgInfo.ReadmeContent + $foundTitle = "" # only replace the version if the formatted header can be found $headerContentMatches = (Select-String -InputObject $pkgInfo.ReadmeContent -Pattern 'Azure .+? (client|plugin|shared) library for (JavaScript|Java|Python|\.NET|C)') if ($headerContentMatches) { - $headerContentMatch = $headerContentMatches.Matches[0] - $header = "---`ntitle: $headerContentMatch`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n" - $fileContent = $pkgInfo.ReadmeContent -replace $headerContentMatch, "$headerContentMatch - Version $($pkgInfo.PackageVersion) `n" + $foundTitle = $headerContentMatches.Matches[0] + $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n" } + $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n" + if ($fileContent) { return "$header`n$fileContent" } @@ -99,7 +99,7 @@ $pkgs = VerifyPackages -pkgRepository $Repository ` if ($pkgs) { Write-Host "Given the visible artifacts, readmes will be copied for the following packages" - Write-Host ($pkgs | % { $_.PackageId }) + Write-Host ($pkgs | % { $_.PackageId }) foreach ($packageInfo in $pkgs) { # sync the doc repo