Skip to content

Commit

Permalink
Sync eng/common directory with azure-sdk-tools for PR 2861 (Azure#20837)
Browse files Browse the repository at this point in the history
* Use common script for git diff changes

* remove extra files

* suppress spell check

* suppress false positive cspelling

* Address comments

* make changes to cspell

* Suppress to right values

* More fix

Co-authored-by: sima-zhu <[email protected]>
  • Loading branch information
azure-sdk and sima-zhu authored Mar 23, 2022
1 parent 0df989b commit 4cff1c1
Show file tree
Hide file tree
Showing 9 changed files with 110 additions and 103 deletions.
11 changes: 5 additions & 6 deletions eng/common/pipelines/templates/steps/credscan.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
# cSpell:ignore changedfiles
# cSpell:ignore credscan
# cSpell:ignore securedevelopmentteam
# cSpell:ignore postanalysis
parameters:
SuppressionFilePath: 'eng/CredScanSuppression.json'
BaselineFilePath: ''
Expand All @@ -7,12 +11,7 @@ parameters:
steps:
- pwsh: |
if ("$(Build.Reason)" -eq 'PullRequest') {
$targetBranch = "origin/$(System.PullRequest.TargetBranch)" -replace "refs/heads/"
# Add config to disable the quote and encoding on file name.
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
$changedFiles = git -c core.quotepath=off -c i18n.logoutputencoding=utf-8 diff $targetBranch HEAD --name-only --diff-filter=d
$changedFiles = & "eng/common/scripts/get-changedfiles.ps1"
$changedFiles | ForEach-Object { Add-Content -Path "${{ parameters.SourceDirectory }}/credscan.tsv" -Value "${{ parameters.SourceDirectory }}/$_"}
}
else {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
# cSpell:ignore changedfiles
# cSpell:ignore Committish
# cSpell:ignore LASTEXITCODE

steps:
- template: /eng/common/pipelines/templates/steps/set-default-branch.yml
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- pwsh: |
Set-PsDebug -Trace 1
# Find the default branch of the repo
$defaultBranch = (git remote show origin | Out-String) -replace "(?ms).*HEAD branch: (\w+).*", '$1'
# Find the default branch of the repo. The variable value sets in build step.
Write-Host "Default Branch: $(DefaultBranch)"
$targetBranch = "$(System.PullRequest.TargetBranch)" -replace "refs/heads/"
Write-Host "Default Branch: ${defaultBranch}"
if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-eng/common")) -and $targetBranch -eq $defaultBranch)
if ((!"$(System.PullRequest.SourceBranch)".StartsWith("sync-eng/common")) -and $targetBranch -eq "$(DefaultBranch)")
{
$filesInCommonDir = git diff "origin/${targetBranch}" HEAD --name-only -- 'eng/common/*'
$filesInCommonDir = & "eng/common/scripts/get-changedfiles.ps1" -TargetCommittish $targetBranch -DiffPath 'eng/common/*'
if (($LASTEXITCODE -eq 0) -and ($filesInCommonDir.Count -gt 0))
{
Write-Host "##vso[task.LogIssue type=error;]Changes to files under 'eng/common' directory should not be made in this Repo`n${filesInCommonDir}"
Expand Down
8 changes: 5 additions & 3 deletions eng/common/scripts/Detect-Api-Changes.ps1
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# cSpell:ignore PULLREQUEST
# cSpell:ignore TARGETBRANCH
[CmdletBinding()]
Param (
[Parameter(Mandatory=$True)]
Expand All @@ -16,6 +18,8 @@ Param (
[string] $TargetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/")
)

. (Join-Path $PSScriptRoot common.ps1)

# Submit API review request and return status whether current revision is approved or pending or failed to create review
function Submit-Request($filePath, $packageName)
{
Expand Down Expand Up @@ -61,8 +65,7 @@ function Should-Process-Package($pkgPath, $packageName)
# Get package info from json file created before updating version to daily dev
$pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json
$packagePath = $pkgInfo.DirectoryPath
$modifiedFiles = git diff --name-only --relative $TargetBranch HEAD
$modifiedFiles = $modifiedFiles.Where({$_.startswith($packagePath)})
$modifiedFiles = Get-ChangedFiles -DiffPath "$packagePath/*" -DiffFilterType ''
$filteredFileCount = $modifiedFiles.Count
Write-Host "Number of modified files for package: $filteredFileCount"
return ($filteredFileCount -gt 0 -and $pkgInfo.IsNewSdk)
Expand All @@ -80,7 +83,6 @@ function Log-Input-Params()
Write-Host "Package Name: $($PackageName)"
}

. (Join-Path $PSScriptRoot common.ps1)
Log-Input-Params

if (!($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn")))
Expand Down
38 changes: 38 additions & 0 deletions eng/common/scripts/Helpers/git-helpers.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# cSpell:ignore Committish
# cSpell:ignore PULLREQUEST
# cSpell:ignore TARGETBRANCH
# cSpell:ignore SOURCECOMMITID
function Get-ChangedFiles {
param (
[string]$SourceCommittish= "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}",
[string]$TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"),
[string]$DiffPath,
[string]$DiffFilterType = "d"
)
# If ${env:SYSTEM_PULLREQUEST_TARGETBRANCH} is empty, then return empty.
if ($TargetCommittish -eq "origin/") {
Write-Host "There is no target branch passed in. "
return ""
}

# Add config to disable the quote and encoding on file name.
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-quoted-file-names
# Ref: https://github.com/msysgit/msysgit/wiki/Git-for-Windows-Unicode-Support#disable-commit-message-transcoding
# Git PR diff: https://docs.github.com/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-comparing-branches-in-pull-requests#three-dot-and-two-dot-git-diff-comparisons
$command = "git -c core.quotepath=off -c i18n.logoutputencoding=utf-8 diff `"$TargetCommittish...$SourceCommittish`" --name-only --diff-filter=$DiffFilterType"
if ($DiffPath) {
$command = $command + " -- `'$DiffPath`'"
}
Write-Host $command
$changedFiles = Invoke-Expression -Command $command
if(!$changedFiles) {
Write-Host "No changed files in git diff between $TargetCommittish and $SourceCommittish"
}
else {
Write-Host "Here are the diff files:"
foreach ($file in $changedFiles) {
Write-Host " $file"
}
}
return $changedFiles
}
11 changes: 4 additions & 7 deletions eng/common/scripts/check-spelling-in-changed-files.ps1
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# cSpell:ignore LASTEXITCODE
# cSpell:ignore errrrrorrrrr
# cSpell:ignore sepleing
<#
.SYNOPSIS
Uses cspell (from NPM) to check spelling of recently changed files
Expand Down Expand Up @@ -286,13 +289,7 @@ if (!(Test-Path $CspellConfigPath)) {
# Lists names of files that were in some way changed between the
# current $SourceBranch and $TargetBranch. Excludes files that were deleted to
# prevent errors in Resolve-Path
Write-Host "git diff --diff-filter=d --name-only --relative $TargetBranch $SourceBranch"
$changedFilesList = git diff `
--diff-filter=d `
--name-only `
--relative `
$TargetBranch `
$SourceBranch
$changedFilesList = Get-ChangedFiles

$changedFiles = @()
foreach ($file in $changedFilesList) {
Expand Down
3 changes: 3 additions & 0 deletions eng/common/scripts/common.ps1
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# cSpell:ignore Apireview
# cSpell:ignore Onboarded
$RepoRoot = Resolve-Path "${PSScriptRoot}..\..\..\.."
$EngDir = Join-Path $RepoRoot "eng"
$EngCommonDir = Join-Path $EngDir "common"
Expand All @@ -12,6 +14,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts"
. (Join-Path $EngCommonScriptsDir Invoke-GitHubAPI.ps1)
. (Join-Path $EngCommonScriptsDir Invoke-DevOpsAPI.ps1)
. (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1)
. (Join-Path $EngCommonScriptsDir "Helpers" git-helpers.ps1)

# Setting expected from common languages settings
$Language = "Unknown"
Expand Down
40 changes: 40 additions & 0 deletions eng/common/scripts/get-changedfiles.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@

# cSpell:ignore Committish
# cSpell:ignore committish
# cSpell:ignore PULLREQUEST
# cSpell:ignore TARGETBRANCH
# cSpell:ignore SOURCECOMMITID
# cSpell:ignore elete
# cSpell:ignore ename
<#
.SYNOPSIS
Returns git diff changes in pull request.
.DESCRIPTION
The script is to return diff changes in pull request.
.PARAMETER SourceCommittish
The branch committish PR merges from.
Definition of committish: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefcommit-ishacommit-ishalsocommittish
.PARAMETER TargetCommittish
The branch committish PR targets to merge into.
.PARAMETER DiffPath
The files which git diff to scan against. Support regex match. E.g. "eng/common/*", "*.md"
.PARAMETER DiffFilterType
The filter type A(a)dd, D(d)elete, R(r)ename, U(u)pate.
E.g. 'ad' means filter out the newly added file and deleted file
E.g. '' means no filter on file mode.
#>
[CmdletBinding()]
param (
[string] $SourceCommittish = "${env:SYSTEM_PULLREQUEST_SOURCECOMMITID}",
[string] $TargetCommittish = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "refs/heads/"),
[string] $DiffPath = "",
[string] $DiffFilterType = 'd'
)

Set-StrictMode -Version 3
. (Join-Path $PSScriptRoot common.ps1)

return Get-ChangedFiles -SourceCommittish $SourceCommittish `
-TargetCommittish $TargetCommittish `
-DiffPath $DiffPath `
-DiffFilterType $DiffFilterType
48 changes: 6 additions & 42 deletions eng/common/scripts/get-markdown-files-from-changed-files.ps1
Original file line number Diff line number Diff line change
@@ -1,49 +1,13 @@
# cSpell:ignore Committish
# cSpell:ignore PULLREQUEST
# cSpell:ignore TARGETBRANCH
param (
# The root repo we scaned with.
# The root repo we scanned with.
[string] $RootRepo = '$PSScriptRoot/../../..',
# The target branch to compare with.
[string] $targetBranch = ("origin/${env:SYSTEM_PULLREQUEST_TARGETBRANCH}" -replace "/refs/heads/")
)
$deletedFiles = (git diff $targetBranch HEAD --name-only --diff-filter=D)
$renamedFiles = (git diff $targetBranch HEAD --diff-filter=R)
$changedMarkdowns = (git diff $targetBranch HEAD --name-only -- '*.md')

$beforeRenameFiles = @()
# Retrieve the 'renamed from' files. Git command only returns back the files after rename.
# In order to have the files path before rename, it has to do some regex checking.
# It is better to be replaced by more reliable commands if any.
foreach ($file in $renamedFiles) {
if ($file -match "^rename from (.*)$") {
$beforeRenameFiles += $file -replace "^rename from (.*)$", '$1'
}
}
# A combined list of deleted and renamed files.
$relativePathLinks = ($deletedFiles + $beforeRenameFiles)
# Removed the deleted markdowns.
$changedMarkdowns = $changedMarkdowns | Where-Object { $deletedFiles -notcontains $_ }
# Scan all markdowns and find if it contains the deleted or renamed files.
$markdownContainLinks = @()
$allMarkdownFiles = Get-ChildItem -Path $RootRepo -Recurse -Include *.md
foreach ($f in $allMarkdownFiles) {
$filePath = $f.FullName
$content = Get-Content -Path $filePath -Raw
foreach ($l in $relativePathLinks) {
if ($content -match $l) {
$markdownContainLinks += $filePath
break
}
}
}
. (Join-Path $PSScriptRoot common.ps1)

# Convert markdowns path of the PR to absolute path.
$adjustedReadmes = $changedMarkdowns | Foreach-Object { Resolve-Path $_ }
$markdownContainLinks += $adjustedReadmes

# Get rid of any duplicated ones.
$allMarkdowns = [string[]]($markdownContainLinks | Sort-Object | Get-Unique)

Write-Host "Here are all markdown files we need to check based on the changed files:"
foreach ($file in $allMarkdowns) {
Write-Host " $file"
}
return $allMarkdowns
return Get-ChangedFiles -TargetCommittish $targetBranch -DiffPath '*.md'
36 changes: 0 additions & 36 deletions eng/common/scripts/git-diff-changes.ps1

This file was deleted.

0 comments on commit 4cff1c1

Please sign in to comment.