From f90cdb0b5878ae1a3963b1a445b38d15dc0df409 Mon Sep 17 00:00:00 2001 From: Yunchi Wang <54880216+wyunchi-ms@users.noreply.github.com> Date: Fri, 4 Nov 2022 20:56:00 +0800 Subject: [PATCH] Add logic for file change check (#20015) * Add logic for necessary change check * Remove tools/CheckChangeLog.ps1 * Rename task name * Rename task name * Rename task name * Update script to be more dotnet * Update script to be more dotnet --- .ci-config.json | 6 +- build.proj | 7 +- .../CIFilterTask.cs | 6 + tools/CheckChangeLog.ps1 | 108 ------------------ tools/PipelineResultTemplate.json | 6 + .../CollectStaticAnalysisPipelineResult.ps1 | 9 +- .../FileChangeAnalyzer/Test-FileChange.ps1 | 73 ++++++++++++ 7 files changed, 102 insertions(+), 113 deletions(-) delete mode 100644 tools/CheckChangeLog.ps1 create mode 100644 tools/StaticAnalysis/FileChangeAnalyzer/Test-FileChange.ps1 diff --git a/.ci-config.json b/.ci-config.json index 97dff94565e2..852e0e8ab2bd 100644 --- a/.ci-config.json +++ b/.ci-config.json @@ -99,7 +99,8 @@ "phases": [ "build:related-module", "dependence:dependence-module", - "test:dependence-module" + "test:dependence-module", + "file-change:module" ] }, { @@ -111,7 +112,8 @@ "breaking-change:module", "help:module", "signature:module", - "test:dependence-module" + "test:dependence-module", + "file-change:module" ] }, { diff --git a/build.proj b/build.proj index 1e964d1ac348..62db9d2829db 100644 --- a/build.proj +++ b/build.proj @@ -268,7 +268,12 @@ - + + + + + + diff --git a/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs b/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs index ae551ab2b11d..b1569c0e04f2 100644 --- a/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs +++ b/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs @@ -74,6 +74,7 @@ public class CIFilterTask : Task private const string ANALYSIS_HELP_PHASE = "help"; private const string ANALYSIS_DEPENDENCY_PHASE = "dependency"; private const string ANALYSIS_SIGNATURE_PHASE = "signature"; + private const string ANALYSIS_FILE_CHANGE_PHASE = "file-change"; private const string TEST_PHASE = "test"; private const string ACCOUNT_MODULE_NAME = "Accounts"; @@ -190,6 +191,7 @@ private bool ProcessTargetModule(Dictionary csprojMap) [ANALYSIS_HELP_EXAMPLE_PHASE] = new HashSet(GetDependenceModuleList(TargetModule, csprojMap).ToList()), [ANALYSIS_HELP_PHASE] = new HashSet(GetDependenceModuleList(TargetModule, csprojMap).ToList()), [ANALYSIS_SIGNATURE_PHASE] = new HashSet(GetDependenceModuleList(TargetModule, csprojMap).ToList()), + [ANALYSIS_FILE_CHANGE_PHASE] = new HashSet(GetDependenceModuleList(TargetModule, csprojMap).ToList()), [TEST_PHASE] = new HashSet(GetTestCsprojList(TargetModule, csprojMap).ToList()) }; @@ -245,6 +247,7 @@ private Dictionary> CalculateInfluencedModuleInfoForEach ANALYSIS_HELP_EXAMPLE_PHASE + ":" + AllModule, ANALYSIS_HELP_PHASE + ":" + AllModule, ANALYSIS_SIGNATURE_PHASE + ":" + AllModule, + ANALYSIS_FILE_CHANGE_PHASE + ":" + AllModule, TEST_PHASE + ":" + AllModule, }; } @@ -293,6 +296,7 @@ private Dictionary> CalculateInfluencedModuleInfoForEach ANALYSIS_HELP_EXAMPLE_PHASE, ANALYSIS_HELP_PHASE, ANALYSIS_SIGNATURE_PHASE, + ANALYSIS_FILE_CHANGE_PHASE, TEST_PHASE }; foreach (string phaseName in expectedKeyList) @@ -413,6 +417,7 @@ private bool ProcessFileChanged(Dictionary csprojMap) [ANALYSIS_HELP_EXAMPLE_PHASE] = influencedModuleInfo[ANALYSIS_HELP_EXAMPLE_PHASE], [ANALYSIS_HELP_PHASE] = influencedModuleInfo[ANALYSIS_HELP_PHASE], [ANALYSIS_SIGNATURE_PHASE] = influencedModuleInfo[ANALYSIS_SIGNATURE_PHASE], + [ANALYSIS_FILE_CHANGE_PHASE] = influencedModuleInfo[ANALYSIS_FILE_CHANGE_PHASE], [TEST_PHASE] = new HashSet(influencedModuleInfo[TEST_PHASE].Select(GetModuleNameFromPath).Where(x => x != null)) }; File.WriteAllText(Path.Combine(config.ArtifactPipelineInfoFolder, "CIPlan.json"), JsonConvert.SerializeObject(CIPlan, Formatting.Indented)); @@ -463,6 +468,7 @@ public override bool Execute() [ANALYSIS_HELP_EXAMPLE_PHASE] = new HashSet(selectedModuleList), [ANALYSIS_HELP_PHASE] = new HashSet(selectedModuleList), [ANALYSIS_SIGNATURE_PHASE] = new HashSet(selectedModuleList), + [ANALYSIS_HELP_EXAMPLE_PHASE] = new HashSet(selectedModuleList), [TEST_PHASE] = new HashSet(selectedModuleList) }; FilterTaskResult.PhaseInfo = CalculateCsprojForBuildAndTest(influencedModuleInfo, csprojMap); diff --git a/tools/CheckChangeLog.ps1 b/tools/CheckChangeLog.ps1 deleted file mode 100644 index e119638ced0f..000000000000 --- a/tools/CheckChangeLog.ps1 +++ /dev/null @@ -1,108 +0,0 @@ -[CmdletBinding()] -Param -( - [Parameter()] - [string]$FilesChanged -) - -$PathsToCheck = @("src") - -$PathStringsToIgnore = @( - "Test", - ".sln", - "Nuget.config", - ".psd1", - "Netcore", - "Stack" -) -Write-Host "Files changed: $FilesChanged" -$FilesChangedList = @() -while ($true) -{ - $Idx = $FilesChanged.IndexOf(";") - if ($Idx -lt 0) - { - $FilesChangedList += $FilesChanged - break - } - - $TempFile = $FilesChanged.Substring(0, $Idx) - Write-Host "Adding '$TempFile' to 'FilesChangedList'" - $FilesChangedList += $TempFile - $FilesChanged = $FilesChanged.Substring($Idx + 1) -} - -if ([string]::IsNullOrEmpty($FilesChanged) -or ($FilesChangedList.Count -eq 300)) -{ - Write-Host "The list of files changed is empty or is past the 300 file limit; skipping check for change log entry" - return -} - -$ChangeLogs = $FilesChangedList | where { $_ -like "*ChangeLog.md*" } -$UpdatedServicePaths = New-Object System.Collections.Generic.HashSet[string] -foreach ($ChangeLog in $ChangeLogs) -{ - if ($ChangeLog -eq "ChangeLog.md") - { - continue - } - elseif ($ChangeLog -like "src/ServiceManagement*") - { - $UpdatedServicePaths.Add("src/ServiceManagement") | Out-Null - } - elseif ($ChangeLog -like "src/Storage*") - { - $UpdatedServicePaths.Add("src/Storage") | Out-Null - } - else - { - # Handle to construct a string like "src/{{service}}" - $SplitPath = @() - while ($true) - { - $Idx = $ChangeLog.IndexOf("/") - if ($Idx -lt 0) - { - $SplitPath += $ChangeLog - break - } - - $TempPath = $ChangeLog.Substring(0, $Idx) - Write-Host "Adding '$TempPath' to 'SplitPath'" - $SplitPath += $TempPath - $ChangeLog = $ChangeLog.Substring($Idx + 1) - } - - $BasePath = $SplitPath[0],$SplitPath[1],$SplitPath[2] -join "/" - Write-Host "Change log '$ChangeLog' processed to base path '$BasePath'" - $UpdatedServicePaths.Add($BasePath) | Out-Null - } -} - -$message = "The following services were found to have a change log update:`n" -$UpdatedServicePaths | % { $message += "`t- $_`n" } -Write-Host "$message`n" - -$FlaggedFiles = @() -foreach ($File in $FilesChangedList) -{ - if ($File -like "*ChangeLog.md*" -or $File -like "*.psd1*" -or $File -like "*.sln") - { - continue - } - - if (($PathsToCheck | where { $File.StartsWith($_) } | Measure-Object).Count -gt 0 -and ` - ($PathStringsToIgnore | where { $File -like "*$_*" } | Measure-Object).Count -eq 0 -and ` - ($UpdatedServicePaths | where { $File.StartsWith($_) } | Measure-Object).Count -eq 0) - { - $FlaggedFiles += $File - } -} - -if ($FlaggedFiles.Count -gt 0) -{ - $message = "The following files were flagged for not having a change log entry:`n" - $FlaggedFiles | % { $message += "`t- $_`n" } - Write-Host $message - throw "Modified files were found with no update to their change log. Please add a snippet to the affected modules' change log." -} diff --git a/tools/PipelineResultTemplate.json b/tools/PipelineResultTemplate.json index 6b4f7ee2a478..9347ea04bfaf 100644 --- a/tools/PipelineResultTemplate.json +++ b/tools/PipelineResultTemplate.json @@ -29,6 +29,12 @@ "Details": [ ] }, + "file-change": { + "Name": "File Change Check", + "Order": 6, + "Details": [ + ] + }, "test": { "Name": "Test", "Order": 100, diff --git a/tools/StaticAnalysis/CollectStaticAnalysisPipelineResult.ps1 b/tools/StaticAnalysis/CollectStaticAnalysisPipelineResult.ps1 index 971e69f5653d..77d7521289df 100644 --- a/tools/StaticAnalysis/CollectStaticAnalysisPipelineResult.ps1 +++ b/tools/StaticAnalysis/CollectStaticAnalysisPipelineResult.ps1 @@ -65,6 +65,10 @@ $Steps = @( @{ StepName = "signature" IssuePath = "$StaticAnalysisOutputDirectory/SignatureIssues.csv" + }, + @{ + StepName = "file-change" + IssuePath = "$StaticAnalysisOutputDirectory/FileChangeIssue.csv" } ) @@ -102,7 +106,8 @@ ForEach ($Step In $Steps) If ($MatchedIssues.Length -Ne 0) { #Region generate table head of each step - If (($StepName -Eq "breaking-change") -Or ($StepName -Eq "help") -Or ($StepName -Eq "signature")) + $NormalSteps = [System.Collections.Generic.HashSet[String]]@("breaking-change", "help", "signature", "file-change") + If ($NormalSteps.Contains($StepName)) { $Content = "|Type|Cmdlet|Description|Remediation|`n|---|---|---|---|`n" } @@ -123,7 +128,7 @@ ForEach ($Step In $Steps) $ErrorTypeEmoji = "⚠️" } #Region generate table content of each step - If (($StepName -Eq "breaking-change") -Or ($StepName -Eq "help") -Or ($StepName -Eq "signature")) + If ($NormalSteps.Contains($StepName)) { $Content += "|$ErrorTypeEmoji|$($Issue.Target)|$($Issue.Description)|$($Issue.Remediation)|`n" } diff --git a/tools/StaticAnalysis/FileChangeAnalyzer/Test-FileChange.ps1 b/tools/StaticAnalysis/FileChangeAnalyzer/Test-FileChange.ps1 new file mode 100644 index 000000000000..4346737f4556 --- /dev/null +++ b/tools/StaticAnalysis/FileChangeAnalyzer/Test-FileChange.ps1 @@ -0,0 +1,73 @@ +# ---------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ---------------------------------------------------------------------------------- + +Param ( +) + +Class FileChangeIssue { + [String]$Module + [Int]$Severity + [String]$Description + [String]$Remediation +} +$ExceptionList = @() + +$ArtifactsFolder = "$PSScriptRoot/../../../artifacts" +$FilesChangedPath = "$ArtifactsFolder/FilesChanged.txt" +$FilesChanged = Get-Content $FilesChangedPath +$ExceptionFilePath = "$ArtifactsFolder/StaticAnalysisResults/FileChangeIssue.csv" +$UpdatedChangeLogs = @{} + +ForEach ($FilePath In ($FilesChanged | Where-Object { $_.EndsWith("ChangeLog.md") })) +{ + $ModuleName = $FilePath.Split("/")[1] + $UpdatedChangeLogs.Add($ModuleName, $FilePath) +} + +ForEach ($FilePath In $FilesChanged) +{ + If ($FilePath.StartsWith("src/")) + { + $ModuleName = $FilePath.Split("/")[1] + + $FileTypeArray = @(".cs", ".psd1", ".csproj", ".ps1xml", ".resx", ".ps1", ".psm1") + $FileType = [System.IO.Path]::GetExtension($FilePath) + If ($FileType -In $FileTypeArray) + { + If (-Not ($UpdatedChangeLogs.ContainsKey($ModuleName))) + { + $ExceptionList += [FileChangeIssue]@{ + Module = "Az.$ModuleName"; + Severity = 2; + Description = "It is required to update `ChangeLog.md` if you want to release a new version for Az.$ModuleName." + Remediation = "Add a changelog record under `Upcoming Release` section with past tense." + } + } + } + + If ([System.IO.Path]::GetFileName($FilePath) -Eq "AssemblyInfo.cs") + { + $ExceptionList += [FileChangeIssue]@{ + Module = "Az.$ModuleName"; + Severity = 2; + Description = "AssemblyInfo.cs will be updated automatically. Please do not update it manually." + Remediation = "Revert AssemblyInfo.cs to its last version." + } + } + } +} + +If ($ExceptionList.Length -Ne 0) +{ + $ExceptionList | Sort-Object -Unique -Property Module,Description | Export-Csv $ExceptionFilePath -NoTypeInformation +} \ No newline at end of file