[StaticAnalysis] CI ExampleAnalysis

.azure-pipelines/util/analyze-steps.yml

@@ -29,9 +29,9 @@ steps:
     packageType: sdk
     version: 3.1.x
 
-- pwsh: 'Install-Module platyPS -Force -Confirm:$false -Scope CurrentUser'
-  displayName: 'Install platyPS'
-
+- pwsh: 'Install-Module "platyPS", "PSScriptAnalyzer" -Force -Confirm:$false -Scope CurrentUser'
+  displayName: 'Install PowerShell Dependencies'
+  
 - task: DotNetCoreCLI@2
   displayName: 'Generate Help'
   inputs:

.ci-config.json

@@ -87,7 +87,7 @@
         "src/{ModuleName}/**/*.md$"
       ],
       "phases": [
-        "build:module",
+        "build:dependent-module",
         "help:module"
       ]
     },

build.proj

@@ -118,7 +118,7 @@
     <Exec Command="dotnet publish $(RepoTools)BuildPackagesTask/Microsoft.Azure.Build.Tasks/Microsoft.Azure.Build.Tasks.csproj -c $(Configuration)" />
 
     <!-- Get all of the files changed in the given pull request -->
-    <FilesChangedTask RepositoryOwner="Azure" RepositoryName="azure-powershell" PullRequestNumber="$(PullRequestNumber)" TargetModule="$(TargetModule)">
+    <FilesChangedTask RepositoryOwner="Azure" RepositoryName="azure-powershell" PullRequestNumber="$(PullRequestNumber)" TargetModule="$(TargetModule)" OutputFile="$(RepoArtifacts)/FilesChanged.txt">
       <Output TaskParameter="FilesChanged" ItemName="FilesChanged" />
     </FilesChangedTask>
 
@@ -263,7 +263,12 @@
     <Exec Command="dotnet $(RepoArtifacts)StaticAnalysis/StaticAnalysis.Netcore.dll -p $(RepoArtifacts)$(Configuration) -r $(StaticAnalysisOutputDirectory) --analyzers help -u -m '%(FilterTaskResult.help)'" />
   </Target>
 
-  <Target Name="StaticAnalysis" DependsOnTargets="StaticAnalysisBreakingChange;StaticAnalysisDependency;StaticAnalysisSignature;StaticAnalysisHelp">
+  <Target Name="StaticAnalysisExample" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build" AfterTargets="StaticAnalysisHelp">
+    <Message Importance="high" Text="Running static analysis for PowerShell examples..." />
+    <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/StaticAnalysis/ExampleAnalyzer/Measure-MarkdownOrScript.ps1 -MarkdownPaths $(RepoArtifacts)/FilesChanged.txt -RulePaths $(RepoTools)/StaticAnalysis/ExampleAnalyzer/AnalyzeRules/*.psm1 -Recurse -AnalyzeScriptsInFile -OutputScriptsInFile -OutputResultsByModule &quot;"/>
+  </Target>
+
+  <Target Name="StaticAnalysis" DependsOnTargets="StaticAnalysisBreakingChange;StaticAnalysisDependency;StaticAnalysisSignature;StaticAnalysisHelp;StaticAnalysisExample">
     <Message Importance="high" Text="Running static analysis..." />
 
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CheckAssemblies.ps1 -BuildConfig $(Configuration) &quot;" />

documentation/Debugging-StaticAnalysis-Errors.md

@@ -8,6 +8,7 @@ Our StaticAnalysis tools help us ensure our modules follow PowerShell guidelines
     - [Breaking Changes](#breaking-changes)
     - [Signature Issues](#signature-issues)
     - [Help Issues](#help-issues)
+    - [Example Issues](#example-issues)
 
 ## How to know if you have a StaticAnalysis Error
 If your build is failing, click on the Jenkins job inside the PR (marked as "Default" within checks).  Then check the Console Output within the Jenkins job.  If you have this error, then you have failed StaticAnalysis:
@@ -34,21 +35,32 @@ If you make a change that could cause a breaking change, it will be listed in `B
 
 _Note_: Sometimes the error listed in the .csv file can be a false positive (for example, if you change a parameter attribute to span all parameter sets rather than individual parameter sets).  Please read the error thoroughly and examine the relevant code before deciding that an error is a false positive, and contact the Azure PowerShell team if you have questions.  If you are releasing a preview module, are releasing during a breaking change release, or have determined that the error is a false positive, please follow these instructions to suppress the errors:
 
-- Download the `BreakingChangeIssues.csv` file from the Jenkins build
+- Download the `BreakingChangeIssues.csv` file from the CI pipeline artifacts
 - Open the file using a text editor (such as VS Code) and copy each of the errors you'd like to suppress
 - Paste each of these errors into the `BreakingChangeIssues.csv` file found in their respective [module folder](../tools/StaticAnalysis/Exceptions) (_e.g._, if a breaking change is being suppressed for Compute, then you would paste the corresponding line(s) in the `tools/StaticAnalysis/Exceptions/Az.Compute/BreakingChangeIssues.csv` file) using the same text editor
-- Push the changes to the .csv file and ensure the errors no longer show up in the `BreakingChangeIssues.csv` file output from the Jenkins build.
+- Push the changes to the .csv file and ensure the errors no longer show up in the `BreakingChangeIssues.csv` file output from the CI pipeline artifacts.
 
 We take breaking changes very seriously, so please be mindful about the violations that you suppress in our repo.
 
 ### Signature Issues
 Signature issues occur when your cmdlets do not follow PowerShell standards.  Please check the [_Cmdlet Best Practices_](https://github.com/Azure/azure-powershell/blob/main/documentation/development-docs/design-guidelines/cmdlet-best-practices.md) and the [_Parameter Best Practices_](https://github.com/Azure/azure-powershell/blob/main/documentation/development-docs/design-guidelines/parameter-best-practices.md) documents to ensure you are following PowerShell guidelines.  Issues with severity 0 or 1 must be addressed, while issues with severity 2 are advisory.  If you have an issue with severity 0 or 1 that has been approved by the Azure PowerShell team, you can suppress them following these steps:
 
-- Download the `SignatureIssues.csv` file from the Jenkins build
+- Download the `SignatureIssues.csv` file from the CI pipeline artifacts
 - Open the file using a text editor (such as VS Code) and copy each of the errors you'd like to suppress
 - Paste each of these errors into the `SignatureIssues.csv` file found in their respective [module folder](../tools/StaticAnalysis/Exceptions) (_e.g.,_ if a signature issue is being suppressed for Sql, then you would paste the corresponding line(s) in the `tools/StaticAnalysis/Exceptions/Az.Sql/SignatureIssues.csv` file) using the same text editor
-- Copy each of the errors you would like to suppress directly from the SignatureIssues.csv file output in the Jenkins build
-- Push the changes to the .csv file and ensure the errors no longer show up in the `SignatureIssues.csv` file output from the Jenkins build.
+- Copy each of the errors you would like to suppress directly from the SignatureIssues.csv file output in the CI pipeline artifacts
+- Push the changes to the .csv file and ensure the errors no longer show up in the `SignatureIssues.csv` file output from the CI pipeline artifacts.
 
 ### Help Issues
 Most help issues that cause StaticAnalysis to fail occur when help has not been added for a particular cmdlet.  If you have not generated help for your new cmdlets, please follow the instructions [here](https://github.com/Azure/azure-powershell/blob/main/documentation/development-docs/help-generation.md). If this is not the issue, follow the steps listed under "Remediation" for each violation listed in HelpIssues.csv.
+
+### Example Issues
+Example issues occur when your changed markdown files in the `help` folder (_e.g.,_ `src/Accounts/Accounts/help`) violate PowerShell language best practices. Please follow the suggestion displayed in "Remediation" entry for each violation listed in `ExampleIssues.csv`. If you have an issue with severity 0 or 1 that has been approved by the Azure PowerShell team, you can suppress them following these steps:
+
+- Download the `ExampleIssues.csv` file from the CI pipeline artifacts
+- Open the file using a text editor (such as VS Code) and copy each of the errors you'd like to suppress
+- Paste each of these errors into the `ExampleIssues.csv` file found in their respective [module folder](../tools/StaticAnalysis/Exceptions) (_e.g.,_ if an example issue is being suppressed for Accounts, then you would paste the corresponding line(s) in the `tools/StaticAnalysis/Exceptions/Az.Accounts/ExampleIssue.csv` file) using the same text editor
+- Copy each of the errors you would like to suppress directly from the ExampleIssues.csv file output in the CI pipeline artifacts
+- Push the changes to the .csv file and ensure the errors no longer show up in the `ExampleIssues.csv` file output from the CI pipeline artifacts.
+
+To better standardize the writing of documents, please also check the warning issues with severity 2 by downloading the `ExampleIssues.csv` file.

documentation/tooling/static-analysis.md

@@ -85,6 +85,8 @@ The dependency analyzer can be found in the [`DependencyAnalyzer`](https://githu
     - The implementation of the `IReportRecord` interface; defines what a missing assembly exception looks like when it's reported in the `MissingAssembly.csv` file that is found in the build artifacts of a CI run, as well as how to compare a new record to a record found in the existing `MissingAssembly.csv` file used for exception suppressions
 - `SharedAssemblyConflict`
     - The implementation of the `IReportRecord` interface; defines what a shared conflict exception looks like when it's reported in the `SharedAssemblyConflict.csv` file that is found in the build artifacts of a CI run, as well as how to compare a new record to a record found in the existing `SharedAssemblyConflict.csv` file used for exception suppressions
+- `ExampleIssue`
+    - The implementation of the `IReportRecord` interface; defines what an example issue exception looks like when it's reported in the `ExampleIssues.csv` file that is found in the build artifacts of a CI run, as well as how to compare a new record to a record found in the existing `ExampleIssues.csv` file used for exception suppressions    
 
 #### Help Analyzer
 

tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/FilesChangedTask.cs

@@ -49,6 +49,11 @@ public class FilesChangedTask : Task
         /// </summary>
         public string TargetModule { get; set; }
 
+        /// <summary>
+        /// Gets or set the OutputFile, store FilesChanged.txt in 'artifacts' folder
+        /// </summary>
+        public string OutputFile { get; set; }
+
         /// <summary>
         /// Gets or sets the files changed produced by the task.
         /// </summary>
@@ -148,10 +153,10 @@ public override bool Execute()
             return true;
         }
 
-        // This method will record the changed files into FilesChanged.txt under root folder for other task to consum.
+        // This method will record the changed files into a text file at `OutputFile` for other task to consum.
         private void SerializeChangedFilesToFile(string[] FilesChanged)
         {
-            File.WriteAllLines("FilesChanged.txt", FilesChanged);
+            File.WriteAllLines(OutputFile, FilesChanged);
         }
     }
 }

tools/PrepareAutorestModule.ps1

@@ -17,7 +17,7 @@
 
 param(
 )
-$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\FilesChanged.txt"
+$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\artifacts\FilesChanged.txt"
 
 $ALL_MODULE = "ALL_MODULE"
 

tools/PrepareForSecurityCheck.ps1

@@ -17,7 +17,7 @@
 
 param(
 )
-$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\FilesChanged.txt"
+$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\artifacts\FilesChanged.txt"
 
 $SecurityTmpFolder = "$PSScriptRoot\..\SecurityTmp"
 New-Item -ItemType Directory -Force -Path $SecurityTmpFolder

tools/StaticAnalysis/ExampleAnalyzer/AnalyzeRules/CommandName.psm1

@@ -0,0 +1,143 @@
+<#
+    .SYNOPSIS
+    Custom rule for command name.
+    .NOTES
+    File: CommandName.psm1
+#>
+
+enum RuleNames {
+    Invalid_Cmdlet
+    Is_Alias
+    Capitalization_Conventions_Violated
+}
+
+<#
+    .SYNOPSIS
+    Returns invaild, alias or unrecognized cmdlets.
+#>
+function Measure-CommandName {
+    [CmdletBinding()]
+    [OutputType([Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord[]])]
+    param(
+        [Parameter(Mandatory)]
+        [ValidateNotNullOrEmpty()]
+        [System.Management.Automation.Language.ScriptBlockAst]
+        $ScriptBlockAst
+    )
+    begin{
+        $modulePath = "$PSScriptRoot\..\..\..\..\artifacts\Debug\Az.*\Az.*.psd1"
+        Get-Item $modulePath | Import-Module -Global
+    }
+    process {
+        $Results = @()
+        $global:CommandParameterPair = @()
+        $global:Ast = $null
+
+        try {
+            [ScriptBlock]$Predicate = {
+                param([System.Management.Automation.Language.Ast]$Ast)
+                $global:Ast = $Ast
+
+                #Find all command in .ps1
+                if ($Ast -is [System.Management.Automation.Language.CommandAst]) {
+                    [System.Management.Automation.Language.CommandAst]$CommandAst = $Ast
+                    # Get wrapper function name by command element
+                    $funcAst = $CommandAst
+                    while($funcAst -isnot [System.Management.Automation.Language.FunctionDefinitionAst] -and $null -ne $funcAst.Parent.Parent.Parent){
+                        $funcAst = $funcAst.Parent
+                    }
+                    $ModuleCmdletExNum = $funcAst.name
+
+                    if ($CommandAst.InvocationOperator -eq "Unknown") {
+                        $CommandName = $CommandAst.CommandElements[0].Extent.Text
+                        $GetCommand = Get-Command $CommandName -ErrorAction SilentlyContinue
+                        if ($null -eq $GetCommand) {
+                            # CommandName is not valid.
+                            $global:CommandParameterPair += @{
+                                CommandName = $CommandName
+                                ParameterName = "<is not valid>"
+                                ModuleCmdletExNum = $ModuleCmdletExNum
+                            }
+                            return $true
+                        }
+                        else {
+                            if ($GetCommand.CommandType -eq "Alias") {
+                                # CommandName is an alias.
+                                $global:CommandParameterPair += @{
+                                    CommandName = $CommandName
+                                    ParameterName = "<is an alias>"
+                                    ModuleCmdletExNum = $ModuleCmdletExNum
+                                }
+                                return $true
+                            }
+                            if ($CommandName -cnotmatch "^([A-Z][a-z]+)+-([A-Z][a-z0-9]*)+$") {
+                                # CommandName doesn't follow the Capitalization Conventions.
+                                $global:CommandParameterPair += @{
+                                    CommandName = $CommandName
+                                    ParameterName = "<doesn't follow the Capitalization Conventions>"
+                                    ModuleCmdletExNum = $ModuleCmdletExNum
+                                }
+                                return $true
+                            }
+                        }
+                    }
+                }
+
+                return $false
+            }
+
+            # Find all false scriptblock
+            [System.Management.Automation.Language.Ast[]]$Asts = $ScriptBlockAst.FindAll($Predicate, $false)
+            for ($i = 0; $i -lt $Asts.Count; $i++) {
+                if ($global:CommandParameterPair[$i].ParameterName -eq "<is not valid>") {
+                    $Message = "$($CommandParameterPair[$i].CommandName) is not a valid command name."
+                    $RuleName = [RuleNames]::Invalid_Cmdlet
+                    $RuleSuppressionID = "5000"
+                    $Remediation = "Check the spell of $($CommandParameterPair[$i].CommandName)."
+                    $Severity = "Error"
+                }
+                if ($global:CommandParameterPair[$i].ParameterName -eq "<is an alias>") {
+                    $Message = "$($CommandParameterPair[$i].CommandName) is an alias of `"$((Get-Alias $CommandParameterPair[$i].CommandName)[0].ResolvedCommandName)`"."
+                    $RuleName = [RuleNames]::Is_Alias
+                    $RuleSuppressionID = "5100"
+                    $Remediation = "Use formal name `"$((Get-Alias $CommandParameterPair[$i].CommandName)[0].ResolvedCommandName)`" of the alias `"$($CommandParameterPair[$i].CommandName)`"."
+                    $Severity = "Warning"
+                }
+                if ($global:CommandParameterPair[$i].ParameterName -eq "<doesn't follow the Capitalization Conventions>") {
+                    $Message = "$($CommandParameterPair[$i].CommandName) doesn't follow the Capitalization Conventions."
+                    $RuleName = [RuleNames]::Capitalization_Conventions_Violated
+                    $RuleSuppressionID = "5101"
+                    $name = $($CommandParameterPair[$i].CommandName)
+                    $textInfo = (Get-Culture).TextInfo
+                    $CorrectName = $textInfo.ToTitleCase(($name -split "-")[0])
+                    $CorrectName += "-Az"
+                    $CorrectName += $textInfo.ToTitleCase(($name -split "Az")[1])
+                    $Remediation = "Check the Capitalization Conventions. Suggest format: $CorrectName"
+                    $Severity = "Warning"
+                }
+                $ModuleCmdletExNum = $($CommandParameterPair[$i].ModuleCmdletExNum)
+                $Result = [Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord]@{
+                    Message = "$ModuleCmdletExNum-@$Message@$Remediation";
+                    Extent = $Asts[$i].Extent;
+                    RuleName = $RuleName;
+                    Severity = $Severity
+                    RuleSuppressionID = $RuleSuppressionID
+                }
+                $Results += $Result
+            }
+            return $Results
+        }
+        catch {
+            $Result = [Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord]@{
+                Message = $_.Exception.Message;
+                Extent = $global:Ast.Extent;
+                RuleName = $PSCmdlet.MyInvocation.InvocationName;
+                Severity = "Error"
+            }
+            $Results += $Result
+            return $Results
+        }
+    }
+}
+
+Export-ModuleMember -Function Measure-*