Merge pull request #1475 from dotnet/darc-release/6.x-d8c50cd4-684b-4…

…86b-bae0-b15426dafaec

[release/6.x] Update dependencies from dotnet/arcade

Directory.Packages.props

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <!-- In order tests against the same version of NuGet as the SDK. We have to set this to match. -->
-    <NuGetVersion>6.0.0-rc.278</NuGetVersion>
+    <NuGetVersion>6.0.0</NuGetVersion>
   </PropertyGroup>
 
   <ItemGroup>
@@ -36,4 +36,4 @@
     <PackageVersion Include="System.CommandLine" Version="$(SystemCommandLineVersion)" />
     <PackageVersion Include="System.CommandLine.Rendering" Version="$(SystemCommandLineRenderingVersion)" />
   </ItemGroup>
-</Project>
+</Project>

eng/Version.Details.xml

@@ -15,9 +15,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21614.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.22122.7">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>47f8ea1d7ef3efd5d4fa93ccb79ccccf4182095e</Sha>
+      <Sha>7215d8265a7fbcd022eb72ff7a6e2048444c985f</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/generate-sbom-prep.ps1

@@ -0,0 +1,19 @@
+Param(
+    [Parameter(Mandatory=$true)][string] $ManifestDirPath    # Manifest directory where sbom will be placed
+)
+
+Write-Host "Creating dir $ManifestDirPath"
+# create directory for sbom manifest to be placed
+if (!(Test-Path -path $ManifestDirPath))
+{
+  New-Item -ItemType Directory -path $ManifestDirPath
+  Write-Host "Successfully created directory $ManifestDirPath"
+}
+else{
+  Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
+}
+
+Write-Host "Updating artifact name"
+$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_'
+Write-Host "Artifact name $artifact_name"
+Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name"

eng/common/generate-sbom-prep.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+manifest_dir=$1
+
+if [ ! -d "$manifest_dir" ] ; then
+  mkdir -p "$manifest_dir"
+  echo "Sbom directory created." $manifest_dir
+else
+  Write-PipelineTelemetryError -category 'Build'  "Unable to create sbom folder."
+fi
+
+artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM"
+echo "Artifact name before : "$artifact_name
+# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts.
+safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}"
+echo "Artifact name after : "$safe_artifact_name
+export ARTIFACT_NAME=$safe_artifact_name
+echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name"
+
+exit 0

eng/common/post-build/publish-using-darc.ps1

@@ -5,13 +5,8 @@ param(
   [Parameter(Mandatory=$true)][string] $MaestroToken,
   [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
-  [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
-  [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
-  [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
-  [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
-  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
-  [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
+  [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters
 )
 
 try {
@@ -35,27 +30,6 @@ try {
     $optionalParams.Add("--no-wait") | Out-Null
   }
 
-  if ("false" -ne $PublishInstallersAndChecksums) {
-    $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
-  }
-
-  if ("true" -eq $EnableNugetValidation) {
-    $optionalParams.Add("--validate-nuget") | Out-Null
-  }
-
-  if ("true" -eq $EnableSourceLinkValidation) {
-    $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null
-  }
-
-  if ("true" -eq $EnableSigningValidation) {
-    $optionalParams.Add("--validate-signingchecksums") | Out-Null
-
-    if ("" -ne $SigningValidationAdditionalParameters) {
-      $optionalParams.Add("--signing-validation-parameters") | Out-Null
-      $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null
-    }
-  }
-
   & $darc add-build-to-channel `
   --id $buildId `
   --publishing-infra-version $PublishingInfraVersion `

eng/common/sdl/configure-sdl-tool.ps1

@@ -15,7 +15,9 @@ Param(
   # Optional: Additional params to add to any tool using CredScan.
   [string[]] $CrScanAdditionalRunConfigParams,
   # Optional: Additional params to add to any tool using PoliCheck.
-  [string[]] $PoliCheckAdditionalRunConfigParams
+  [string[]] $PoliCheckAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using CodeQL/Semmle.
+  [string[]] $CodeQLAdditionalRunConfigParams
 )
 
 $ErrorActionPreference = 'Stop'
@@ -78,6 +80,11 @@ try {
         $tool.Args += "Target < $TargetDirectory"
       }
       $tool.Args += $PoliCheckAdditionalRunConfigParams
+    } elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') {
+      if ($targetDirectory) {
+        $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+      }
+      $tool.Args += $CodeQLAdditionalRunConfigParams
     }
 
     # Create variable pointing to the args array directly so we can use splat syntax later.

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -34,6 +34,7 @@ Param(
   [string] $GuardianLoggerLevel='Standard',                                                      # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
   [string[]] $CrScanAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
   [string[]] $PoliCheckAdditionalRunConfigParams,                                                # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+  [string[]] $CodeQLAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
   [bool] $BreakOnFailure=$False                                                                  # Optional: Fail the build if there were errors during the run
 )
 
@@ -105,7 +106,8 @@ try {
           -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
           -GuardianLoggerLevel $GuardianLoggerLevel `
           -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
+          -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams
         if ($BreakOnFailure) {
           Exit-IfNZEC "Sdl"
         }

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.53.3"/>
+  <package id="Microsoft.Guardian.Cli" version="0.110.1"/>
 </packages>

eng/common/templates/job/execute-sdl.yml

@@ -29,14 +29,6 @@ parameters:
   # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
   # not pipeline artifacts, so doesn't affect the use of this parameter.
   pipelineArtifactNames: []
-  # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be
-  # downloaded from. By default, uses runtime expressions to decide based on the variables set by
-  # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are
-  # running without Maestro++/BAR involved, or to download artifacts from a specific existing build
-  # to iterate quickly on SDL changes.
-  AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
-  AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
-  AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
 
 jobs:
 - job: Run_SDL
@@ -54,21 +46,26 @@ jobs:
     # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
     # sync with the packages.config file.
     - name: DefaultGuardianVersion
-      value: 0.53.3
+      value: 0.110.1
     - name: GuardianVersion
       value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
     - name: GuardianPackagesConfigFile
       value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
   pool:
-    # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
-    ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
-      vmImage: windows-2019
-    ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
-      vmImage: windows-2019
+    # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+    ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+      name: VSEngSS-MicroBuild2022-1ES
+      demands: Cmd
+    # If it's not devdiv, it's dnceng
+    ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+      name: NetCore1ESPool-Internal
+      demands: ImageOverride -equals Build.Server.Amd64.VS2019
   steps:
   - checkout: self
     clean: true
 
+  - template: /eng/common/templates/post-build/setup-maestro-vars.yml
+
   - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
     - ${{ if ne(parameters.artifactNames, '') }}:
       - ${{ each artifactName in parameters.artifactNames }}:

eng/common/templates/job/job.yml

@@ -24,12 +24,17 @@ parameters:
   enablePublishBuildAssets: false
   enablePublishTestResults: false
   enablePublishUsingPipelines: false
+  disableComponentGovernance: false
   mergeTestResults: false
   testRunTitle: ''
   testResultsFormat: ''
   name: ''
   preSteps: []
   runAsPublic: false
+# Sbom related params
+  enableSbom: true
+  PackageVersion: 6.0.0
+  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
 
 jobs:
 - job: ${{ parameters.name }}
@@ -136,6 +141,10 @@ jobs:
         richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
       continueOnError: true
 
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}:
+      - task: ComponentGovernanceComponentDetection@0
+        continueOnError: true
+
   - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: MicroBuildCleanup@1
@@ -242,3 +251,9 @@ jobs:
         ArtifactName: AssetManifests
       continueOnError: ${{ parameters.continueOnError }}
       condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
+    - template: /eng/common/templates/steps/generate-sbom.yml
+      parameters:
+        PackageVersion: ${{ parameters.packageVersion}}
+        BuildDropPath: ${{ parameters.buildDropPath }}