diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
index 5a3a32ea..238945cb 100644
--- a/eng/common/post-build/publish-using-darc.ps1
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -2,7 +2,6 @@ param(
[Parameter(Mandatory=$true)][int] $BuildId,
[Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
[Parameter(Mandatory=$true)][string] $AzdoToken,
- [Parameter(Mandatory=$true)][string] $MaestroToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
[Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
@@ -31,13 +30,13 @@ try {
}
& $darc add-build-to-channel `
- --id $buildId `
- --publishing-infra-version $PublishingInfraVersion `
- --default-channels `
- --source-branch main `
- --azdev-pat $AzdoToken `
- --bar-uri $MaestroApiEndPoint `
- --password $MaestroToken `
+ --id $buildId `
+ --publishing-infra-version $PublishingInfraVersion `
+ --default-channels `
+ --source-branch main `
+ --azdev-pat "$AzdoToken" `
+ --bar-uri "$MaestroApiEndPoint" `
+ --ci `
@optionalParams
if ($LastExitCode -ne 0) {
diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config
index 3849bdb3..5bfbb02e 100644
--- a/eng/common/sdl/NuGet.config
+++ b/eng/common/sdl/NuGet.config
@@ -5,11 +5,11 @@
-
+
-
+
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
index 4715d75e..81ded5b7 100644
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -6,7 +6,6 @@ Param(
[string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
- [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
# Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
# format.
@@ -75,7 +74,7 @@ try {
}
Exec-BlockVerbosely {
- & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+ & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel
}
$gdnFolder = Join-Path $workingDirectory '.gdn'
@@ -104,7 +103,6 @@ try {
-TargetDirectory $targetDirectory `
-GdnFolder $gdnFolder `
-ToolsList $tools `
- -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
-GuardianLoggerLevel $GuardianLoggerLevel `
-CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
index 3ac1d92b..588ff8e2 100644
--- a/eng/common/sdl/init-sdl.ps1
+++ b/eng/common/sdl/init-sdl.ps1
@@ -3,7 +3,6 @@ Param(
[string] $Repository,
[string] $BranchName='master',
[string] $WorkingDirectory,
- [string] $AzureDevOpsAccessToken,
[string] $GuardianLoggerLevel='Standard'
)
@@ -21,14 +20,7 @@ $ci = $true
# Don't display the console progress UI - it's a huge perf hit
$ProgressPreference = 'SilentlyContinue'
-# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
-$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
-$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
-$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0"
-$zipFile = "$WorkingDirectory/gdn.zip"
-
Add-Type -AssemblyName System.IO.Compression.FileSystem
-$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository
diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1
index 648c5068..7fe603fe 100644
--- a/eng/common/sdl/sdl.ps1
+++ b/eng/common/sdl/sdl.ps1
@@ -4,6 +4,8 @@ function Install-Gdn {
[Parameter(Mandatory=$true)]
[string]$Path,
+ [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json",
+
# If omitted, install the latest version of Guardian, otherwise install that specific version.
[string]$Version
)
@@ -19,7 +21,7 @@ function Install-Gdn {
$ci = $true
. $PSScriptRoot\..\tools.ps1
- $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
+ $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
if ($Version) {
$argumentList += "-Version $Version"
diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml
index 589ac80a..01173288 100644
--- a/eng/common/templates-official/job/publish-build-assets.yml
+++ b/eng/common/templates-official/job/publish-build-assets.yml
@@ -76,13 +76,16 @@ jobs:
- task: NuGetAuthenticate@1
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Build Assets
inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+ arguments: >
+ -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
- /p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:OfficialBuildId=$(Build.BuildNumber)
@@ -137,14 +140,16 @@ jobs:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Using Darc
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
- arguments: -BuildId $(BARBuildId)
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ arguments: -BuildId $(BARBuildId)
-PublishingInfraVersion 3
- -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
- -MaestroToken '$(MaestroApiAccessToken)'
+ -AzdoToken '$(System.AccessToken)'
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml
index f193dfbe..f983033b 100644
--- a/eng/common/templates-official/job/source-build.yml
+++ b/eng/common/templates-official/job/source-build.yml
@@ -31,6 +31,12 @@ parameters:
# container and pool.
platform: {}
+ # If set to true and running on a non-public project,
+ # Internal blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
@@ -62,6 +68,8 @@ jobs:
clean: all
steps:
+ - ${{ if eq(parameters.enableInternalSources, true) }}:
+ - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml
- template: /eng/common/templates-official/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml
index 43ee0c20..60dfb6b2 100644
--- a/eng/common/templates-official/job/source-index-stage1.yml
+++ b/eng/common/templates-official/job/source-index-stage1.yml
@@ -23,7 +23,7 @@ jobs:
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- - template: /eng/common/templates/variables/pool-providers.yml
+ - template: /eng/common/templates-official/variables/pool-providers.yml
${{ if ne(parameters.pool, '') }}:
pool: ${{ parameters.pool }}
@@ -34,7 +34,8 @@ jobs:
demands: ImageOverride -equals windows.vs2019.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $(DncEngInternalBuildPool)
- demands: ImageOverride -equals windows.vs2019.amd64
+ image: windows.vs2022.amd64
+ os: windows
steps:
- ${{ each preStep in parameters.preSteps }}:
@@ -70,16 +71,13 @@ jobs:
scriptType: 'ps'
scriptLocation: 'inlineScript'
inlineScript: |
- echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
- echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
- echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
+ echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
+ echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
+ echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
- script: |
- echo "Client ID: $(ARM_CLIENT_ID)"
- echo "ID Token: $(ARM_ID_TOKEN)"
- echo "Tenant ID: $(ARM_TENANT_ID)"
az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
displayName: "Login to Azure"
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
- displayName: Upload stage1 artifacts to source index
\ No newline at end of file
+ displayName: Upload stage1 artifacts to source index
diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml
index 08e5db9b..5cf6a269 100644
--- a/eng/common/templates-official/jobs/source-build.yml
+++ b/eng/common/templates-official/jobs/source-build.yml
@@ -21,6 +21,12 @@ parameters:
# one job runs on 'defaultManagedPlatform'.
platforms: []
+ # If set to true and running on a non-public project,
+ # Internal nuget and blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
jobs:
- ${{ if ne(parameters.allCompletedJobId, '') }}:
@@ -38,9 +44,11 @@ jobs:
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates-official/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml
index da1f4095..b81b8770 100644
--- a/eng/common/templates-official/post-build/post-build.yml
+++ b/eng/common/templates-official/post-build/post-build.yml
@@ -272,14 +272,16 @@ stages:
- task: NuGetAuthenticate@1
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Using Darc
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
arguments: -BuildId $(BARBuildId)
-PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
- -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
- -MaestroToken '$(MaestroApiAccessToken)'
+ -AzdoToken '$(System.AccessToken)'
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml
new file mode 100644
index 00000000..93a8394a
--- /dev/null
+++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml
@@ -0,0 +1,28 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+ type: string
+ default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+ type: string
+ default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: true
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+ - template: /eng/common/templates-official/steps/get-delegation-sas.yml
+ parameters:
+ federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+ outputVariableName: ${{ parameters.outputVariableName }}
+ expiryInHours: ${{ parameters.expiryInHours }}
+ base64Encode: ${{ parameters.base64Encode }}
+ storageAccount: dotnetbuilds
+ container: internal
+ permissions: rl
diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml
index 07426fde..301d5c59 100644
--- a/eng/common/templates-official/steps/execute-sdl.yml
+++ b/eng/common/templates-official/steps/execute-sdl.yml
@@ -9,8 +9,6 @@ parameters:
steps:
- task: NuGetAuthenticate@1
- inputs:
- nuGetServiceConnections: GuardianConnect
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml
new file mode 100644
index 00000000..c690cc0a
--- /dev/null
+++ b/eng/common/templates-official/steps/get-delegation-sas.yml
@@ -0,0 +1,52 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: false
+- name: storageAccount
+ type: string
+- name: container
+ type: string
+- name: permissions
+ type: string
+ default: 'rl'
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ # Calculate the expiration of the SAS token and convert to UTC
+ $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+ # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+ # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+ $sas = ""
+ do {
+ $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+ } while($sas.IndexOf('/') -ne -1)
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+
+ if ('${{ parameters.base64Encode }}' -eq 'true') {
+ $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+ }
+
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"
diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml
new file mode 100644
index 00000000..55e33bd3
--- /dev/null
+++ b/eng/common/templates-official/steps/get-federated-access-token.yml
@@ -0,0 +1,40 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+- name: stepName
+ type: string
+ default: 'getFederatedAccessToken'
+- name: condition
+ type: string
+ default: ''
+# Resource to get a token for. Common values include:
+# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
+# - 'https://storage.azure.com/' for storage
+# Defaults to Azure DevOps
+- name: resource
+ type: string
+ default: '499b84ac-1321-427f-aa17-267ca6975798'
+- name: isStepOutputVariable
+ type: boolean
+ default: false
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Getting federated access token for feeds'
+ name: ${{ parameters.stepName }}
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
+ exit 1
+ }
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken"
\ No newline at end of file
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index 8ec0151d..cc2b346b 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -74,13 +74,16 @@ jobs:
- task: NuGetAuthenticate@1
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Build Assets
inputs:
- filePath: eng\common\sdk-task.ps1
- arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1
+ arguments: >
+ -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
- /p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro.dot.net
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:OfficialBuildId=$(Build.BuildNumber)
@@ -133,14 +136,16 @@ jobs:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Using Darc
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
- arguments: -BuildId $(BARBuildId)
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ arguments: -BuildId $(BARBuildId)
-PublishingInfraVersion 3
- -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
- -MaestroToken '$(MaestroApiAccessToken)'
+ -AzdoToken '$(System.AccessToken)'
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
index 8a3deef2..c0ff472b 100644
--- a/eng/common/templates/job/source-build.yml
+++ b/eng/common/templates/job/source-build.yml
@@ -31,6 +31,12 @@ parameters:
# container and pool.
platform: {}
+ # If set to true and running on a non-public project,
+ # Internal blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
@@ -61,6 +67,8 @@ jobs:
clean: all
steps:
+ - ${{ if eq(parameters.enableInternalSources, true) }}:
+ - template: /eng/common/templates/steps/enable-internal-runtimes.yml
- template: /eng/common/templates/steps/source-build.yml
parameters:
platform: ${{ parameters.platform }}
diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml
index 43ee0c20..0b6bb89d 100644
--- a/eng/common/templates/job/source-index-stage1.yml
+++ b/eng/common/templates/job/source-index-stage1.yml
@@ -70,16 +70,13 @@ jobs:
scriptType: 'ps'
scriptLocation: 'inlineScript'
inlineScript: |
- echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
- echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
- echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
+ echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId"
+ echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken"
+ echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId"
- script: |
- echo "Client ID: $(ARM_CLIENT_ID)"
- echo "ID Token: $(ARM_ID_TOKEN)"
- echo "Tenant ID: $(ARM_TENANT_ID)"
az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
displayName: "Login to Azure"
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1
- displayName: Upload stage1 artifacts to source index
\ No newline at end of file
+ displayName: Upload stage1 artifacts to source index
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
index a15b07eb..5f46bfa8 100644
--- a/eng/common/templates/jobs/source-build.yml
+++ b/eng/common/templates/jobs/source-build.yml
@@ -21,6 +21,12 @@ parameters:
# one job runs on 'defaultManagedPlatform'.
platforms: []
+ # If set to true and running on a non-public project,
+ # Internal nuget and blob storage locations will be enabled.
+ # This is not enabled by default because many repositories do not need internal sources
+ # and do not need to have the required service connections approved in the pipeline.
+ enableInternalSources: false
+
jobs:
- ${{ if ne(parameters.allCompletedJobId, '') }}:
@@ -38,9 +44,11 @@ jobs:
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ parameters.defaultManagedPlatform }}
+ enableInternalSources: ${{ parameters.enableInternalSources }}
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index aba44a25..c3b6a301 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -268,14 +268,16 @@ stages:
- task: NuGetAuthenticate@1
- - task: PowerShell@2
+ - task: AzureCLI@2
displayName: Publish Using Darc
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: ps
+ scriptLocation: scriptPath
+ scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
arguments: -BuildId $(BARBuildId)
-PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
- -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
- -MaestroToken '$(MaestroApiAccessToken)'
+ -AzdoToken '$(System.AccessToken)'
-WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml
index 0c87f149..64b9abc6 100644
--- a/eng/common/templates/post-build/setup-maestro-vars.yml
+++ b/eng/common/templates/post-build/setup-maestro-vars.yml
@@ -11,13 +11,14 @@ steps:
artifactName: ReleaseConfigs
checkDownloadedFiles: true
- - task: PowerShell@2
+ - task: AzureCLI@2
name: setReleaseVars
displayName: Set Release Configs Vars
inputs:
- targetType: inline
- pwsh: true
- script: |
+ azureSubscription: "Darc: Maestro Production"
+ scriptType: pscore
+ scriptLocation: inlineScript
+ inlineScript: |
try {
if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') {
$Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
@@ -31,15 +32,16 @@ steps:
$AzureDevOpsBuildId = $Env:Build_BuildId
}
else {
- $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
+ . $(Build.SourcesDirectory)\eng\common\tools.ps1
+ $darc = Get-Darc
+ $buildInfo = & $darc get-build `
+ --id ${{ parameters.BARBuildId }} `
+ --extended `
+ --output-format json `
+ --ci `
+ | convertFrom-Json
- $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
- $apiHeaders.Add('Accept', 'application/json')
- $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
-
- $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
-
- $BarId = $Env:BARBuildId
+ $BarId = ${{ parameters.BARBuildId }}
$Channels = $Env:PromoteToMaestroChannels -split ","
$Channels = $Channels -join "]["
$Channels = "[$Channels]"
@@ -65,6 +67,4 @@ steps:
exit 1
}
env:
- MAESTRO_API_TOKEN: $(MaestroApiAccessToken)
- BARBuildId: ${{ parameters.BARBuildId }}
PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }}
diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml
new file mode 100644
index 00000000..54dc9416
--- /dev/null
+++ b/eng/common/templates/steps/enable-internal-runtimes.yml
@@ -0,0 +1,28 @@
+# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64'
+# variable with the base64-encoded SAS token, by default
+
+parameters:
+- name: federatedServiceConnection
+ type: string
+ default: 'dotnetbuilds-internal-read'
+- name: outputVariableName
+ type: string
+ default: 'dotnetbuilds-internal-container-read-token-base64'
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: true
+
+steps:
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+ - template: /eng/common/templates/steps/get-delegation-sas.yml
+ parameters:
+ federatedServiceConnection: ${{ parameters.federatedServiceConnection }}
+ outputVariableName: ${{ parameters.outputVariableName }}
+ expiryInHours: ${{ parameters.expiryInHours }}
+ base64Encode: ${{ parameters.base64Encode }}
+ storageAccount: dotnetbuilds
+ container: internal
+ permissions: rl
diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml
index 07426fde..fe0ebf8c 100644
--- a/eng/common/templates/steps/execute-sdl.yml
+++ b/eng/common/templates/steps/execute-sdl.yml
@@ -9,8 +9,6 @@ parameters:
steps:
- task: NuGetAuthenticate@1
- inputs:
- nuGetServiceConnections: GuardianConnect
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
@@ -36,16 +34,19 @@ steps:
displayName: Execute SDL (Overridden)
continueOnError: ${{ parameters.sdlContinueOnError }}
condition: ${{ parameters.condition }}
+ env:
+ GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken)
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }}
-GuardianCliLocation $(GuardianCliLocation)
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
- -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
displayName: Execute SDL
continueOnError: ${{ parameters.sdlContinueOnError }}
condition: ${{ parameters.condition }}
+ env:
+ GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken)
- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
# We want to publish the Guardian results and configuration for easy diagnosis. However, the
diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml
new file mode 100644
index 00000000..c690cc0a
--- /dev/null
+++ b/eng/common/templates/steps/get-delegation-sas.yml
@@ -0,0 +1,52 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+- name: expiryInHours
+ type: number
+ default: 1
+- name: base64Encode
+ type: boolean
+ default: false
+- name: storageAccount
+ type: string
+- name: container
+ type: string
+- name: permissions
+ type: string
+ default: 'rl'
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}'
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ # Calculate the expiration of the SAS token and convert to UTC
+ $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
+
+ # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads
+ # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484
+ $sas = ""
+ do {
+ $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+ } while($sas.IndexOf('/') -ne -1)
+
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to generate SAS token."
+ exit 1
+ }
+
+ if ('${{ parameters.base64Encode }}' -eq 'true') {
+ $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas))
+ }
+
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas"
diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml
new file mode 100644
index 00000000..55e33bd3
--- /dev/null
+++ b/eng/common/templates/steps/get-federated-access-token.yml
@@ -0,0 +1,40 @@
+parameters:
+- name: federatedServiceConnection
+ type: string
+- name: outputVariableName
+ type: string
+- name: stepName
+ type: string
+ default: 'getFederatedAccessToken'
+- name: condition
+ type: string
+ default: ''
+# Resource to get a token for. Common values include:
+# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
+# - 'https://storage.azure.com/' for storage
+# Defaults to Azure DevOps
+- name: resource
+ type: string
+ default: '499b84ac-1321-427f-aa17-267ca6975798'
+- name: isStepOutputVariable
+ type: boolean
+ default: false
+
+steps:
+- task: AzureCLI@2
+ displayName: 'Getting federated access token for feeds'
+ name: ${{ parameters.stepName }}
+ ${{ if ne(parameters.condition, '') }}:
+ condition: ${{ parameters.condition }}
+ inputs:
+ azureSubscription: ${{ parameters.federatedServiceConnection }}
+ scriptType: 'pscore'
+ scriptLocation: 'inlineScript'
+ inlineScript: |
+ $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv
+ if ($LASTEXITCODE -ne 0) {
+ Write-Error "Failed to get access token for resource '${{ parameters.resource }}'"
+ exit 1
+ }
+ Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
+ Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken"
\ No newline at end of file
diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml
index 32c01ef0..6abbcb33 100644
--- a/eng/common/templates/steps/telemetry-start.yml
+++ b/eng/common/templates/steps/telemetry-start.yml
@@ -8,7 +8,7 @@ parameters:
steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}:
- - task: AzureKeyVault@1
+ - task: AzureKeyVault@2
inputs:
azureSubscription: 'HelixProd_KeyVault'
KeyVaultName: HelixProdKV
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index eb188cfd..a2dedaa5 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -892,7 +892,7 @@ function IsWindowsPlatform() {
}
function Get-Darc($version) {
- $darcPath = "$TempDir\darc\$(New-Guid)"
+ $darcPath = "$TempDir\darc\$([guid]::NewGuid())"
if ($version -ne $null) {
& $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
} else {
diff --git a/global.json b/global.json
index af3a3a5d..c893835f 100644
--- a/global.json
+++ b/global.json
@@ -1,16 +1,15 @@
{
"tools": {
- "dotnet": "8.0.302",
+ "dotnet": "8.0.403",
"runtimes": {
"dotnet/x64": [
- "8.0.6"
+ "8.0.10"
]
- },
- "xcopy-msbuild": "17.8.5"
+ }
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23463.1",
- "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23463.1",
+ "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24359.3",
+ "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24359.3",
"Microsoft.Build.NoTargets": "3.7.0"
}
}