From 2c8d95c1a2650c5559c33f6e4c6c343629ecd0e9 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 08:42:26 -0700 Subject: [PATCH 1/9] Changes to scripts to deal with PATs and AccessTokens --- eng/common/scripts/Add-RetentionLease.ps1 | 13 ++-- eng/common/scripts/Invoke-DevOpsAPI.ps1 | 74 ++++++++++++++++------- eng/common/scripts/Queue-Pipeline.ps1 | 14 +++-- eng/pipelines/pipeline-queue-test.yml | 57 +++++++++++++++++ 4 files changed, 128 insertions(+), 30 deletions(-) create mode 100644 eng/pipelines/pipeline-queue-test.yml diff --git a/eng/common/scripts/Add-RetentionLease.ps1 b/eng/common/scripts/Add-RetentionLease.ps1 index ae7b80119c2..6f8799370c2 100644 --- a/eng/common/scripts/Add-RetentionLease.ps1 +++ b/eng/common/scripts/Add-RetentionLease.ps1 @@ -19,14 +19,19 @@ param( [string]$OwnerId = "azure-sdk-pipeline-automation", [Parameter(Mandatory = $false)] - [string]$AccessToken = $env:DEVOPS_PAT + [string]$AuthToken = $env:DEVOPS_PAT, + + [Parameter(Mandatory = $false)] + [string]$AccessToken=$null ) Set-StrictMode -Version 3 . (Join-Path $PSScriptRoot common.ps1) -$encodedAuthToken = Get-Base64EncodedToken $AccessToken +if (![string]::IsNullOrWhiteSpace($AuthToken)) { + $encodedAuthToken = Get-Base64EncodedToken $AuthToken +} LogDebug "Checking for existing leases on run: $RunId" $existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedAuthToken $encodedAuthToken @@ -36,11 +41,11 @@ if ($existingLeases.count -ne 0) { foreach ($lease in $existingLeases.value) { LogDebug "Deleting lease: $($lease.leaseId)" - Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken + Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AccessToken } } LogDebug "Creating new lease on run: $RunId" -$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken +$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AccessToken LogDebug "Lease ID is: $($lease.value.leaseId)" \ No newline at end of file diff --git a/eng/common/scripts/Invoke-DevOpsAPI.ps1 b/eng/common/scripts/Invoke-DevOpsAPI.ps1 index c0fcd360c0a..de9b513e41f 100644 --- a/eng/common/scripts/Invoke-DevOpsAPI.ps1 +++ b/eng/common/scripts/Invoke-DevOpsAPI.ps1 @@ -16,9 +16,28 @@ function Get-Base64EncodedToken([string]$AuthToken) return $encodedAuthToken } -function Get-DevOpsApiHeaders ($Base64EncodedToken) { - $headers = @{ - Authorization = "Basic $Base64EncodedToken" +# The Base64EncodedToken would be from a PAT that was passed in and the header requires Basic authorization +# The AccessToken would be the querying the Azure resource with the following command: +# az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv +# The header for an AccessToken requires Bearer authorization +function Get-DevOpsApiHeaders ($Base64EncodedToken, $AccessToken) { + $headers = $null + if (![string]::IsNullOrWhiteSpace($Base64EncodedToken) -and + ![string]::IsNullOrWhiteSpace($AccessToken)) { + LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because Base64EncodedToken and AccessToken are both set and only one should be." + exit 1 + } + if (![string]::IsNullOrWhiteSpace($Base64EncodedToken)) { + $headers = @{ + Authorization = "Basic $Base64EncodedToken" + } + } elseif (![string]::IsNullOrWhiteSpace($AccessToken)) { + $headers = @{ + Authorization = "Bearer $AccessToken" + } + } else { + LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because neither Base64EncodedToken nor AccessToken are set." + exit 1 } return $headers } @@ -30,9 +49,8 @@ function Start-DevOpsBuild { $SourceBranch, [Parameter(Mandatory = $true)] $DefinitionId, - [ValidateNotNullOrEmpty()] - [Parameter(Mandatory = $true)] - $Base64EncodedAuthToken, + $Base64EncodedAuthToken=$null, + $AccessToken=$null, [Parameter(Mandatory = $false)] [string]$BuildParametersJson ) @@ -45,11 +63,13 @@ function Start-DevOpsBuild { parameters = $BuildParametersJson } + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + return Invoke-RestMethod ` -Method POST ` -Body ($parameters | ConvertTo-Json) ` -Uri $uri ` - -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) ` + -Headers $headers ` -MaximumRetryCount 3 ` -ContentType "application/json" } @@ -62,9 +82,8 @@ function Update-DevOpsBuild { [Parameter(Mandatory = $true)] $BuildId, $Status, # pass canceling to cancel build - [ValidateNotNullOrEmpty()] - [Parameter(Mandatory = $true)] - $Base64EncodedAuthToken + $Base64EncodedAuthToken, + $AccessToken ) $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project, "build", "builds/$BuildId", "" @@ -72,11 +91,13 @@ function Update-DevOpsBuild { if ($Status) { $parameters["status"] = $Status} + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + return Invoke-RestMethod ` -Method PATCH ` -Body ($parameters | ConvertTo-Json) ` -Uri $uri ` - -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) ` + -Headers $headers ` -MaximumRetryCount 3 ` -ContentType "application/json" } @@ -88,9 +109,8 @@ function Get-DevOpsBuilds { $BranchName, # Should start with 'refs/heads/' $Definitions, # Comma seperated string of definition IDs $StatusFilter, # Comma seperated string 'cancelling, completed, inProgress, notStarted' - [ValidateNotNullOrEmpty()] - [Parameter(Mandatory = $true)] - $Base64EncodedAuthToken + $Base64EncodedAuthToken, + $AccessToken ) $query = "" @@ -100,10 +120,12 @@ function Get-DevOpsBuilds { if ($StatusFilter) { $query += "statusFilter=$StatusFilter&" } $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project , "build" , "builds", $query + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + return Invoke-RestMethod ` -Method GET ` -Uri $uri ` - -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) ` + -Headers $headers ` -MaximumRetryCount 3 } @@ -112,15 +134,18 @@ function Delete-RetentionLease { $Organization, $Project, $LeaseId, - $Base64EncodedAuthToken + $Base64EncodedAuthToken, + $AccessToken ) $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ids=$LeaseId&api-version=6.0-preview.1" + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + return Invoke-RestMethod ` -Method DELETE ` -Uri $uri ` - -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) ` + -Headers $headers ` -MaximumRetryCount 3 } @@ -131,15 +156,18 @@ function Get-RetentionLeases { $DefinitionId, $RunId, $OwnerId, - $Base64EncodedAuthToken + $Base64EncodedAuthToken, + $AccessToken ) $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ownerId=$OwnerId&definitionId=$DefinitionId&runId=$RunId&api-version=6.0-preview.1" + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + return Invoke-RestMethod ` -Method GET ` -Uri $uri ` - -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) ` + -Headers $headers ` -MaximumRetryCount 3 } @@ -151,7 +179,8 @@ function Add-RetentionLease { $RunId, $OwnerId, $DaysValid, - $Base64EncodedAuthToken + $Base64EncodedAuthToken, + $AccessToken ) $parameter = @{} @@ -165,12 +194,13 @@ function Add-RetentionLease { $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?api-version=6.0-preview.1" + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + return Invoke-RestMethod ` -Method POST ` -Body "[$body]" ` -Uri $uri ` - -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) ` + -Headers $headers ` -MaximumRetryCount 3 ` -ContentType "application/json" - } diff --git a/eng/common/scripts/Queue-Pipeline.ps1 b/eng/common/scripts/Queue-Pipeline.ps1 index 281bc2f9a71..fb7eeba0f7f 100644 --- a/eng/common/scripts/Queue-Pipeline.ps1 +++ b/eng/common/scripts/Queue-Pipeline.ps1 @@ -58,10 +58,13 @@ param( [string]$VsoQueuedPipelines, # Already base 64 encoded authentication token - [string]$Base64EncodedAuthToken, + [string]$Base64EncodedAuthToken=$null, - # Unencoded authentication token - [string]$AuthToken, + # Unencoded authentication token from a PAT + [string]$AuthToken=$null, + + # Temp access token from the logged in az cli user for azure devops resource + [string]$AccessToken=$null, [Parameter(Mandatory = $false)] [string]$BuildParametersJson @@ -71,7 +74,9 @@ param( if (!$Base64EncodedAuthToken) { - $Base64EncodedAuthToken = Get-Base64EncodedToken $AuthToken + if (![string]::IsNullOrWhiteSpace($AuthToken)) { + $Base64EncodedAuthToken = Get-Base64EncodedToken $AuthToken + } } # Skip if SourceBranch is empty because it we cannot generate a target branch @@ -105,6 +110,7 @@ try { -SourceBranch $SourceBranch ` -DefinitionId $DefinitionId ` -Base64EncodedAuthToken $Base64EncodedAuthToken ` + -AccessToken $AccessToken ` -BuildParametersJson $BuildParametersJson } catch { diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml new file mode 100644 index 00000000000..d80fac52ee7 --- /dev/null +++ b/eng/pipelines/pipeline-queue-test.yml @@ -0,0 +1,57 @@ +trigger: none + +pr: none + +jobs: + - job: Run + pool: + name: azsdk-pool-mms-ubuntu-2204-general + vmImage: ubuntu-22.04 + variables: + ToolsCODEOWNERSLinterId: 6597 + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - task: AzureCLI@2 + displayName: Test Authenticate to OpenSource API and queue pipeline + inputs: + azureSubscription: opensource-api-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + /eng/common/scripts/Queue-Pipeline.ps1 ` + -Organization "azure-sdk" ` + -Project "internal" ` + -DefinitionId "$(ToolsCODEOWNERSLinterId)" ` + -AccessToken $accessToken ` + + # This task is going to become obsolete once the PATs go away + # the queueing PAT will be gone first but there's another PAT + # for queuing docs and this task is just testing the pipeline + # scripts. This task will need to be commented out or removed. + - task: PowerShell@2 + displayName: Test Queue Pipeline with PAT + inputs: + pwsh: true + filePath: eng/common/scripts/Queue-Pipeline.ps1 + arguments: > + -Organization "azure-sdk" + -Project "internal" + -DefinitionId "$(ToolsCODEOWNERSLinterId)" + -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)" + + - task: PowerShell@2 + displayName: Test Retain pipeline run + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Add-RetentionLease.ps1 + arguments: > + -Organization azure-sdk + -Project $(System.TeamProject) + -DefinitionId $(System.DefinitionId) + -RunId $(Build.BuildId) + -DaysValid 7 + -AccessToken $env:SYSTEM_ACCESSTOKEN + -Debug From 3e35349b366c87ca4291667578a71f3f5de59e2c Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 09:09:18 -0700 Subject: [PATCH 2/9] Remove trailing backtick from the last line of the inline script --- eng/pipelines/pipeline-queue-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml index d80fac52ee7..eb50f94c0a4 100644 --- a/eng/pipelines/pipeline-queue-test.yml +++ b/eng/pipelines/pipeline-queue-test.yml @@ -23,7 +23,7 @@ jobs: -Organization "azure-sdk" ` -Project "internal" ` -DefinitionId "$(ToolsCODEOWNERSLinterId)" ` - -AccessToken $accessToken ` + -AccessToken $accessToken # This task is going to become obsolete once the PATs go away # the queueing PAT will be gone first but there's another PAT From 25a4748968de58997748f56f3a1bf22b77dac19a Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 09:13:54 -0700 Subject: [PATCH 3/9] fix path --- eng/pipelines/pipeline-queue-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml index eb50f94c0a4..ced13d17d6a 100644 --- a/eng/pipelines/pipeline-queue-test.yml +++ b/eng/pipelines/pipeline-queue-test.yml @@ -19,7 +19,7 @@ jobs: scriptLocation: inlineScript inlineScript: | $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv - /eng/common/scripts/Queue-Pipeline.ps1 ` + eng/common/scripts/Queue-Pipeline.ps1 ` -Organization "azure-sdk" ` -Project "internal" ` -DefinitionId "$(ToolsCODEOWNERSLinterId)" ` From 09ff9bb7a24f48d820919a1e612bb3803040a4e8 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 09:20:00 -0700 Subject: [PATCH 4/9] the linter pipeline is public, not internal --- eng/pipelines/pipeline-queue-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml index ced13d17d6a..37429e5ac25 100644 --- a/eng/pipelines/pipeline-queue-test.yml +++ b/eng/pipelines/pipeline-queue-test.yml @@ -21,7 +21,7 @@ jobs: $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv eng/common/scripts/Queue-Pipeline.ps1 ` -Organization "azure-sdk" ` - -Project "internal" ` + -Project "public" ` -DefinitionId "$(ToolsCODEOWNERSLinterId)" ` -AccessToken $accessToken @@ -36,7 +36,7 @@ jobs: filePath: eng/common/scripts/Queue-Pipeline.ps1 arguments: > -Organization "azure-sdk" - -Project "internal" + -Project "public" -DefinitionId "$(ToolsCODEOWNERSLinterId)" -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)" From 603e46b57a1a686abdd91fae9b8cff408d6875ed Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 09:38:39 -0700 Subject: [PATCH 5/9] swap access and auth for add-retention-lease --- eng/common/scripts/Add-RetentionLease.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Add-RetentionLease.ps1 b/eng/common/scripts/Add-RetentionLease.ps1 index 6f8799370c2..40ed3e0fe4c 100644 --- a/eng/common/scripts/Add-RetentionLease.ps1 +++ b/eng/common/scripts/Add-RetentionLease.ps1 @@ -19,18 +19,18 @@ param( [string]$OwnerId = "azure-sdk-pipeline-automation", [Parameter(Mandatory = $false)] - [string]$AuthToken = $env:DEVOPS_PAT, + [string]$AccessToken = $env:DEVOPS_PAT, [Parameter(Mandatory = $false)] - [string]$AccessToken=$null + [string]$AuthToken=$null ) Set-StrictMode -Version 3 . (Join-Path $PSScriptRoot common.ps1) -if (![string]::IsNullOrWhiteSpace($AuthToken)) { - $encodedAuthToken = Get-Base64EncodedToken $AuthToken +if (![string]::IsNullOrWhiteSpace($AccessToken)) { + $encodedAuthToken = Get-Base64EncodedToken $AccessToken } LogDebug "Checking for existing leases on run: $RunId" @@ -41,11 +41,11 @@ if ($existingLeases.count -ne 0) { foreach ($lease in $existingLeases.value) { LogDebug "Deleting lease: $($lease.leaseId)" - Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AccessToken + Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AuthToken } } LogDebug "Creating new lease on run: $RunId" -$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AccessToken +$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AuthToken LogDebug "Lease ID is: $($lease.value.leaseId)" \ No newline at end of file From 3b05e52f19c457bf5b25cd4e2cdfa260fbb55b56 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 10:40:44 -0700 Subject: [PATCH 6/9] comment out the task that queues with the PAT --- eng/pipelines/pipeline-queue-test.yml | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml index 37429e5ac25..09f0ca140eb 100644 --- a/eng/pipelines/pipeline-queue-test.yml +++ b/eng/pipelines/pipeline-queue-test.yml @@ -25,20 +25,20 @@ jobs: -DefinitionId "$(ToolsCODEOWNERSLinterId)" ` -AccessToken $accessToken - # This task is going to become obsolete once the PATs go away - # the queueing PAT will be gone first but there's another PAT - # for queuing docs and this task is just testing the pipeline - # scripts. This task will need to be commented out or removed. - - task: PowerShell@2 - displayName: Test Queue Pipeline with PAT - inputs: - pwsh: true - filePath: eng/common/scripts/Queue-Pipeline.ps1 - arguments: > - -Organization "azure-sdk" - -Project "public" - -DefinitionId "$(ToolsCODEOWNERSLinterId)" - -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)" + # # This task is going to become obsolete once the PATs go away + # # the queueing PAT will be gone first but there's another PAT + # # for queuing docs and this task is just testing the pipeline + # # scripts. This task will need to be commented out or removed. + # - task: PowerShell@2 + # displayName: Test Queue Pipeline with PAT + # inputs: + # pwsh: true + # filePath: eng/common/scripts/Queue-Pipeline.ps1 + # arguments: > + # -Organization "azure-sdk" + # -Project "public" + # -DefinitionId "$(ToolsCODEOWNERSLinterId)" + # -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)" - task: PowerShell@2 displayName: Test Retain pipeline run From 90234e2310984bdf91e5f53fde984066b8165d64 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 13:47:25 -0700 Subject: [PATCH 7/9] AuthToken to BearerToken and remove unused Base64EncodedAuthToken from the script parameters --- eng/common/scripts/Add-RetentionLease.ps1 | 20 ++++----- eng/common/scripts/Invoke-DevOpsAPI.ps1 | 53 +++++++++++------------ eng/common/scripts/Queue-Pipeline.ps1 | 22 ++++------ eng/pipelines/pipeline-queue-test.yml | 2 +- 4 files changed, 45 insertions(+), 52 deletions(-) diff --git a/eng/common/scripts/Add-RetentionLease.ps1 b/eng/common/scripts/Add-RetentionLease.ps1 index 40ed3e0fe4c..0f4bc633464 100644 --- a/eng/common/scripts/Add-RetentionLease.ps1 +++ b/eng/common/scripts/Add-RetentionLease.ps1 @@ -18,34 +18,34 @@ param( [Parameter(Mandatory = $false)] [string]$OwnerId = "azure-sdk-pipeline-automation", - [Parameter(Mandatory = $false)] - [string]$AccessToken = $env:DEVOPS_PAT, - - [Parameter(Mandatory = $false)] - [string]$AuthToken=$null + # This script shouldn't need anything other than the $System.AccessToken from + # from the build pipeline. The retain-run.yml template doesn't run outside + # of the pipeline it's manipulating the retention leases for. + [Parameter(Mandatory = $true)] + [string]$AccessToken = $env:DEVOPS_PAT ) Set-StrictMode -Version 3 . (Join-Path $PSScriptRoot common.ps1) +$Base64EncodedToken=$null if (![string]::IsNullOrWhiteSpace($AccessToken)) { - $encodedAuthToken = Get-Base64EncodedToken $AccessToken + $Base64EncodedToken = Get-Base64EncodedToken $AccessToken } LogDebug "Checking for existing leases on run: $RunId" -$existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedAuthToken $encodedAuthToken +$existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedToken $Base64EncodedToken if ($existingLeases.count -ne 0) { LogDebug "Found $($existingLeases.count) leases, will delete them first." foreach ($lease in $existingLeases.value) { LogDebug "Deleting lease: $($lease.leaseId)" - Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AuthToken + Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedToken $Base64EncodedToken } } - LogDebug "Creating new lease on run: $RunId" -$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken -AccessToken $AuthToken +$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedToken $Base64EncodedToken LogDebug "Lease ID is: $($lease.value.leaseId)" \ No newline at end of file diff --git a/eng/common/scripts/Invoke-DevOpsAPI.ps1 b/eng/common/scripts/Invoke-DevOpsAPI.ps1 index de9b513e41f..dc525ce7b10 100644 --- a/eng/common/scripts/Invoke-DevOpsAPI.ps1 +++ b/eng/common/scripts/Invoke-DevOpsAPI.ps1 @@ -20,23 +20,22 @@ function Get-Base64EncodedToken([string]$AuthToken) # The AccessToken would be the querying the Azure resource with the following command: # az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv # The header for an AccessToken requires Bearer authorization -function Get-DevOpsApiHeaders ($Base64EncodedToken, $AccessToken) { +function Get-DevOpsApiHeaders { + param ( + $Base64EncodedToken=$null, + $BearerToken=$null + ) $headers = $null - if (![string]::IsNullOrWhiteSpace($Base64EncodedToken) -and - ![string]::IsNullOrWhiteSpace($AccessToken)) { - LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because Base64EncodedToken and AccessToken are both set and only one should be." - exit 1 - } if (![string]::IsNullOrWhiteSpace($Base64EncodedToken)) { $headers = @{ Authorization = "Basic $Base64EncodedToken" } - } elseif (![string]::IsNullOrWhiteSpace($AccessToken)) { + } elseif (![string]::IsNullOrWhiteSpace($BearerToken)) { $headers = @{ - Authorization = "Bearer $AccessToken" + Authorization = "Bearer $BearerToken" } } else { - LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because neither Base64EncodedToken nor AccessToken are set." + LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because neither Base64EncodedToken nor BearerToken are set." exit 1 } return $headers @@ -49,8 +48,8 @@ function Start-DevOpsBuild { $SourceBranch, [Parameter(Mandatory = $true)] $DefinitionId, - $Base64EncodedAuthToken=$null, - $AccessToken=$null, + $Base64EncodedToken=$null, + $BearerToken=$null, [Parameter(Mandatory = $false)] [string]$BuildParametersJson ) @@ -63,7 +62,7 @@ function Start-DevOpsBuild { parameters = $BuildParametersJson } - $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken) return Invoke-RestMethod ` -Method POST ` @@ -82,8 +81,8 @@ function Update-DevOpsBuild { [Parameter(Mandatory = $true)] $BuildId, $Status, # pass canceling to cancel build - $Base64EncodedAuthToken, - $AccessToken + $Base64EncodedToken=$null, + $BearerToken=$null ) $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project, "build", "builds/$BuildId", "" @@ -91,7 +90,7 @@ function Update-DevOpsBuild { if ($Status) { $parameters["status"] = $Status} - $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken) return Invoke-RestMethod ` -Method PATCH ` @@ -109,8 +108,8 @@ function Get-DevOpsBuilds { $BranchName, # Should start with 'refs/heads/' $Definitions, # Comma seperated string of definition IDs $StatusFilter, # Comma seperated string 'cancelling, completed, inProgress, notStarted' - $Base64EncodedAuthToken, - $AccessToken + $Base64EncodedToken=$null, + $BearerToken=$null ) $query = "" @@ -120,7 +119,7 @@ function Get-DevOpsBuilds { if ($StatusFilter) { $query += "statusFilter=$StatusFilter&" } $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project , "build" , "builds", $query - $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken) return Invoke-RestMethod ` -Method GET ` @@ -134,13 +133,13 @@ function Delete-RetentionLease { $Organization, $Project, $LeaseId, - $Base64EncodedAuthToken, - $AccessToken + $Base64EncodedToken=$null, + $BearerToken=$null ) $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ids=$LeaseId&api-version=6.0-preview.1" - $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken) return Invoke-RestMethod ` -Method DELETE ` @@ -156,13 +155,13 @@ function Get-RetentionLeases { $DefinitionId, $RunId, $OwnerId, - $Base64EncodedAuthToken, - $AccessToken + $Base64EncodedToken=$null, + $BearerToken=$null ) $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ownerId=$OwnerId&definitionId=$DefinitionId&runId=$RunId&api-version=6.0-preview.1" - $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken) return Invoke-RestMethod ` -Method GET ` @@ -179,8 +178,8 @@ function Add-RetentionLease { $RunId, $OwnerId, $DaysValid, - $Base64EncodedAuthToken, - $AccessToken + $Base64EncodedToken=$null, + $BearerToken=$null ) $parameter = @{} @@ -194,7 +193,7 @@ function Add-RetentionLease { $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?api-version=6.0-preview.1" - $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken -AccessToken $AccessToken) + $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken) return Invoke-RestMethod ` -Method POST ` diff --git a/eng/common/scripts/Queue-Pipeline.ps1 b/eng/common/scripts/Queue-Pipeline.ps1 index fb7eeba0f7f..e100300edc8 100644 --- a/eng/common/scripts/Queue-Pipeline.ps1 +++ b/eng/common/scripts/Queue-Pipeline.ps1 @@ -57,26 +57,20 @@ param( [string]$VsoQueuedPipelines, - # Already base 64 encoded authentication token - [string]$Base64EncodedAuthToken=$null, - # Unencoded authentication token from a PAT [string]$AuthToken=$null, # Temp access token from the logged in az cli user for azure devops resource - [string]$AccessToken=$null, + [string]$BearerToken=$null, [Parameter(Mandatory = $false)] [string]$BuildParametersJson ) . (Join-Path $PSScriptRoot common.ps1) - -if (!$Base64EncodedAuthToken) -{ - if (![string]::IsNullOrWhiteSpace($AuthToken)) { - $Base64EncodedAuthToken = Get-Base64EncodedToken $AuthToken - } +$Base64EncodedToken=$null +if (![string]::IsNullOrWhiteSpace($AuthToken)) { + $Base64EncodedToken = Get-Base64EncodedToken $AuthToken } # Skip if SourceBranch is empty because it we cannot generate a target branch @@ -85,7 +79,7 @@ if ($CancelPreviousBuilds -and $SourceBranch) { try { $queuedBuilds = Get-DevOpsBuilds -BranchName "refs/heads/$SourceBranch" -Definitions $DefinitionId ` - -StatusFilter "inProgress, notStarted" -Base64EncodedAuthToken $Base64EncodedAuthToken + -StatusFilter "inProgress, notStarted" -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken if ($queuedBuilds.count -eq 0) { LogDebug "There is no previous build still inprogress or about to start." @@ -94,7 +88,7 @@ if ($CancelPreviousBuilds -and $SourceBranch) foreach ($build in $queuedBuilds.Value) { $buildID = $build.id LogDebug "Canceling build [ $($build._links.web.href) ]" - Update-DevOpsBuild -BuildId $buildID -Status "cancelling" -Base64EncodedAuthToken $Base64EncodedAuthToken + Update-DevOpsBuild -BuildId $buildID -Status "cancelling" -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken } } catch { @@ -109,8 +103,8 @@ try { -Project $Project ` -SourceBranch $SourceBranch ` -DefinitionId $DefinitionId ` - -Base64EncodedAuthToken $Base64EncodedAuthToken ` - -AccessToken $AccessToken ` + -Base64EncodedToken $Base64EncodedToken ` + -BearerToken $BearerToken ` -BuildParametersJson $BuildParametersJson } catch { diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml index 09f0ca140eb..40e0c61c481 100644 --- a/eng/pipelines/pipeline-queue-test.yml +++ b/eng/pipelines/pipeline-queue-test.yml @@ -23,7 +23,7 @@ jobs: -Organization "azure-sdk" ` -Project "public" ` -DefinitionId "$(ToolsCODEOWNERSLinterId)" ` - -AccessToken $accessToken + -BearerToken $accessToken # # This task is going to become obsolete once the PATs go away # # the queueing PAT will be gone first but there's another PAT From 3533c4a1d23b78e83138ee5c5f28545e4f97cd0c Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 14:20:16 -0700 Subject: [PATCH 8/9] remove unneccsary if not null check for the mandatory parameter --- eng/common/scripts/Add-RetentionLease.ps1 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/common/scripts/Add-RetentionLease.ps1 b/eng/common/scripts/Add-RetentionLease.ps1 index 0f4bc633464..3532aecf071 100644 --- a/eng/common/scripts/Add-RetentionLease.ps1 +++ b/eng/common/scripts/Add-RetentionLease.ps1 @@ -29,10 +29,7 @@ Set-StrictMode -Version 3 . (Join-Path $PSScriptRoot common.ps1) -$Base64EncodedToken=$null -if (![string]::IsNullOrWhiteSpace($AccessToken)) { - $Base64EncodedToken = Get-Base64EncodedToken $AccessToken -} +$Base64EncodedToken = Get-Base64EncodedToken $AccessToken LogDebug "Checking for existing leases on run: $RunId" $existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedToken $Base64EncodedToken From f52692272625bfa168780ee4b2fff71484d0fa4e Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 28 May 2024 15:14:55 -0700 Subject: [PATCH 9/9] Adding sync-directory changes to the PR --- .../templates/steps/sync-directory.yml | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml index ecd41aedb7c..3afbdb35ed6 100644 --- a/eng/pipelines/templates/steps/sync-directory.yml +++ b/eng/pipelines/templates/steps/sync-directory.yml @@ -79,37 +79,43 @@ steps: -PushArgs "${{ parameters.PushArgs }}" -AmendCommit $True - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Queue template pipeline condition: and(succeeded(), ne(variables['${{repo}}-template-definition-id'], '')) inputs: - pwsh: true + azureSubscription: opensource-api-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1 ` + -Organization "azure-sdk" ` + -Project "internal" ` + -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline" ` + -DefinitionId "$(${{repo}}-template-definition-id)" ` + -VsoQueuedPipelines "QUEUEDPIPELINES" ` + -CancelPreviousBuilds $True ` + -BearerToken $accessToken workingDirectory: ${{ parameters.WorkingDirectory }} - filePath: ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1 - arguments: > - -Organization "azure-sdk" - -Project "internal" - -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline" - -DefinitionId "$(${{repo}}-template-definition-id)" - -VsoQueuedPipelines "QUEUEDPIPELINES" - -CancelPreviousBuilds $True - -AuthToken $(azuresdk-azure-sdk-devops-build-queuing-pat) - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Queue live-test template pipeline condition: and(succeeded(), ne(variables['${{repo}}-template-tests-definition-id'], '')) inputs: - pwsh: true + azureSubscription: opensource-api-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1 ` + -Organization "azure-sdk" ` + -Project "internal" ` + -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline" ` + -DefinitionId "$(${{repo}}-template-tests-definition-id)" ` + -VsoQueuedPipelines "QUEUEDPIPELINES" ` + -CancelPreviousBuilds $True ` + -BearerToken $accessToken workingDirectory: ${{ parameters.WorkingDirectory }} - filePath: ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1 - arguments: > - -Organization "azure-sdk" - -Project "internal" - -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline" - -DefinitionId "$(${{repo}}-template-tests-definition-id)" - -VsoQueuedPipelines "QUEUEDPIPELINES" - -CancelPreviousBuilds $True - -AuthToken $(azuresdk-azure-sdk-devops-build-queuing-pat) - task: PowerShell@2 displayName: Write Queued Pipeline Information to Tools PR