Changes to scripts to deal with PATs and AccessTokens

eng/common/scripts/Add-RetentionLease.ps1

@@ -18,29 +18,34 @@ param(
   [Parameter(Mandatory = $false)]
   [string]$OwnerId = "azure-sdk-pipeline-automation",
 
-  [Parameter(Mandatory = $false)]
+  # This script shouldn't need anything other than the $System.AccessToken from
+  # from the build pipeline. The retain-run.yml template doesn't run outside
+  # of the pipeline it's manipulating the retention leases for.
+  [Parameter(Mandatory = $true)]
   [string]$AccessToken = $env:DEVOPS_PAT
 )
 
 Set-StrictMode -Version 3
 
 . (Join-Path $PSScriptRoot common.ps1)
 
-$encodedAuthToken = Get-Base64EncodedToken $AccessToken
+$Base64EncodedToken=$null
+if (![string]::IsNullOrWhiteSpace($AccessToken)) {
+  $Base64EncodedToken = Get-Base64EncodedToken $AccessToken
+}
 
 LogDebug "Checking for existing leases on run: $RunId"
-$existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedAuthToken $encodedAuthToken
+$existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedToken $Base64EncodedToken
 
 if ($existingLeases.count -ne 0) {
     LogDebug "Found $($existingLeases.count) leases, will delete them first."
 
     foreach ($lease in $existingLeases.value) {
         LogDebug "Deleting lease: $($lease.leaseId)"
-        Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken
+        Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedToken $Base64EncodedToken
     }
 
 }
-
 LogDebug "Creating new lease on run: $RunId"
-$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken
+$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedToken $Base64EncodedToken
 LogDebug "Lease ID is: $($lease.value.leaseId)"

eng/common/scripts/Invoke-DevOpsAPI.ps1

@@ -16,9 +16,27 @@ function Get-Base64EncodedToken([string]$AuthToken)
   return $encodedAuthToken
 }
 
-function Get-DevOpsApiHeaders ($Base64EncodedToken) {
-  $headers = @{
-    Authorization = "Basic $Base64EncodedToken"
+# The Base64EncodedToken would be from a PAT that was passed in and the header requires Basic authorization
+# The AccessToken would be the querying the Azure resource with the following command:
+#   az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+# The header for an AccessToken requires Bearer authorization
+function Get-DevOpsApiHeaders {
+  param (
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
+  )
+  $headers = $null
+  if (![string]::IsNullOrWhiteSpace($Base64EncodedToken)) {
+    $headers = @{
+      Authorization = "Basic $Base64EncodedToken"
+    }
+  } elseif (![string]::IsNullOrWhiteSpace($BearerToken)) {
+    $headers = @{
+      Authorization = "Bearer $BearerToken"
+    }
+  } else {
+    LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because neither Base64EncodedToken nor BearerToken are set."
+    exit 1
   }
   return $headers
 }
@@ -30,9 +48,8 @@ function Start-DevOpsBuild {
     $SourceBranch,
     [Parameter(Mandatory = $true)]
     $DefinitionId,
-    [ValidateNotNullOrEmpty()]
-    [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken,
+    $Base64EncodedToken=$null,
+    $BearerToken=$null,
     [Parameter(Mandatory = $false)]
     [string]$BuildParametersJson
   )
@@ -45,11 +62,13 @@ function Start-DevOpsBuild {
     parameters = $BuildParametersJson
   }
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method POST `
           -Body ($parameters | ConvertTo-Json) `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3 `
           -ContentType "application/json"
 }
@@ -62,21 +81,22 @@ function Update-DevOpsBuild {
     [Parameter(Mandatory = $true)]
     $BuildId,
     $Status, # pass canceling to cancel build
-    [ValidateNotNullOrEmpty()]
-    [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project, "build", "builds/$BuildId", ""
   $parameters = @{}
 
   if ($Status) { $parameters["status"] = $Status}
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method PATCH `
           -Body ($parameters | ConvertTo-Json) `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3 `
           -ContentType "application/json"
 }
@@ -88,9 +108,8 @@ function Get-DevOpsBuilds {
     $BranchName, # Should start with 'refs/heads/'
     $Definitions, # Comma seperated string of definition IDs
     $StatusFilter, # Comma seperated string 'cancelling, completed, inProgress, notStarted'
-    [ValidateNotNullOrEmpty()]
-    [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $query = ""
@@ -100,10 +119,12 @@ function Get-DevOpsBuilds {
   if ($StatusFilter) { $query += "statusFilter=$StatusFilter&" }
   $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project , "build" , "builds", $query
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method GET `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3
 }
 
@@ -112,15 +133,18 @@ function Delete-RetentionLease {
     $Organization,
     $Project,
     $LeaseId,
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ids=$LeaseId&api-version=6.0-preview.1"
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
     -Method DELETE `
     -Uri $uri `
-    -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+    -Headers $headers `
     -MaximumRetryCount 3
 }
 
@@ -131,15 +155,18 @@ function Get-RetentionLeases {
     $DefinitionId,
     $RunId,
     $OwnerId,
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ownerId=$OwnerId&definitionId=$DefinitionId&runId=$RunId&api-version=6.0-preview.1"
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
     -Method GET `
     -Uri $uri `
-    -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+    -Headers $headers `
     -MaximumRetryCount 3
 }
 
@@ -151,7 +178,8 @@ function Add-RetentionLease {
     $RunId,
     $OwnerId,
     $DaysValid,
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $parameter = @{}
@@ -165,12 +193,13 @@ function Add-RetentionLease {
 
   $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?api-version=6.0-preview.1"
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method POST `
           -Body "[$body]" `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3 `
           -ContentType "application/json"
-
 }

eng/common/scripts/Queue-Pipeline.ps1

@@ -57,21 +57,20 @@ param(
 
   [string]$VsoQueuedPipelines,
 
-  # Already base 64 encoded authentication token
-  [string]$Base64EncodedAuthToken,
+  # Unencoded authentication token from a PAT
+  [string]$AuthToken=$null,
 
-  # Unencoded authentication token
-  [string]$AuthToken,
+  # Temp access token from the logged in az cli user for azure devops resource
+  [string]$BearerToken=$null,
 
   [Parameter(Mandatory = $false)]
   [string]$BuildParametersJson
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
-
-if (!$Base64EncodedAuthToken)
-{
-  $Base64EncodedAuthToken = Get-Base64EncodedToken $AuthToken
+$Base64EncodedToken=$null
+if (![string]::IsNullOrWhiteSpace($AuthToken)) {
+  $Base64EncodedToken = Get-Base64EncodedToken $AuthToken
 }
 
 # Skip if SourceBranch is empty because it we cannot generate a target branch
@@ -80,7 +79,7 @@ if ($CancelPreviousBuilds -and $SourceBranch)
 {
   try {
     $queuedBuilds = Get-DevOpsBuilds -BranchName "refs/heads/$SourceBranch" -Definitions $DefinitionId `
-    -StatusFilter "inProgress, notStarted" -Base64EncodedAuthToken $Base64EncodedAuthToken
+    -StatusFilter "inProgress, notStarted" -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken
 
     if ($queuedBuilds.count -eq 0) {
       LogDebug "There is no previous build still inprogress or about to start."
@@ -89,7 +88,7 @@ if ($CancelPreviousBuilds -and $SourceBranch)
     foreach ($build in $queuedBuilds.Value) {
       $buildID = $build.id
       LogDebug "Canceling build [ $($build._links.web.href) ]"
-      Update-DevOpsBuild -BuildId $buildID -Status "cancelling" -Base64EncodedAuthToken $Base64EncodedAuthToken
+      Update-DevOpsBuild -BuildId $buildID -Status "cancelling" -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken
     }
   }
   catch {
@@ -104,7 +103,8 @@ try {
     -Project $Project `
     -SourceBranch $SourceBranch `
     -DefinitionId $DefinitionId `
-    -Base64EncodedAuthToken $Base64EncodedAuthToken `
+    -Base64EncodedToken $Base64EncodedToken `
+    -BearerToken $BearerToken `
     -BuildParametersJson $BuildParametersJson
 }
 catch {

eng/pipelines/pipeline-queue-test.yml

@@ -0,0 +1,57 @@
+trigger: none
+
+pr: none
+
+jobs:
+  - job: Run
+    pool:
+      name: azsdk-pool-mms-ubuntu-2204-general
+      vmImage: ubuntu-22.04
+    variables:
+      ToolsCODEOWNERSLinterId: 6597
+    steps:
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      - task: AzureCLI@2
+        displayName: Test Authenticate to OpenSource API and queue pipeline
+        inputs:
+          azureSubscription: opensource-api-connection
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: |
+            $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+            eng/common/scripts/Queue-Pipeline.ps1 `
+              -Organization "azure-sdk" `
+              -Project "public" `
+              -DefinitionId "$(ToolsCODEOWNERSLinterId)" `
+              -BearerToken $accessToken
+
+      # # This task is going to become obsolete once the PATs go away
+      # # the queueing PAT will be gone first but there's another PAT
+      # # for queuing docs and this task is just testing the pipeline
+      # # scripts. This task will need to be commented out or removed.
+      # - task: PowerShell@2
+      #   displayName: Test Queue Pipeline with PAT
+      #   inputs:
+      #     pwsh: true
+      #     filePath: eng/common/scripts/Queue-Pipeline.ps1
+      #     arguments: >
+      #       -Organization "azure-sdk"
+      #       -Project "public"
+      #       -DefinitionId "$(ToolsCODEOWNERSLinterId)"
+      #       -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)"
+
+      - task: PowerShell@2
+        displayName: Test Retain pipeline run
+        env:
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+        inputs:
+          pwsh: true
+          filePath: $(Build.SourcesDirectory)/eng/common/scripts/Add-RetentionLease.ps1
+          arguments: >
+            -Organization azure-sdk
+            -Project $(System.TeamProject)
+            -DefinitionId $(System.DefinitionId)
+            -RunId $(Build.BuildId)
+            -DaysValid 7
+            -AccessToken $env:SYSTEM_ACCESSTOKEN
+            -Debug