Changes to scripts to deal with PATs and AccessTokens (#8340)

* Changes to scripts to deal with PATs and AccessTokens * Remove trailing backtick from the last line of the inline script * fix path * the linter pipeline is public, not internal * swap access and auth for add-retention-lease * comment out the task that queues with the PAT * AuthToken to BearerToken and remove unused Base64EncodedAuthToken from the script parameters * remove unneccsary if not null check for the mandatory parameter * Adding sync-directory changes to the PR
Azure · May 28, 2024 · b7f9ce2 · b7f9ce2
1 parent e78dfab
commit b7f9ce2
Show file tree

Hide file tree

Showing 5 changed files with 155 additions and 61 deletions.
diff --git a/eng/common/scripts/Add-RetentionLease.ps1 b/eng/common/scripts/Add-RetentionLease.ps1
@@ -18,29 +18,31 @@ param(
   [Parameter(Mandatory = $false)]
   [string]$OwnerId = "azure-sdk-pipeline-automation",
 
-  [Parameter(Mandatory = $false)]
+  # This script shouldn't need anything other than the $System.AccessToken from
+  # from the build pipeline. The retain-run.yml template doesn't run outside
+  # of the pipeline it's manipulating the retention leases for.
+  [Parameter(Mandatory = $true)]
   [string]$AccessToken = $env:DEVOPS_PAT
 )
 
 Set-StrictMode -Version 3
 
 . (Join-Path $PSScriptRoot common.ps1)
 
-$encodedAuthToken = Get-Base64EncodedToken $AccessToken
+$Base64EncodedToken = Get-Base64EncodedToken $AccessToken
 
 LogDebug "Checking for existing leases on run: $RunId"
-$existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedAuthToken $encodedAuthToken
+$existingLeases = Get-RetentionLeases -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -Base64EncodedToken $Base64EncodedToken
 
 if ($existingLeases.count -ne 0) {
     LogDebug "Found $($existingLeases.count) leases, will delete them first."
 
     foreach ($lease in $existingLeases.value) {
         LogDebug "Deleting lease: $($lease.leaseId)"
-        Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedAuthToken $encodedAuthToken
+        Delete-RetentionLease -Organization $Organization -Project $Project -LeaseId $lease.leaseId -Base64EncodedToken $Base64EncodedToken
     }
 
 }
-
 LogDebug "Creating new lease on run: $RunId"
-$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedAuthToken $encodedAuthToken
+$lease = Add-RetentionLease -Organization $Organization -Project $Project -DefinitionId $DefinitionId -RunId $RunId -OwnerId $OwnerId -DaysValid $DaysValid -Base64EncodedToken $Base64EncodedToken
 LogDebug "Lease ID is: $($lease.value.leaseId)"
diff --git a/eng/common/scripts/Invoke-DevOpsAPI.ps1 b/eng/common/scripts/Invoke-DevOpsAPI.ps1
@@ -16,9 +16,27 @@ function Get-Base64EncodedToken([string]$AuthToken)
   return $encodedAuthToken
 }
 
-function Get-DevOpsApiHeaders ($Base64EncodedToken) {
-  $headers = @{
-    Authorization = "Basic $Base64EncodedToken"
+# The Base64EncodedToken would be from a PAT that was passed in and the header requires Basic authorization
+# The AccessToken would be the querying the Azure resource with the following command:
+#   az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+# The header for an AccessToken requires Bearer authorization
+function Get-DevOpsApiHeaders {
+  param (
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
+  )
+  $headers = $null
+  if (![string]::IsNullOrWhiteSpace($Base64EncodedToken)) {
+    $headers = @{
+      Authorization = "Basic $Base64EncodedToken"
+    }
+  } elseif (![string]::IsNullOrWhiteSpace($BearerToken)) {
+    $headers = @{
+      Authorization = "Bearer $BearerToken"
+    }
+  } else {
+    LogError "Get-DevOpsApiHeaders::Unable to set the Authentication in the header because neither Base64EncodedToken nor BearerToken are set."
+    exit 1
   }
   return $headers
 }
@@ -30,9 +48,8 @@ function Start-DevOpsBuild {
     $SourceBranch,
     [Parameter(Mandatory = $true)]
     $DefinitionId,
-    [ValidateNotNullOrEmpty()]
-    [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken,
+    $Base64EncodedToken=$null,
+    $BearerToken=$null,
     [Parameter(Mandatory = $false)]
     [string]$BuildParametersJson
   )
@@ -45,11 +62,13 @@ function Start-DevOpsBuild {
     parameters = $BuildParametersJson
   }
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method POST `
           -Body ($parameters | ConvertTo-Json) `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3 `
           -ContentType "application/json"
 }
@@ -62,21 +81,22 @@ function Update-DevOpsBuild {
     [Parameter(Mandatory = $true)]
     $BuildId,
     $Status, # pass canceling to cancel build
-    [ValidateNotNullOrEmpty()]
-    [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project, "build", "builds/$BuildId", ""
   $parameters = @{}
 
   if ($Status) { $parameters["status"] = $Status}
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method PATCH `
           -Body ($parameters | ConvertTo-Json) `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3 `
           -ContentType "application/json"
 }
@@ -88,9 +108,8 @@ function Get-DevOpsBuilds {
     $BranchName, # Should start with 'refs/heads/'
     $Definitions, # Comma seperated string of definition IDs
     $StatusFilter, # Comma seperated string 'cancelling, completed, inProgress, notStarted'
-    [ValidateNotNullOrEmpty()]
-    [Parameter(Mandatory = $true)]
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $query = ""
@@ -100,10 +119,12 @@ function Get-DevOpsBuilds {
   if ($StatusFilter) { $query += "statusFilter=$StatusFilter&" }
   $uri = "$DevOpsAPIBaseURI" -F $Organization, $Project , "build" , "builds", $query
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method GET `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3
 }
 
@@ -112,15 +133,18 @@ function Delete-RetentionLease {
     $Organization,
     $Project,
     $LeaseId,
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ids=$LeaseId&api-version=6.0-preview.1"
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
     -Method DELETE `
     -Uri $uri `
-    -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+    -Headers $headers `
     -MaximumRetryCount 3
 }
 
@@ -131,15 +155,18 @@ function Get-RetentionLeases {
     $DefinitionId,
     $RunId,
     $OwnerId,
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?ownerId=$OwnerId&definitionId=$DefinitionId&runId=$RunId&api-version=6.0-preview.1"
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
     -Method GET `
     -Uri $uri `
-    -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+    -Headers $headers `
     -MaximumRetryCount 3
 }
 
@@ -151,7 +178,8 @@ function Add-RetentionLease {
     $RunId,
     $OwnerId,
     $DaysValid,
-    $Base64EncodedAuthToken
+    $Base64EncodedToken=$null,
+    $BearerToken=$null
   )
 
   $parameter = @{}
@@ -165,12 +193,13 @@ function Add-RetentionLease {
 
   $uri = "https://dev.azure.com/$Organization/$Project/_apis/build/retention/leases?api-version=6.0-preview.1"
 
+  $headers = (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken)
+
   return Invoke-RestMethod `
           -Method POST `
           -Body "[$body]" `
           -Uri $uri `
-          -Headers (Get-DevOpsApiHeaders -Base64EncodedToken $Base64EncodedAuthToken) `
+          -Headers $headers `
           -MaximumRetryCount 3 `
           -ContentType "application/json"
-
 }
diff --git a/eng/common/scripts/Queue-Pipeline.ps1 b/eng/common/scripts/Queue-Pipeline.ps1
@@ -57,21 +57,20 @@ param(
 
   [string]$VsoQueuedPipelines,
 
-  # Already base 64 encoded authentication token
-  [string]$Base64EncodedAuthToken,
+  # Unencoded authentication token from a PAT
+  [string]$AuthToken=$null,
 
-  # Unencoded authentication token
-  [string]$AuthToken,
+  # Temp access token from the logged in az cli user for azure devops resource
+  [string]$BearerToken=$null,
 
   [Parameter(Mandatory = $false)]
   [string]$BuildParametersJson
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
-
-if (!$Base64EncodedAuthToken)
-{
-  $Base64EncodedAuthToken = Get-Base64EncodedToken $AuthToken
+$Base64EncodedToken=$null
+if (![string]::IsNullOrWhiteSpace($AuthToken)) {
+  $Base64EncodedToken = Get-Base64EncodedToken $AuthToken
 }
 
 # Skip if SourceBranch is empty because it we cannot generate a target branch
@@ -80,7 +79,7 @@ if ($CancelPreviousBuilds -and $SourceBranch)
 {
   try {
     $queuedBuilds = Get-DevOpsBuilds -BranchName "refs/heads/$SourceBranch" -Definitions $DefinitionId `
-    -StatusFilter "inProgress, notStarted" -Base64EncodedAuthToken $Base64EncodedAuthToken
+    -StatusFilter "inProgress, notStarted" -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken
 
     if ($queuedBuilds.count -eq 0) {
       LogDebug "There is no previous build still inprogress or about to start."
@@ -89,7 +88,7 @@ if ($CancelPreviousBuilds -and $SourceBranch)
     foreach ($build in $queuedBuilds.Value) {
       $buildID = $build.id
       LogDebug "Canceling build [ $($build._links.web.href) ]"
-      Update-DevOpsBuild -BuildId $buildID -Status "cancelling" -Base64EncodedAuthToken $Base64EncodedAuthToken
+      Update-DevOpsBuild -BuildId $buildID -Status "cancelling" -Base64EncodedToken $Base64EncodedToken -BearerToken $BearerToken
     }
   }
   catch {
@@ -104,7 +103,8 @@ try {
     -Project $Project `
     -SourceBranch $SourceBranch `
     -DefinitionId $DefinitionId `
-    -Base64EncodedAuthToken $Base64EncodedAuthToken `
+    -Base64EncodedToken $Base64EncodedToken `
+    -BearerToken $BearerToken `
     -BuildParametersJson $BuildParametersJson
 }
 catch {

diff --git a/eng/pipelines/pipeline-queue-test.yml b/eng/pipelines/pipeline-queue-test.yml
@@ -0,0 +1,57 @@
+trigger: none
+
+pr: none
+
+jobs:
+  - job: Run
+    pool:
+      name: azsdk-pool-mms-ubuntu-2204-general
+      vmImage: ubuntu-22.04
+    variables:
+      ToolsCODEOWNERSLinterId: 6597
+    steps:
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      - task: AzureCLI@2
+        displayName: Test Authenticate to OpenSource API and queue pipeline
+        inputs:
+          azureSubscription: opensource-api-connection
+          scriptType: pscore
+          scriptLocation: inlineScript
+          inlineScript: |
+            $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+            eng/common/scripts/Queue-Pipeline.ps1 `
+              -Organization "azure-sdk" `
+              -Project "public" `
+              -DefinitionId "$(ToolsCODEOWNERSLinterId)" `
+              -BearerToken $accessToken
+
+      # # This task is going to become obsolete once the PATs go away
+      # # the queueing PAT will be gone first but there's another PAT
+      # # for queuing docs and this task is just testing the pipeline
+      # # scripts. This task will need to be commented out or removed.
+      # - task: PowerShell@2
+      #   displayName: Test Queue Pipeline with PAT
+      #   inputs:
+      #     pwsh: true
+      #     filePath: eng/common/scripts/Queue-Pipeline.ps1
+      #     arguments: >
+      #       -Organization "azure-sdk"
+      #       -Project "public"
+      #       -DefinitionId "$(ToolsCODEOWNERSLinterId)"
+      #       -AuthToken "$(azuresdk-azure-sdk-devops-build-queuing-pat)"
+
+      - task: PowerShell@2
+        displayName: Test Retain pipeline run
+        env:
+          SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+        inputs:
+          pwsh: true
+          filePath: $(Build.SourcesDirectory)/eng/common/scripts/Add-RetentionLease.ps1
+          arguments: >
+            -Organization azure-sdk
+            -Project $(System.TeamProject)
+            -DefinitionId $(System.DefinitionId)
+            -RunId $(Build.BuildId)
+            -DaysValid 7
+            -AccessToken $env:SYSTEM_ACCESSTOKEN
+            -Debug
diff --git a/eng/pipelines/templates/steps/sync-directory.yml b/eng/pipelines/templates/steps/sync-directory.yml
@@ -79,37 +79,43 @@ steps:
           -PushArgs "${{ parameters.PushArgs }}"
           -AmendCommit $True
 
-    - task: PowerShell@2
+    - task: AzureCLI@2
       displayName: Queue template pipeline
       condition: and(succeeded(), ne(variables['${{repo}}-template-definition-id'], ''))
       inputs:
-        pwsh: true
+        azureSubscription: opensource-api-connection
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: |
+          $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+          ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1 `
+            -Organization "azure-sdk" `
+            -Project "internal" `
+            -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline" `
+            -DefinitionId "$(${{repo}}-template-definition-id)" `
+            -VsoQueuedPipelines "QUEUEDPIPELINES" `
+            -CancelPreviousBuilds $True `
+            -BearerToken $accessToken
         workingDirectory: ${{ parameters.WorkingDirectory }}
-        filePath: ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1
-        arguments: >
-          -Organization "azure-sdk"
-          -Project "internal"
-          -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline"
-          -DefinitionId "$(${{repo}}-template-definition-id)"
-          -VsoQueuedPipelines "QUEUEDPIPELINES"
-          -CancelPreviousBuilds $True
-          -AuthToken $(azuresdk-azure-sdk-devops-build-queuing-pat)
 
-    - task: PowerShell@2
+    - task: AzureCLI@2
       displayName: Queue live-test template pipeline
       condition: and(succeeded(), ne(variables['${{repo}}-template-tests-definition-id'], ''))
       inputs:
-        pwsh: true
+        azureSubscription: opensource-api-connection
+        scriptType: pscore
+        scriptLocation: inlineScript
+        inlineScript: |
+          $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
+          ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1 `
+            -Organization "azure-sdk" `
+            -Project "internal" `
+            -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline" `
+            -DefinitionId "$(${{repo}}-template-tests-definition-id)" `
+            -VsoQueuedPipelines "QUEUEDPIPELINES" `
+            -CancelPreviousBuilds $True `
+            -BearerToken $accessToken
         workingDirectory: ${{ parameters.WorkingDirectory }}
-        filePath: ${{ parameters.ScriptDirectory }}/Queue-Pipeline.ps1
-        arguments: >
-          -Organization "azure-sdk"
-          -Project "internal"
-          -SourceBranch "${{ parameters.UpstreamBranchName }}-ForTestPipeline"
-          -DefinitionId "$(${{repo}}-template-tests-definition-id)"
-          -VsoQueuedPipelines "QUEUEDPIPELINES"
-          -CancelPreviousBuilds $True
-          -AuthToken $(azuresdk-azure-sdk-devops-build-queuing-pat)
 
   - task: PowerShell@2
     displayName: Write Queued Pipeline Information to Tools PR