From 341c402cce0fa1ec3de2afb75afdcfff49600c7f Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 7 Nov 2024 13:42:26 -0800 Subject: [PATCH 1/6] Add step for emitting rate limits for a token --- .../steps/emit-ratelimit-metrics.yml | 25 +++++++++++++++++++ .../templates/steps/git-push-changes.yml | 5 ++++ 2 files changed, 30 insertions(+) create mode 100644 eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml diff --git a/eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml b/eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml new file mode 100644 index 0000000000..44274a981d --- /dev/null +++ b/eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml @@ -0,0 +1,25 @@ +parameters: + - name: User + type: string + - name: GitHubToken + type: string + +steps: +- pwsh: | + $headers = @{ + "Authorization" = "Bearer $env:GITHUB_TOKEN" + "X-GitHub-Api-Version" = "2022-11-28" + } + + $response = Invoke-RestMethod -Uri 'https://api.github.com/rate_limit' -Headers $headers -Method Get + $timestamp = Get-Date + foreach ($property in $response.resources.PSObject.Properties.Value) + { + Write-Host "logmetric: $( @{ name= "github_rate_limit_remaining_total"; value= $property.remaining; timestamp= $timestamp; labels= @{ user=$env:GITHUB_USER; resource=$property.resource }} | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( @{ name= "github_rate_limit_limit_total"; value= $property.limit; timestamp= $timestamp; labels= @{ user=$env:GITHUB_USER; resource=$property.resource }} | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( @{ name= "github_rate_limit_used_total"; value= $property.used; timestamp= $timestamp; labels= @{ user=$env:GITHUB_USER; resource=$property.resource }} | ConvertTo-Json -Compress)" + } + displayName: Check GitHub Rate Limit + env: + GITHUB_TOKEN: ${{ parameters.GitHubToken}} + GITHUB_USER: ${{ parameters.User}} \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/git-push-changes.yml b/eng/common/pipelines/templates/steps/git-push-changes.yml index c8fbeaa976..da2a101764 100644 --- a/eng/common/pipelines/templates/steps/git-push-changes.yml +++ b/eng/common/pipelines/templates/steps/git-push-changes.yml @@ -28,6 +28,11 @@ steps: condition: succeeded() workingDirectory: ${{ parameters.WorkingDirectory }} +- template: ./emit-ratelimit-metrics.yml + parameters: + User: 'azure-sdk' + GitHubToken: $(azuresdk-github-pat) + - task: PowerShell@2 displayName: Push changes condition: and(succeeded(), eq(variables['HasChanges'], 'true')) From a1d9867c3566da800c16fee206c4de42a1a09153 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 7 Nov 2024 13:49:03 -0800 Subject: [PATCH 2/6] add step to eng/common sync template --- ...emit-ratelimit-metrics.yml => emit-rate-limit-metrics.yml} | 4 ++-- eng/common/pipelines/templates/steps/git-push-changes.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename eng/common/pipelines/templates/steps/{emit-ratelimit-metrics.yml => emit-rate-limit-metrics.yml} (94%) diff --git a/eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml similarity index 94% rename from eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml rename to eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml index 44274a981d..8d0ccbfefc 100644 --- a/eng/common/pipelines/templates/steps/emit-ratelimit-metrics.yml +++ b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml @@ -1,5 +1,5 @@ parameters: - - name: User + - name: GitHubUser type: string - name: GitHubToken type: string @@ -22,4 +22,4 @@ steps: displayName: Check GitHub Rate Limit env: GITHUB_TOKEN: ${{ parameters.GitHubToken}} - GITHUB_USER: ${{ parameters.User}} \ No newline at end of file + GITHUB_USER: ${{ parameters.GitHubUser}} \ No newline at end of file diff --git a/eng/common/pipelines/templates/steps/git-push-changes.yml b/eng/common/pipelines/templates/steps/git-push-changes.yml index da2a101764..53d70fac45 100644 --- a/eng/common/pipelines/templates/steps/git-push-changes.yml +++ b/eng/common/pipelines/templates/steps/git-push-changes.yml @@ -28,9 +28,9 @@ steps: condition: succeeded() workingDirectory: ${{ parameters.WorkingDirectory }} -- template: ./emit-ratelimit-metrics.yml +- template: /eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml parameters: - User: 'azure-sdk' + GitHubUser: azure-sdk GitHubToken: $(azuresdk-github-pat) - task: PowerShell@2 From 79ab2871f05520406dbf3213232a2d0ab2df1cf9 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 7 Nov 2024 14:29:46 -0800 Subject: [PATCH 3/6] Fix resource label output --- .../pipelines/templates/steps/emit-rate-limit-metrics.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml index 8d0ccbfefc..41fd30e4f0 100644 --- a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml +++ b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml @@ -15,9 +15,10 @@ steps: $timestamp = Get-Date foreach ($property in $response.resources.PSObject.Properties.Value) { - Write-Host "logmetric: $( @{ name= "github_rate_limit_remaining_total"; value= $property.remaining; timestamp= $timestamp; labels= @{ user=$env:GITHUB_USER; resource=$property.resource }} | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( @{ name= "github_rate_limit_limit_total"; value= $property.limit; timestamp= $timestamp; labels= @{ user=$env:GITHUB_USER; resource=$property.resource }} | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( @{ name= "github_rate_limit_used_total"; value= $property.used; timestamp= $timestamp; labels= @{ user=$env:GITHUB_USER; resource=$property.resource }} | ConvertTo-Json -Compress)" + $labels = @{ user=$env:GITHUB_USER; resource=$property.resource } + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_remaining_total"; value= $property.remaining; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_limit_total"; value= $property.limit; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_total"; value= $property.used; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" } displayName: Check GitHub Rate Limit env: From 65a10d0cbddf5e6d4c09856f2a118af45cfc3246 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 7 Nov 2024 15:26:46 -0800 Subject: [PATCH 4/6] Use propery name instead of nested 'resource' property --- .../templates/steps/emit-rate-limit-metrics.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml index 41fd30e4f0..bc6cc0c86d 100644 --- a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml +++ b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml @@ -13,12 +13,12 @@ steps: $response = Invoke-RestMethod -Uri 'https://api.github.com/rate_limit' -Headers $headers -Method Get $timestamp = Get-Date - foreach ($property in $response.resources.PSObject.Properties.Value) - { - $labels = @{ user=$env:GITHUB_USER; resource=$property.resource } - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_remaining_total"; value= $property.remaining; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_limit_total"; value= $property.limit; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_total"; value= $property.used; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + foreach ($property in $response.resources.PSObject.Properties) + { + $labels = @{ user= $env:GITHUB_USER; resource= $property.Name } + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_remaining_total"; value= $property.Value.remaining; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_limit_total"; value= $property.Value.limit; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_total"; value= $property.Value.used; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" } displayName: Check GitHub Rate Limit env: From 4c94e2f5d13b4d1d5eb150be6de8511d5d89c7cc Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 7 Nov 2024 15:37:03 -0800 Subject: [PATCH 5/6] Add percent metric too --- .../templates/steps/emit-rate-limit-metrics.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml index bc6cc0c86d..45c75677a9 100644 --- a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml +++ b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml @@ -16,9 +16,16 @@ steps: foreach ($property in $response.resources.PSObject.Properties) { $labels = @{ user= $env:GITHUB_USER; resource= $property.Name } - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_remaining_total"; value= $property.Value.remaining; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_limit_total"; value= $property.Value.limit; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_total"; value= $property.Value.used; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + + $remaining = $property.Value.remaining + $limit = $property.Value.limit + $used = $property.Value.used + $percent = $used / $limit * 100 + + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_remaining_total"; value= $remaining; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_limit_total"; value= $limit; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_total"; value= $used; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_percent"; value= $percent; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" } displayName: Check GitHub Rate Limit env: From b57e9c321ab91b9451e63d5a40deabd67f47b452 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Thu, 7 Nov 2024 15:43:33 -0800 Subject: [PATCH 6/6] Add divide by zero safety --- .../templates/steps/emit-rate-limit-metrics.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml index 45c75677a9..1dd34a8da3 100644 --- a/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml +++ b/eng/common/pipelines/templates/steps/emit-rate-limit-metrics.yml @@ -16,16 +16,19 @@ steps: foreach ($property in $response.resources.PSObject.Properties) { $labels = @{ user= $env:GITHUB_USER; resource= $property.Name } - + $remaining = $property.Value.remaining $limit = $property.Value.limit $used = $property.Value.used - $percent = $used / $limit * 100 Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_remaining_total"; value= $remaining; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_limit_total"; value= $limit; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_total"; value= $used; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" - Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_percent"; value= $percent; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + + if ($limit -ne 0) { + $percent = $used / $limit * 100 + Write-Host "logmetric: $( [ordered]@{ name= "github_ratelimit_used_percent"; value= $percent; timestamp= $timestamp; labels= $labels } | ConvertTo-Json -Compress)" + } } displayName: Check GitHub Rate Limit env: