From 5c0628bf3b3b09ec91e68d555c51fc71559a2edd Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 23 Apr 2024 00:26:20 -0300 Subject: [PATCH] feat(host-rules): Support `readOnly` request matching (#28562) Co-authored-by: Rhys Arkins --- docs/usage/configuration-options.md | 21 +++++++++++++++++++ lib/config/options/index.ts | 10 +++++++++ lib/modules/platform/github/index.ts | 4 ++++ lib/types/host-rules.ts | 3 ++- lib/util/github/graphql/datasource-fetcher.ts | 1 + lib/util/host-rules.spec.ts | 19 +++++++++++++++++ lib/util/host-rules.ts | 18 +++++++++++++--- lib/util/http/github.ts | 12 ++++++++++- lib/util/http/host-rules.ts | 9 ++++---- lib/util/http/index.ts | 15 +++++++++++++ lib/util/http/types.ts | 2 ++ .../update/pr/changelog/github/source.ts | 1 + 12 files changed, 106 insertions(+), 9 deletions(-) diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index c3c79115b399e0..e52762dabfcbf7 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -898,6 +898,27 @@ It will be compiled using Handlebars and the regex `groups` result. It will be compiled using Handlebars and the regex `groups` result. It will default to the value of `depName` if left unconfigured/undefined. +### readOnly + +If the `readOnly` field is being set to `true` inside the host rule, it will match only against the requests that are known to be read operations. +Examples are `GET` requests or `HEAD` requests, but also it could be certain types of GraphQL queries. + +This option could be used to avoid rate limits for certain platforms like GitHub or Bitbucket, by offloading the read operations to a different user. + +```json +{ + "hostRules": [ + { + "matchHost": "api.github.com", + "readOnly": true, + "token": "********" + } + ] +} +``` + +If more than one token matches for a read-only request then the `readOnly` token will be given preference. + ### currentValueTemplate If the `currentValue` for a dependency is not captured with a named group then it can be defined in config using this field. diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 8f1151fc6a6eba..1c1285115fa559 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2476,6 +2476,16 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, + { + name: 'readOnly', + description: + 'Match against requests that only read data and do not mutate anything.', + type: 'boolean', + stage: 'repository', + parents: ['hostRules'], + cli: false, + env: false, + }, { name: 'timeout', description: 'Timeout (in milliseconds) for queries to external endpoints.', diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts index 9e1016824a06af..9357a3adc93844 100644 --- a/lib/modules/platform/github/index.ts +++ b/lib/modules/platform/github/index.ts @@ -461,6 +461,7 @@ export async function initRepo({ const opts = hostRules.find({ hostType: 'github', url: platformConfig.endpoint, + readOnly: true, }); config.renovateUsername = renovateUsername; [config.repositoryOwner, config.repositoryName] = repository.split('/'); @@ -499,6 +500,7 @@ export async function initRepo({ name: config.repositoryName, user: renovateUsername, }, + readOnly: true, }); if (res?.errors) { @@ -1214,6 +1216,7 @@ async function getIssues(): Promise { name: config.repositoryName, user: config.renovateUsername, }, + readOnly: true, }, ); @@ -1975,6 +1978,7 @@ export async function getVulnerabilityAlerts(): Promise { variables: { owner: config.repositoryOwner, name: config.repositoryName }, paginate: false, acceptHeader: 'application/vnd.github.vixen-preview+json', + readOnly: true, }); } catch (err) { logger.debug({ err }, 'Error retrieving vulnerability alerts'); diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts index f8a61fd9f708d7..23ac43d7f64cbd 100644 --- a/lib/types/host-rules.ts +++ b/lib/types/host-rules.ts @@ -25,9 +25,10 @@ export interface HostRule { hostType?: string; matchHost?: string; resolvedHost?: string; + readOnly?: boolean; } export type CombinedHostRule = Omit< HostRule, - 'encrypted' | 'hostType' | 'matchHost' | 'resolvedHost' + 'encrypted' | 'hostType' | 'matchHost' | 'resolvedHost' | 'readOnly' >; diff --git a/lib/util/github/graphql/datasource-fetcher.ts b/lib/util/github/graphql/datasource-fetcher.ts index b32995881c262e..f8fe2d47a90e5b 100644 --- a/lib/util/github/graphql/datasource-fetcher.ts +++ b/lib/util/github/graphql/datasource-fetcher.ts @@ -107,6 +107,7 @@ export class GithubGraphqlDatasourceFetcher< return { baseUrl, repository, + readOnly: true, body: { query, variables }, }; } diff --git a/lib/util/host-rules.spec.ts b/lib/util/host-rules.spec.ts index a16a9332e3d29c..f5fa0cbf239a53 100644 --- a/lib/util/host-rules.spec.ts +++ b/lib/util/host-rules.spec.ts @@ -295,6 +295,25 @@ describe('util/host-rules', () => { }), ).toEqual({ token: 'longest' }); }); + + it('matches readOnly requests', () => { + add({ + matchHost: 'https://api.github.com/repos/', + token: 'aaa', + hostType: 'github', + }); + add({ + matchHost: 'https://api.github.com', + token: 'bbb', + readOnly: true, + }); + expect( + find({ + url: 'https://api.github.com/repos/foo/bar/tags', + readOnly: true, + }), + ).toEqual({ token: 'bbb' }); + }); }); describe('hosts()', () => { diff --git a/lib/util/host-rules.ts b/lib/util/host-rules.ts index 3054c6d698e00a..1da59fcd9e3c49 100644 --- a/lib/util/host-rules.ts +++ b/lib/util/host-rules.ts @@ -73,6 +73,7 @@ export function add(params: HostRule): void { export interface HostRuleSearch { hostType?: string; url?: string; + readOnly?: boolean; } function matchesHost(url: string, matchHost: string): boolean { @@ -107,8 +108,9 @@ function fromShorterToLongerMatchHost(a: HostRule, b: HostRule): number { return a.matchHost.length - b.matchHost.length; } -function hostRuleRank({ hostType, matchHost }: HostRule): number { - if (hostType && matchHost) { +function hostRuleRank({ hostType, matchHost, readOnly }: HostRule): number { + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing + if ((hostType || readOnly) && matchHost) { return 3; } @@ -142,6 +144,7 @@ export function find(search: HostRuleSearch): CombinedHostRule { for (const rule of sortedRules) { let hostTypeMatch = true; let hostMatch = true; + let readOnlyMatch = true; if (rule.hostType) { hostTypeMatch = false; @@ -157,7 +160,15 @@ export function find(search: HostRuleSearch): CombinedHostRule { } } - if (hostTypeMatch && hostMatch) { + if (!is.undefined(rule.readOnly)) { + readOnlyMatch = false; + if (search.readOnly === rule.readOnly) { + readOnlyMatch = true; + hostTypeMatch = true; // When we match `readOnly`, we don't care about `hostType` + } + } + + if (hostTypeMatch && readOnlyMatch && hostMatch) { matchedRules.push(clone(rule)); } } @@ -166,6 +177,7 @@ export function find(search: HostRuleSearch): CombinedHostRule { delete res.hostType; delete res.resolvedHost; delete res.matchHost; + delete res.readOnly; return res; } diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index 67276d3c50c13c..3db9d7b4a741dc 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -276,7 +276,7 @@ export class GithubHttp extends Http { options?: InternalHttpOptions & GithubHttpOptions, okToRetry = true, ): Promise> { - const opts: GithubHttpOptions = { + const opts: InternalHttpOptions & GithubHttpOptions = { baseUrl, ...options, throwHttpErrors: true, @@ -296,8 +296,17 @@ export class GithubHttp extends Http { ); } + let readOnly = opts.readOnly; + const { method = 'get' } = opts; + if ( + readOnly === undefined && + ['get', 'head'].includes(method.toLowerCase()) + ) { + readOnly = true; + } const { token } = findMatchingRule(authUrl.toString(), { hostType: this.hostType, + readOnly, }); opts.token = token; } @@ -393,6 +402,7 @@ export class GithubHttp extends Http { baseUrl: baseUrl.replace('/v3/', '/'), // GHE uses unversioned graphql path body, headers: { accept: options?.acceptHeader }, + readOnly: options.readOnly, }; if (options.token) { opts.token = options.token; diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 20c898103743a2..09775fe9f412fe 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -14,10 +14,10 @@ import { matchRegexOrGlobList } from '../string-match'; import { parseUrl } from '../url'; import { dnsLookup } from './dns'; import { keepAliveAgents } from './keep-alive'; -import type { GotOptions } from './types'; +import type { GotOptions, InternalHttpOptions } from './types'; export type HostRulesGotOptions = Pick< - GotOptions, + GotOptions & InternalHttpOptions, | 'hostType' | 'url' | 'noAuth' @@ -34,14 +34,15 @@ export type HostRulesGotOptions = Pick< | 'agent' | 'http2' | 'https' + | 'readOnly' >; export function findMatchingRule( url: string, options: GotOptions, ): HostRule { - const { hostType } = options; - let res = hostRules.find({ hostType, url }); + const { hostType, readOnly } = options; + let res = hostRules.find({ hostType, url, readOnly }); if ( is.nonEmptyString(res.token) || diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 1b084542ba15aa..b93c659497e729 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -162,6 +162,13 @@ export class Http { applyDefaultHeaders(options); + if ( + is.undefined(options.readOnly) && + ['head', 'get'].includes(options.method) + ) { + options.readOnly = true; + } + const hostRule = findMatchingRule(url, options); options = applyHostRule(url, options, hostRule); if (options.enabled === false) { @@ -457,6 +464,14 @@ export class Http { } applyDefaultHeaders(combinedOptions); + + if ( + is.undefined(combinedOptions.readOnly) && + ['head', 'get'].includes(combinedOptions.method) + ) { + combinedOptions.readOnly = true; + } + const hostRule = findMatchingRule(url, combinedOptions); combinedOptions = applyHostRule(resolvedUrl, combinedOptions, hostRule); if (combinedOptions.enabled === false) { diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts index 396ccbfa8cd08e..a767c29c5b7c6b 100644 --- a/lib/util/http/types.ts +++ b/lib/util/http/types.ts @@ -48,6 +48,7 @@ export interface GraphqlOptions { cursor?: string | null; acceptHeader?: string; token?: string; + readOnly?: boolean; } export interface HttpOptions { @@ -67,6 +68,7 @@ export interface HttpOptions { token?: string; memCache?: boolean; cacheProvider?: HttpCacheProvider; + readOnly?: boolean; } export interface InternalHttpOptions extends HttpOptions { diff --git a/lib/workers/repository/update/pr/changelog/github/source.ts b/lib/workers/repository/update/pr/changelog/github/source.ts index e1efda0f7350ea..c8909dad98fcec 100644 --- a/lib/workers/repository/update/pr/changelog/github/source.ts +++ b/lib/workers/repository/update/pr/changelog/github/source.ts @@ -53,6 +53,7 @@ export class GitHubChangeLogSource extends ChangeLogSource { const { token } = hostRules.find({ hostType: 'github', url, + readOnly: true, }); // istanbul ignore if if (host && !token) {