From 6da68172e939b313a49607917dbba24524f30f61 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 26 Jul 2024 10:27:29 -0300 Subject: [PATCH] refactor(https): Separate rules lists by rate limit type (#30399) --- lib/util/http/rate-limit.spec.ts | 38 +++++++++++++++++++++----------- lib/util/http/rate-limits.ts | 36 +++++++++++++++++++----------- lib/util/http/types.ts | 10 ++++++--- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/lib/util/http/rate-limit.spec.ts b/lib/util/http/rate-limit.spec.ts index 5fc1f75add3e38..2618b454db2175 100644 --- a/lib/util/http/rate-limit.spec.ts +++ b/lib/util/http/rate-limit.spec.ts @@ -8,7 +8,7 @@ import { describe('util/http/rate-limit', () => { beforeEach(() => { hostRules.clear(); - setHttpRateLimits([]); + setHttpRateLimits([], []); }); describe('getConcurrentRequestsLimit', () => { @@ -17,9 +17,10 @@ describe('util/http/rate-limit', () => { }); it('returns null if host does not match', () => { - setHttpRateLimits([ - { matchHost: 'https://crates.io/api/', throttleMs: 1000 }, - ]); + setHttpRateLimits( + [{ matchHost: 'https://crates.io/api/', concurrency: 42 }], + undefined, + ); expect(getConcurrentRequestsLimit('https://index.crates.io')).toBeNull(); }); @@ -29,19 +30,25 @@ describe('util/http/rate-limit', () => { }); it('selects default value if host rule is greater', () => { - setHttpRateLimits([{ matchHost: 'example.com', concurrency: 123 }]); + setHttpRateLimits( + [{ matchHost: 'example.com', concurrency: 123 }], + undefined, + ); hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 456 }); expect(getConcurrentRequestsLimit('https://example.com')).toBe(123); }); it('selects host rule value if default is greater', () => { - setHttpRateLimits([{ matchHost: 'example.com', concurrency: 456 }]); + setHttpRateLimits( + [{ matchHost: 'example.com', concurrency: 456 }], + undefined, + ); hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 123 }); expect(getConcurrentRequestsLimit('https://example.com')).toBe(123); }); it('matches wildcard host', () => { - setHttpRateLimits([{ matchHost: '*', concurrency: 123 }]); + setHttpRateLimits([{ matchHost: '*', concurrency: 123 }], undefined); expect(getConcurrentRequestsLimit('https://example.com')).toBe(123); }); }); @@ -52,9 +59,10 @@ describe('util/http/rate-limit', () => { }); it('returns null if host does not match', () => { - setHttpRateLimits([ - { matchHost: 'https://crates.io/api/', concurrency: 123 }, - ]); + setHttpRateLimits( + [{ matchHost: 'https://crates.io/api/', concurrency: 123 }], + undefined, + ); expect(getThrottleIntervalMs('https://index.crates.io')).toBeNull(); }); @@ -64,19 +72,23 @@ describe('util/http/rate-limit', () => { }); it('selects maximum throttle when default is greater', () => { - setHttpRateLimits([{ matchHost: 'example.com', throttleMs: 500 }]); + setHttpRateLimits(undefined, [ + { matchHost: 'example.com', throttleMs: 500 }, + ]); hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 8 }); expect(getThrottleIntervalMs('https://example.com')).toBe(500); }); it('selects maximum throttle when host rule is greater', () => { - setHttpRateLimits([{ matchHost: 'example.com', throttleMs: 125 }]); + setHttpRateLimits(undefined, [ + { matchHost: 'example.com', throttleMs: 125 }, + ]); hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 2 }); expect(getThrottleIntervalMs('https://example.com')).toBe(500); }); it('matches wildcard host', () => { - setHttpRateLimits([{ matchHost: '*', throttleMs: 123 }]); + setHttpRateLimits(undefined, [{ matchHost: '*', throttleMs: 123 }]); expect(getThrottleIntervalMs('https://example.com')).toBe(123); }); }); diff --git a/lib/util/http/rate-limits.ts b/lib/util/http/rate-limits.ts index ffcab7a2239ef7..c7699a240c4018 100644 --- a/lib/util/http/rate-limits.ts +++ b/lib/util/http/rate-limits.ts @@ -1,9 +1,18 @@ import is from '@sindresorhus/is'; import { matchesHost } from '../host-rules'; import * as hostRules from '../host-rules'; -import type { RateLimitRule } from './types'; +import type { ConcurrencyLimitRule, ThrottleLimitRule } from './types'; -const defaults: RateLimitRule[] = [ +// The first match wins +const concurrencyDefaults: ConcurrencyLimitRule[] = [ + { + matchHost: '*', + concurrency: 16, + }, +]; + +// The first match wins +const throttleDefaults: ThrottleLimitRule[] = [ { // https://guides.rubygems.org/rubygems-org-rate-limits/ matchHost: 'rubygems.org', @@ -14,16 +23,17 @@ const defaults: RateLimitRule[] = [ matchHost: 'https://crates.io/api/', throttleMs: 1000, }, - { - matchHost: '*', - concurrency: 16, - }, ]; -let limits: RateLimitRule[] = []; +let throttleLimits: ThrottleLimitRule[] = []; +let concurrencyLimits: ConcurrencyLimitRule[] = []; -export function setHttpRateLimits(rules?: RateLimitRule[]): void { - limits = rules ?? defaults; +export function setHttpRateLimits( + concurrencyRules?: ConcurrencyLimitRule[], + throttleRules?: ThrottleLimitRule[], +): void { + concurrencyLimits = concurrencyRules ?? concurrencyDefaults; + throttleLimits = throttleRules ?? throttleDefaults; } function matches(url: string, host: string): boolean { @@ -46,8 +56,8 @@ export function getConcurrentRequestsLimit(url: string): number | null { result = hostRuleLimit; } - for (const { matchHost, concurrency: limit } of limits) { - if (!matches(url, matchHost) || !is.number(limit)) { + for (const { matchHost, concurrency: limit } of concurrencyLimits) { + if (!matches(url, matchHost)) { continue; } @@ -70,8 +80,8 @@ export function getThrottleIntervalMs(url: string): number | null { result = Math.ceil(1000 / maxRequestsPerSecond); } - for (const { matchHost, throttleMs: limit } of limits) { - if (!matches(url, matchHost) || !is.number(limit)) { + for (const { matchHost, throttleMs: limit } of throttleLimits) { + if (!matches(url, matchHost)) { continue; } diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts index 899766bc86751f..66f260a48fb295 100644 --- a/lib/util/http/types.ts +++ b/lib/util/http/types.ts @@ -92,8 +92,12 @@ export interface HttpResponse { export type Task = () => Promise; export type GotTask = Task>; -export interface RateLimitRule { +export interface ThrottleLimitRule { matchHost: string; - throttleMs?: number; - concurrency?: number; + throttleMs: number; +} + +export interface ConcurrencyLimitRule { + matchHost: string; + concurrency: number; }