refactor(https): Separate rules lists by rate limit type (#30399)

renovatebot · Jul 26, 2024 · 6da6817 · 6da6817
1 parent 6cc830a
commit 6da6817
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 29 deletions.
diff --git a/lib/util/http/rate-limit.spec.ts b/lib/util/http/rate-limit.spec.ts
@@ -8,7 +8,7 @@ import {
 describe('util/http/rate-limit', () => {
   beforeEach(() => {
     hostRules.clear();
-    setHttpRateLimits([]);
+    setHttpRateLimits([], []);
   });
 
   describe('getConcurrentRequestsLimit', () => {
@@ -17,9 +17,10 @@ describe('util/http/rate-limit', () => {
     });
 
     it('returns null if host does not match', () => {
-      setHttpRateLimits([
-        { matchHost: 'https://crates.io/api/', throttleMs: 1000 },
-      ]);
+      setHttpRateLimits(
+        [{ matchHost: 'https://crates.io/api/', concurrency: 42 }],
+        undefined,
+      );
       expect(getConcurrentRequestsLimit('https://index.crates.io')).toBeNull();
     });
 
@@ -29,19 +30,25 @@ describe('util/http/rate-limit', () => {
     });
 
     it('selects default value if host rule is greater', () => {
-      setHttpRateLimits([{ matchHost: 'example.com', concurrency: 123 }]);
+      setHttpRateLimits(
+        [{ matchHost: 'example.com', concurrency: 123 }],
+        undefined,
+      );
       hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 456 });
       expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
     });
 
     it('selects host rule value if default is greater', () => {
-      setHttpRateLimits([{ matchHost: 'example.com', concurrency: 456 }]);
+      setHttpRateLimits(
+        [{ matchHost: 'example.com', concurrency: 456 }],
+        undefined,
+      );
       hostRules.add({ matchHost: 'example.com', concurrentRequestLimit: 123 });
       expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
     });
 
     it('matches wildcard host', () => {
-      setHttpRateLimits([{ matchHost: '*', concurrency: 123 }]);
+      setHttpRateLimits([{ matchHost: '*', concurrency: 123 }], undefined);
       expect(getConcurrentRequestsLimit('https://example.com')).toBe(123);
     });
   });
@@ -52,9 +59,10 @@ describe('util/http/rate-limit', () => {
     });
 
     it('returns null if host does not match', () => {
-      setHttpRateLimits([
-        { matchHost: 'https://crates.io/api/', concurrency: 123 },
-      ]);
+      setHttpRateLimits(
+        [{ matchHost: 'https://crates.io/api/', concurrency: 123 }],
+        undefined,
+      );
       expect(getThrottleIntervalMs('https://index.crates.io')).toBeNull();
     });
 
@@ -64,19 +72,23 @@ describe('util/http/rate-limit', () => {
     });
 
     it('selects maximum throttle when default is greater', () => {
-      setHttpRateLimits([{ matchHost: 'example.com', throttleMs: 500 }]);
+      setHttpRateLimits(undefined, [
+        { matchHost: 'example.com', throttleMs: 500 },
+      ]);
       hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 8 });
       expect(getThrottleIntervalMs('https://example.com')).toBe(500);
     });
 
     it('selects maximum throttle when host rule is greater', () => {
-      setHttpRateLimits([{ matchHost: 'example.com', throttleMs: 125 }]);
+      setHttpRateLimits(undefined, [
+        { matchHost: 'example.com', throttleMs: 125 },
+      ]);
       hostRules.add({ matchHost: 'example.com', maxRequestsPerSecond: 2 });
       expect(getThrottleIntervalMs('https://example.com')).toBe(500);
     });
 
     it('matches wildcard host', () => {
-      setHttpRateLimits([{ matchHost: '*', throttleMs: 123 }]);
+      setHttpRateLimits(undefined, [{ matchHost: '*', throttleMs: 123 }]);
       expect(getThrottleIntervalMs('https://example.com')).toBe(123);
     });
   });

diff --git a/lib/util/http/rate-limits.ts b/lib/util/http/rate-limits.ts
@@ -1,9 +1,18 @@
 import is from '@sindresorhus/is';
 import { matchesHost } from '../host-rules';
 import * as hostRules from '../host-rules';
-import type { RateLimitRule } from './types';
+import type { ConcurrencyLimitRule, ThrottleLimitRule } from './types';
 
-const defaults: RateLimitRule[] = [
+// The first match wins
+const concurrencyDefaults: ConcurrencyLimitRule[] = [
+  {
+    matchHost: '*',
+    concurrency: 16,
+  },
+];
+
+// The first match wins
+const throttleDefaults: ThrottleLimitRule[] = [
   {
     // https://guides.rubygems.org/rubygems-org-rate-limits/
     matchHost: 'rubygems.org',
@@ -14,16 +23,17 @@ const defaults: RateLimitRule[] = [
     matchHost: 'https://crates.io/api/',
     throttleMs: 1000,
   },
-  {
-    matchHost: '*',
-    concurrency: 16,
-  },
 ];
 
-let limits: RateLimitRule[] = [];
+let throttleLimits: ThrottleLimitRule[] = [];
+let concurrencyLimits: ConcurrencyLimitRule[] = [];
 
-export function setHttpRateLimits(rules?: RateLimitRule[]): void {
-  limits = rules ?? defaults;
+export function setHttpRateLimits(
+  concurrencyRules?: ConcurrencyLimitRule[],
+  throttleRules?: ThrottleLimitRule[],
+): void {
+  concurrencyLimits = concurrencyRules ?? concurrencyDefaults;
+  throttleLimits = throttleRules ?? throttleDefaults;
 }
 
 function matches(url: string, host: string): boolean {
@@ -46,8 +56,8 @@ export function getConcurrentRequestsLimit(url: string): number | null {
     result = hostRuleLimit;
   }
 
-  for (const { matchHost, concurrency: limit } of limits) {
-    if (!matches(url, matchHost) || !is.number(limit)) {
+  for (const { matchHost, concurrency: limit } of concurrencyLimits) {
+    if (!matches(url, matchHost)) {
       continue;
     }
 
@@ -70,8 +80,8 @@ export function getThrottleIntervalMs(url: string): number | null {
     result = Math.ceil(1000 / maxRequestsPerSecond);
   }
 
-  for (const { matchHost, throttleMs: limit } of limits) {
-    if (!matches(url, matchHost) || !is.number(limit)) {
+  for (const { matchHost, throttleMs: limit } of throttleLimits) {
+    if (!matches(url, matchHost)) {
       continue;
     }
 

diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
@@ -92,8 +92,12 @@ export interface HttpResponse<T = string> {
 export type Task<T> = () => Promise<T>;
 export type GotTask<T> = Task<HttpResponse<T>>;
 
-export interface RateLimitRule {
+export interface ThrottleLimitRule {
   matchHost: string;
-  throttleMs?: number;
-  concurrency?: number;
+  throttleMs: number;
+}
+
+export interface ConcurrencyLimitRule {
+  matchHost: string;
+  concurrency: number;
 }