refactor: Extract "HttpStats" utility class

lib/util/http/index.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/order */
 import merge from 'deepmerge';
 import got, { Options, RequestError } from 'got';
 import type { SetRequired } from 'type-fest';
@@ -26,10 +27,10 @@ import type {
   HttpRequestOptions,
   HttpResponse,
   InternalHttpOptions,
-  RequestStats,
 } from './types';
 // TODO: refactor code to remove this (#9651)
 import './legacy';
+import { type HttpRequestStatsDataPoint, HttpStats } from '../stats';
 
 export { RequestError as HttpError };
 
@@ -76,6 +77,8 @@ function applyDefaultHeaders(options: Options): void {
   };
 }
 
+type QueueStatsData = Pick<HttpRequestStatsDataPoint, 'queueMs'>;
+
 // Note on types:
 // options.requestType can be either 'json' or 'buffer', but `T` should be
 // `Buffer` in the latter case.
@@ -84,7 +87,7 @@ function applyDefaultHeaders(options: Options): void {
 async function gotTask<T>(
   url: string,
   options: SetRequired<GotOptions, 'method'>,
-  requestStats: Omit<RequestStats, 'duration' | 'statusCode'>,
+  queueStats: QueueStatsData,
 ): Promise<HttpResponse<T>> {
   logger.trace({ url, options }, 'got request');
 
@@ -119,9 +122,13 @@ async function gotTask<T>(
 
     throw error;
   } finally {
-    const httpRequests = memCache.get<RequestStats[]>('http-requests') || [];
-    httpRequests.push({ ...requestStats, duration, statusCode });
-    memCache.set('http-requests', httpRequests);
+    HttpStats.write({
+      method: options.method,
+      url,
+      reqMs: duration,
+      queueMs: queueStats.queueMs,
+      status: statusCode,
+    });
   }
 }
 
@@ -236,12 +243,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
       }
       const startTime = Date.now();
       const httpTask: GotTask<T> = () => {
-        const queueDuration = Date.now() - startTime;
-        return gotTask(url, options, {
-          method: options.method,
-          url,
-          queueDuration,
-        });
+        const queueMs = Date.now() - startTime;
+        return gotTask(url, options, { queueMs });
       };
 
       const throttle = this.getThrottle(url);

lib/util/stats.spec.ts

@@ -1,6 +1,11 @@
 import { logger } from '../../test/util';
 import * as memCache from './cache/memory';
-import { LookupStats, PackageCacheStats, makeTimingReport } from './stats';
+import {
+  HttpStats,
+  LookupStats,
+  PackageCacheStats,
+  makeTimingReport,
+} from './stats';
 
 describe('util/stats', () => {
   beforeEach(() => {
@@ -223,4 +228,215 @@ describe('util/stats', () => {
       });
     });
   });
+
+  describe('HttpStats', () => {
+    it('returns empty report', () => {
+      const res = HttpStats.getReport();
+      expect(res).toEqual({
+        allRequests: [],
+        requestsByHost: {},
+        statsByHost: {},
+        totalRequests: 0,
+        urlCounts: {},
+      });
+    });
+
+    it('writes data points', () => {
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 100,
+        queueMs: 10,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 200,
+        queueMs: 20,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/bar',
+        reqMs: 400,
+        queueMs: 40,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 800,
+        queueMs: 80,
+        status: 404,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: '<invalid>',
+        reqMs: 100,
+        queueMs: 100,
+        status: 400,
+      });
+
+      const res = HttpStats.getReport();
+
+      expect(res).toEqual({
+        allRequests: [
+          'GET https://example.com/bar 200 400 40',
+          'GET https://example.com/foo 200 100 10',
+          'GET https://example.com/foo 200 200 20',
+          'GET https://example.com/foo 404 800 80',
+        ],
+        requestsByHost: {
+          'example.com': [
+            {
+              method: 'GET',
+              queueMs: 40,
+              reqMs: 400,
+              status: 200,
+              url: 'https://example.com/bar',
+            },
+            {
+              method: 'GET',
+              queueMs: 10,
+              reqMs: 100,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 20,
+              reqMs: 200,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 80,
+              reqMs: 800,
+              status: 404,
+              url: 'https://example.com/foo',
+            },
+          ],
+        },
+        statsByHost: {
+          'example.com': {
+            count: 4,
+            queueAvgMs: 38,
+            queueMaxMs: 80,
+            queueMedianMs: 40,
+            reqAvgMs: 375,
+            reqMaxMs: 800,
+            reqMedianMs: 400,
+          },
+        },
+        totalRequests: 4,
+        urlCounts: {
+          'https://example.com/bar (GET, 200)': 1,
+          'https://example.com/foo (GET, 200)': 2,
+          'https://example.com/foo (GET, 404)': 1,
+        },
+      });
+    });
+
+    it('logs report', () => {
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 100,
+        queueMs: 10,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 200,
+        queueMs: 20,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/bar',
+        reqMs: 400,
+        queueMs: 40,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 800,
+        queueMs: 80,
+        status: 404,
+      });
+
+      HttpStats.report();
+
+      expect(logger.logger.trace).toHaveBeenCalledTimes(1);
+      const [traceData, traceMsg] = logger.logger.trace.mock.calls[0];
+      expect(traceMsg).toBe('HTTP full stats');
+      expect(traceData).toEqual({
+        allRequests: [
+          'GET https://example.com/bar 200 400 40',
+          'GET https://example.com/foo 200 100 10',
+          'GET https://example.com/foo 200 200 20',
+          'GET https://example.com/foo 404 800 80',
+        ],
+        requestsByHost: {
+          'example.com': [
+            {
+              method: 'GET',
+              queueMs: 40,
+              reqMs: 400,
+              status: 200,
+              url: 'https://example.com/bar',
+            },
+            {
+              method: 'GET',
+              queueMs: 10,
+              reqMs: 100,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 20,
+              reqMs: 200,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 80,
+              reqMs: 800,
+              status: 404,
+              url: 'https://example.com/foo',
+            },
+          ],
+        },
+      });
+
+      expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+      const [debugData, debugMsg] = logger.logger.debug.mock.calls[0];
+      expect(debugMsg).toBe('HTTP stats');
+      expect(debugData).toEqual({
+        statsByHost: {
+          'example.com': {
+            count: 4,
+            queueAvgMs: 38,
+            queueMaxMs: 80,
+            queueMedianMs: 40,
+            reqAvgMs: 375,
+            reqMaxMs: 800,
+            reqMedianMs: 400,
+          },
+        },
+        totalRequests: 4,
+        urlCounts: {
+          'https://example.com/bar (GET, 200)': 1,
+          'https://example.com/foo (GET, 200)': 2,
+          'https://example.com/foo (GET, 404)': 1,
+        },
+      });
+    });
+  });
 });