feat(platform): support configurable userAgent header via `SECUTILS…

…_WEB_SCRAPER_USER_AGENT` envvar

src/api/web_page/content/get.test.ts

@@ -5,6 +5,7 @@ import type { Browser } from 'playwright/index.js';
 
 import { registerWebPageContentGetRoutes } from './get.js';
 import type { WebPageContext } from './web_page_context.js';
+import { configure } from '../../../config.js';
 import {
   createBrowserContextMock,
   createBrowserMock,
@@ -133,7 +134,10 @@ await test('[/api/web_page/content] can inject content extractor', async (t) =>
 
   const browserMock = createBrowserMock(browserContextMock);
   const response = await registerWebPageContentGetRoutes(
-    createMock({ browser: browserMock as unknown as Browser }),
+    createMock({
+      browser: browserMock as unknown as Browser,
+      config: { ...configure(), userAgent: 'secutils/1.0.0' },
+    }),
   ).inject({
     method: 'POST',
     url: '/api/web_page/content',
@@ -162,7 +166,11 @@ await test('[/api/web_page/content] can inject content extractor', async (t) =>
 
   assert.strictEqual(browserMock.newContext.mock.callCount(), 1);
   assert.deepEqual(browserMock.newContext.mock.calls[0].arguments, [
-    { extraHTTPHeaders: { Cookie: 'my-cookie' }, bypassCSP: false },
+    {
+      extraHTTPHeaders: { Cookie: 'my-cookie' },
+      bypassCSP: false,
+      userAgent: 'secutils/1.0.0',
+    },
   ]);
   assert.strictEqual(browserContextMock.newPage.mock.callCount(), 1);
 

src/api/web_page/content/get.ts

@@ -7,6 +7,7 @@ import jsBeautify from 'js-beautify';
 import type { Browser, JSHandle, Page, Response } from 'playwright';
 
 import type { WebPageContext } from './web_page_context.js';
+import type { Config } from '../../../config.js';
 import { createObjectHash } from '../../../utilities/index.js';
 import type { ApiResult } from '../../api_result.js';
 import type { ApiRouteParams } from '../../api_route_params.js';
@@ -72,7 +73,7 @@ interface OutputBodyType {
   content: string;
 }
 
-export function registerWebPageContentGetRoutes({ server, cache, acquireBrowser }: ApiRouteParams) {
+export function registerWebPageContentGetRoutes({ server, cache, acquireBrowser, config }: ApiRouteParams) {
   return server.post<{ Body: InputBodyParamsType }>(
     '/api/web_page/content',
     {
@@ -111,7 +112,7 @@ export function registerWebPageContentGetRoutes({ server, cache, acquireBrowser
         const browser = await acquireBrowser();
         const log = server.log.child({ provider: 'web_page_content_get' });
         try {
-          const result = await getContent(browser, log, request.body);
+          const result = await getContent(config, browser, log, request.body);
           if (result.type === 'client-error') {
             log.error(`Cannot retrieve content for page "${request.body.url}" due to client error: ${result.error}`);
             await Diagnostics.screenshot(log, browser);
@@ -135,6 +136,7 @@ export function registerWebPageContentGetRoutes({ server, cache, acquireBrowser
 }
 
 async function getContent(
+  config: Config,
   browser: Browser,
   log: FastifyBaseLogger,
   {
@@ -147,7 +149,11 @@ async function getContent(
     headers,
   }: InputBodyParamsType,
 ): Promise<ApiResult<OutputBodyType>> {
-  const context = await browser.newContext({ extraHTTPHeaders: headers, bypassCSP: false });
+  const context = await browser.newContext({
+    extraHTTPHeaders: headers,
+    bypassCSP: false,
+    userAgent: config.userAgent,
+  });
   const page = await context.newPage();
 
   // Disable browser cache.

src/api/web_page/resources/list.test.ts

@@ -6,6 +6,7 @@ import type { Browser } from 'playwright/index.js';
 
 import type { WebPageResourceWithRawData } from './list.js';
 import { registerWebPageResourcesListRoutes } from './list.js';
+import { configure } from '../../../config.js';
 import {
   createBrowserContextMock,
   createBrowserMock,
@@ -275,7 +276,10 @@ await test('[/api/web_page/resources] can inject resource filters', async (t) =>
 
   const browserMock = createBrowserMock(browserContextMock);
   const response = await registerWebPageResourcesListRoutes(
-    createMock({ browser: browserMock as unknown as Browser }),
+    createMock({
+      browser: browserMock as unknown as Browser,
+      config: { ...configure(), userAgent: 'secutils/1.0.0' },
+    }),
   ).inject({
     method: 'POST',
     url: '/api/web_page/resources',
@@ -321,7 +325,7 @@ await test('[/api/web_page/resources] can inject resource filters', async (t) =>
 
   assert.strictEqual(browserMock.newContext.mock.callCount(), 1);
   assert.deepEqual(browserMock.newContext.mock.calls[0].arguments, [
-    { extraHTTPHeaders: { Cookie: 'my-cookie' }, bypassCSP: false },
+    { extraHTTPHeaders: { Cookie: 'my-cookie' }, bypassCSP: false, userAgent: 'secutils/1.0.0' },
   ]);
 
   // Make sure we didn't wait for a selector since it wasn't specified.

src/api/web_page/resources/list.ts

@@ -5,6 +5,7 @@ import type { FastifyBaseLogger } from 'fastify';
 import type { Browser, JSHandle } from 'playwright';
 
 import type { WebPageResource, WebPageResourceContent, WebPageResourceContentData } from './web_page_resource.js';
+import type { Config } from '../../../config.js';
 import { createObjectHash } from '../../../utilities/index.js';
 import type { ApiResult } from '../../api_result.js';
 import type { ApiRouteParams } from '../../api_route_params.js';
@@ -91,7 +92,7 @@ const RESOURCES_SCHEMA = {
   },
 };
 
-export function registerWebPageResourcesListRoutes({ server, cache, acquireBrowser }: ApiRouteParams) {
+export function registerWebPageResourcesListRoutes({ server, cache, acquireBrowser, config }: ApiRouteParams) {
   return server.post<{ Body: InputBodyParamsType }>(
     '/api/web_page/resources',
     {
@@ -130,7 +131,7 @@ export function registerWebPageResourcesListRoutes({ server, cache, acquireBrows
         const log = server.log.child({ provider: 'web_page_resources_list' });
 
         try {
-          const result = await getResourcesList(browser, log, request.body);
+          const result = await getResourcesList(config, browser, log, request.body);
           if (result.type === 'client-error') {
             log.error(`Cannot retrieve resources for page "${request.body.url}" due to client error: ${result.error}`);
             await Diagnostics.screenshot(log, browser);
@@ -154,11 +155,16 @@ export function registerWebPageResourcesListRoutes({ server, cache, acquireBrows
 }
 
 async function getResourcesList(
+  config: Config,
   browser: Browser,
   log: FastifyBaseLogger,
   { url, waitSelector, timeout = DEFAULT_TIMEOUT_MS, delay = DEFAULT_DELAY_MS, scripts, headers }: InputBodyParamsType,
 ): Promise<ApiResult<OutputBodyType>> {
-  const context = await browser.newContext({ extraHTTPHeaders: headers, bypassCSP: false });
+  const context = await browser.newContext({
+    extraHTTPHeaders: headers,
+    bypassCSP: false,
+    userAgent: config.userAgent,
+  });
   const page = await context.newPage();
 
   // Disable browser cache.

src/config.ts

@@ -7,6 +7,7 @@ export interface Config {
   port: number;
   cacheTTLSec: number;
   browserTTLSec: number;
+  userAgent?: string;
 }
 
 export function configure(): Config {
@@ -17,5 +18,6 @@ export function configure(): Config {
     port: +(process.env.SECUTILS_WEB_SCRAPER_PORT ?? 0) || 7272,
     cacheTTLSec: +(process.env.SECUTILS_WEB_SCRAPER_CACHE_TTL_SEC ?? 0) || 20 * 60,
     browserTTLSec: +(process.env.SECUTILS_WEB_SCRAPER_BROWSER_TTL_SEC ?? 0) || 10 * 60,
+    userAgent: process.env.SECUTILS_WEB_SCRAPER_USER_AGENT,
   };
 }

src/mocks.ts

@@ -19,6 +19,7 @@ export function createBrowserContextMock(
   return {
     newCDPSession: mock.fn(() => Promise.resolve(cdpSessionMock ?? createCDPSessionMock())),
     newPage: mock.fn(() => Promise.resolve(pageMock ?? createPageMock())),
+    close: mock.fn(),
   };
 }