feat: config.botName for describing environment in the reports (#28507)

Reference #27284

docs/src/test-api/class-testconfig.md

@@ -17,6 +17,23 @@ export default defineConfig({
 });
 ```
 
+## property: TestConfig.botName
+* since: v1.41
+- type: ?<[string]>
+
+Unique name of the environment where the tests run. It may be composed of, e.g., operating system name and
+test run parameters. Test reporters can access the name via `TestProject.botName` property.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  botName: process.env.BOT_NAME,
+});
+```
+
 ## property: TestConfig.build
 * since: v1.35
 - type: ?<[Object]>

packages/html-reporter/src/filter.ts

@@ -108,7 +108,7 @@ export class Filter {
       if (test.outcome === 'skipped')
         status = 'skipped';
       const searchValues: SearchValues = {
-        text: (status + ' ' + test.projectName + ' ' + (test.reportName || '') + ' ' + test.location.file + ' ' + test.path.join(' ') + ' ' + test.title).toLowerCase(),
+        text: (status + ' ' + test.projectName + ' ' + (test.botName || '') + ' ' + test.location.file + ' ' + test.path.join(' ') + ' ' + test.title).toLowerCase(),
         project: test.projectName.toLowerCase(),
         status: status as any,
         file: test.location.file,

packages/html-reporter/src/labelUtils.tsx

@@ -27,8 +27,8 @@ export function escapeRegExp(string: string) {
 
 export function testCaseLabels(test: TestCaseSummary): string[] {
   const tags = matchTags(test.path.join(' ') + ' ' + test.title).sort((a, b) => a.localeCompare(b));
-  if (test.reportName)
-    tags.unshift(test.reportName);
+  if (test.botName)
+    tags.unshift(test.botName);
   return tags;
 }
 

packages/html-reporter/src/types.ts

@@ -66,7 +66,7 @@ export type TestCaseSummary = {
   title: string;
   path: string[];
   projectName: string;
-  reportName?: string;
+  botName?: string;
   location: Location;
   annotations: TestCaseAnnotation[];
   outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';

packages/playwright/src/common/config.ts

@@ -168,6 +168,7 @@ export class FullProjectInternal {
     this.snapshotPathTemplate = takeFirst(projectConfig.snapshotPathTemplate, config.snapshotPathTemplate, defaultSnapshotPathTemplate);
 
     this.project = {
+      botName: config.botName,
       grep: takeFirst(projectConfig.grep, config.grep, defaultGrep),
       grepInvert: takeFirst(projectConfig.grepInvert, config.grepInvert, null),
       outputDir: takeFirst(configCLIOverrides.outputDir, pathResolve(configDir, projectConfig.outputDir), pathResolve(configDir, config.outputDir), path.join(throwawayArtifactsPath, 'test-results')),

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -41,6 +41,7 @@ export type JsonPattern = {
 
 export type JsonProject = {
   id: string;
+  botName?: string;
   grep: JsonPattern[];
   grepInvert: JsonPattern[];
   metadata: Metadata;
@@ -334,6 +335,7 @@ export class TeleReporterReceiver {
   private _parseProject(project: JsonProject): TeleFullProject {
     return {
       __projectId: project.id,
+      botName: project.botName,
       metadata: project.metadata,
       name: project.name,
       outputDir: this._absolutePath(project.outputDir),

packages/playwright/src/reporters/blob.ts

@@ -58,7 +58,7 @@ export class BlobReporter extends TeleReporterEmitter {
     const metadata: BlobReportMetadata = {
       version: currentBlobReportVersion,
       userAgent: getUserAgent(),
-      name: process.env.PWTEST_BLOB_REPORT_NAME,
+      name: config.botName || process.env.PWTEST_BLOB_REPORT_NAME,
       shard: config.shard ?? undefined,
       pathSeparator: path.sep,
     };

packages/playwright/src/reporters/html.ts

@@ -240,7 +240,7 @@ class HtmlBuilder {
         }
         const { testFile, testFileSummary } = fileEntry;
         const testEntries: TestEntry[] = [];
-        this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, projectSuite.project()!.metadata?.reportName, [], testEntries);
+        this._processJsonSuite(fileSuite, fileId, projectSuite.project()!.name, projectSuite.project()!.botName, [], testEntries);
         for (const test of testEntries) {
           testFile.tests.push(test.testCase);
           testFileSummary.tests.push(test.testCaseSummary);
@@ -340,13 +340,13 @@ class HtmlBuilder {
     this._dataZipFile.addBuffer(Buffer.from(JSON.stringify(data)), fileName);
   }
 
-  private _processJsonSuite(suite: Suite, fileId: string, projectName: string, reportName: string | undefined, path: string[], outTests: TestEntry[]) {
+  private _processJsonSuite(suite: Suite, fileId: string, projectName: string, botName: string | undefined, path: string[], outTests: TestEntry[]) {
     const newPath = [...path, suite.title];
-    suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, reportName, newPath, outTests));
-    suite.tests.forEach(t => outTests.push(this._createTestEntry(t, projectName, reportName, newPath)));
+    suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, botName, newPath, outTests));
+    suite.tests.forEach(t => outTests.push(this._createTestEntry(t, projectName, botName, newPath)));
   }
 
-  private _createTestEntry(test: TestCasePublic, projectName: string, reportName: string | undefined, path: string[]): TestEntry {
+  private _createTestEntry(test: TestCasePublic, projectName: string, botName: string | undefined, path: string[]): TestEntry {
     const duration = test.results.reduce((a, r) => a + r.duration, 0);
     const location = this._relativeLocation(test.location)!;
     path = path.slice(1);
@@ -358,7 +358,7 @@ class HtmlBuilder {
         testId: test.id,
         title: test.title,
         projectName,
-        reportName,
+        botName,
         location,
         duration,
         annotations: test.annotations,
@@ -371,7 +371,7 @@ class HtmlBuilder {
         testId: test.id,
         title: test.title,
         projectName,
-        reportName,
+        botName,
         location,
         duration,
         annotations: test.annotations,

packages/playwright/src/reporters/teleEmitter.ts

@@ -162,6 +162,7 @@ export class TeleReporterEmitter implements ReporterV2 {
     const project = suite.project()!;
     const report: JsonProject = {
       id: getProjectId(project),
+      botName: project.botName,
       metadata: project.metadata,
       name: project.name,
       outputDir: this._relativePath(project.outputDir),

packages/playwright/types/test.d.ts

@@ -161,6 +161,7 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends TestProject {
  *
  */
 export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
+  botName?: string;
   /**
    * Filter to only run tests with a title matching one of the patterns. For example, passing `grep: /cart/` should only
    * run tests with "cart" in the title. Also available globally and in the [command line](https://playwright.dev/docs/test-cli) with the `-g`
@@ -570,6 +571,24 @@ interface TestConfig {
    *
    */
   webServer?: TestConfigWebServer | TestConfigWebServer[];
+  /**
+   * Unique name of the environment where the tests run. It may be composed of, e.g., operating system name and test run
+   * parameters. Test reporters can access the name via `TestProject.botName` property.
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   botName: process.env.BOT_NAME,
+   * });
+   * ```
+   *
+   */
+  botName?: string;
+
   /**
    * Playwright transpiler configuration.
    *
@@ -1451,6 +1470,23 @@ export type Metadata = { [key: string]: any };
  *
  */
 export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
+  /**
+   * Unique name of the environment where the tests run. It may be composed of, e.g., operating system name and test run
+   * parameters. Test reporters can access the name via `TestProject.botName` property.
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   botName: process.env.BOT_NAME,
+   * });
+   * ```
+   *
+   */
+  botName?: string;
   /**
    * Whether to exit with an error if any tests or groups are marked as
    * [test.only(title, testFunction)](https://playwright.dev/docs/api/class-test#test-only) or

tests/playwright-test/reporter-blob.spec.ts

@@ -1211,6 +1211,45 @@ test('same project different suffixes', async ({ runInlineTest, mergeReports })
   expect(output).toContain(`reportNames: first,second`);
 });
 
+test('preserve botName on projects', async ({ runInlineTest, mergeReports }) => {
+  const files = (botName: string) => ({
+    'echo-reporter.js': `
+      import fs from 'fs';
+
+      class EchoReporter {
+        onBegin(config, suite) {
+          const projects = suite.suites.map(s => s.project()).sort((a, b) => a.metadata.reportName.localeCompare(b.metadata.reportName));
+          console.log('projectNames: ' + projects.map(p => p.name));
+          console.log('botNames: ' + projects.map(p => p.botName));
+        }
+      }
+      module.exports = EchoReporter;
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: 'blob',
+        botName: '${botName}',
+        projects: [
+          { name: 'foo' },
+        ]
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 1 @smoke', async ({}) => {});
+    `,
+  });
+
+  await runInlineTest(files('first'), undefined, { PWTEST_BLOB_REPORT_NAME: 'first' });
+  await runInlineTest(files('second'), undefined, { PWTEST_BLOB_REPORT_NAME: 'second', PWTEST_BLOB_DO_NOT_REMOVE: '1' });
+
+  const reportDir = test.info().outputPath('blob-report');
+  const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] });
+  expect(exitCode).toBe(0);
+  expect(output).toContain(`projectNames: foo,foo`);
+  expect(output).toContain(`botNames: first,second`);
+});
+
 test('no reports error', async ({ runInlineTest, mergeReports }) => {
   const reportDir = test.info().outputPath('blob-report');
   fs.mkdirSync(reportDir, { recursive: true });

utils/generate_types/index.js

@@ -113,7 +113,7 @@ class TypesGenerator {
         return '';
       this.handledMethods.add(`${className}.${methodName}#${overloadIndex}`);
       if (!method) {
-        if (new Set(['on', 'addListener', 'off', 'removeListener', 'once', 'prependListener']).has(methodName))
+        if (new Set(['on', 'addListener', 'off', 'removeListener', 'once', 'prependListener', 'botName']).has(methodName))
           return '';
         throw new Error(`Unknown override method "${className}.${methodName}"`);
       }

utils/generate_types/overrides-test.d.ts

@@ -39,6 +39,7 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends TestProject {
 // [internal] It is part of the public API and is computed from the user's config.
 // [internal] If you need new fields internally, add them to FullProjectInternal instead.
 export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
+  botName?: string;
   grep: RegExp | RegExp[];
   grepInvert: RegExp | RegExp[] | null;
   metadata: Metadata;
@@ -75,6 +76,7 @@ export type Metadata = { [key: string]: any };
 // [internal] It is part of the public API and is computed from the user's config.
 // [internal] If you need new fields internally, add them to FullConfigInternal instead.
 export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
+  botName?: string;
   forbidOnly: boolean;
   fullyParallel: boolean;
   globalSetup: string | null;