Feat/manual flag

.changeset/khaki-carpets-tan.md

@@ -0,0 +1,6 @@
+---
+'@web/test-runner': patch
+'@web/test-runner-core': patch
+---
+
+improve error message when no browsers are configured

.changeset/orange-carpets-jam.md

@@ -0,0 +1,7 @@
+---
+'@web/test-runner-cli': patch
+'@web/test-runner-core': patch
+'@web/test-runner': patch
+---
+
+added manual testing and open browser options

docs/docs/test-runner/cli-and-configuration.md

@@ -170,6 +170,11 @@ interface TestRunnerConfig {
   // configuration for code coverage
   coverageConfig?: CoverageConfig;
 
+  /** Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL. */
+  manual?: boolean;
+  /** Opens browser for manual testing. Requires the manual option to be set. */
+  open?: boolean;
+
   // how long a browser can take to start up before failing. defaults to 30000
   browserStartTimeout?: number;
   // how long a test file can take to load. defaults to 10000

packages/test-runner-cli/src/cli/TestRunnerCli.ts

@@ -19,9 +19,10 @@ import { TestRunnerLogger } from '../logger/TestRunnerLogger';
 import { BufferedLogger } from '../reporter/BufferedLogger';
 import { getManualDebugMenu } from './getManualDebugMenu';
 
-export type MenuType = 'overview' | 'focus' | 'debug' | 'manual-debug';
+export type MenuType = 'none' | 'overview' | 'focus' | 'debug' | 'manual-debug';
 
 export const MENUS = {
+  NONE: 'none' as MenuType,
   OVERVIEW: 'overview' as MenuType,
   FOCUS_SELECT_FILE: 'focus' as MenuType,
   DEBUG_SELECT_FILE: 'debug' as MenuType,
@@ -41,7 +42,7 @@ export class TestRunnerCli {
   private terminal = new DynamicTerminal();
   private reportedFilesByTestRun = new Map<number, Set<string>>();
   private sessions: TestSessionManager;
-  private activeMenu: MenuType = MENUS.OVERVIEW;
+  private activeMenu: MenuType = MENUS.NONE;
   private menuSucceededAndPendingFiles: string[] = [];
   private menuFailedFiles: string[] = [];
   private testCoverage?: TestCoverage;
@@ -67,9 +68,6 @@ export class TestRunnerCli {
     this.setupRunnerEvents();
 
     this.terminal.start();
-    if (this.config.watch) {
-      this.terminal.clear();
-    }
 
     for (const reporter of this.config.reporters) {
       reporter.start?.({
@@ -81,16 +79,19 @@ export class TestRunnerCli {
       });
     }
 
-    this.reportTestResults();
-    this.reportTestProgress();
+    this.switchMenu(this.config.manual ? MENUS.MANUAL_DEBUG : MENUS.OVERVIEW);
 
-    if (this.config.watch) {
+    if (this.config.watch || (this.config.manual && this.terminal.isInteractive)) {
       this.terminal.observeDirectInput();
     }
 
     if (this.config.staticLogging || !this.terminal.isInteractive) {
       this.logger.log(chalk.bold(`Running ${this.runner.testFiles.length} test files...\n`));
     }
+
+    if (this.config.open) {
+      openBrowser(this.localAddress);
+    }
   }
 
   private setupTerminalEvents() {
@@ -107,7 +108,12 @@ export class TestRunnerCli {
         case KEYCODES.CTRL_C:
         case KEYCODES.CTRL_D:
         case 'Q':
-          this.runner.stop();
+          if (
+            this.activeMenu === MENUS.OVERVIEW ||
+            (this.config.manual && this.activeMenu === MENUS.MANUAL_DEBUG)
+          ) {
+            this.runner.stop();
+          }
           return;
         case 'D':
           if (this.activeMenu === MENUS.OVERVIEW) {
@@ -297,6 +303,10 @@ export class TestRunnerCli {
   }
 
   private reportTestProgress(final = false) {
+    if (this.config.manual) {
+      return;
+    }
+
     const logStatic = this.config.staticLogging || !this.terminal.isInteractive;
     if (logStatic && !final) {
       // print a static progress log only once every 10000ms

packages/test-runner-cli/src/cli/getManualDebugMenu.ts

@@ -16,6 +16,6 @@ export function getManualDebugMenu(config: TestRunnerCoreConfig): string[] {
     `Network address: ${chalk.cyanBright(networkAddress)}`,
     ' ',
     `${chalk.gray('Press')} D ${chalk.gray('to open the browser.')}`,
-    `${chalk.gray('Press')} ESC ${chalk.gray('to exit manual debug.')}`,
+    `${chalk.gray('Press')} ${config.manual ? 'Q' : 'ESC'} ${chalk.gray('to exit manual debug.')}`,
   ].filter(_ => !!_);
 }

packages/test-runner-cli/src/config/readCliArgsConfig.ts

@@ -46,6 +46,17 @@ const defaultOptions: OptionDefinition[] = [
     type: Boolean,
     description: 'Disables rendering a progress bar dynamically to the terminal.',
   },
+  {
+    name: 'manual',
+    type: Boolean,
+    description:
+      'Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL.',
+  },
+  {
+    name: 'open',
+    type: Boolean,
+    description: 'Opens browser for manual testing. Requires the manual option to be set.',
+  },
   {
     name: 'port',
     type: Number,

packages/test-runner-core/src/config/TestRunnerCoreConfig.ts

@@ -49,6 +49,11 @@ export interface TestRunnerCoreConfig {
   testsFinishTimeout: number;
   staticLogging?: boolean;
 
+  /** Ignores browsers option and prints manual testing URL. */
+  manual?: boolean;
+  /** Opens browser for manual testing. Requires the manual option to be set. */
+  open?: boolean;
+
   debug?: boolean;
   mimeTypes?: Record<string, string>;
   plugins?: TestRunnerPlugin[];

packages/test-runner-core/src/runner/TestRunner.ts

@@ -39,11 +39,24 @@ export class TestRunner extends EventEmitter<EventMap> {
 
   constructor(config: TestRunnerCoreConfig, groupConfigs: TestRunnerGroupConfig[] = []) {
     super();
+    if (!config.manual && (!config.browsers || config.browsers.length === 0)) {
+      throw new Error('No browsers are configured to run tests');
+    }
+
+    if (config.manual && config.watch) {
+      throw new Error('Cannot combine the manual and watch options.');
+    }
+
+    if (config.open && !config.manual) {
+      throw new Error('The open option requires the manual option to be set.');
+    }
+
     const { sessionGroups, testFiles, testSessions, browsers } = createTestSessions(
       config,
       groupConfigs,
     );
     this.config = config;
+
     this.testFiles = testFiles;
     this.browsers = browsers;
     this.browserNames = Array.from(new Set(this.browsers.map(b => b.name)));
@@ -73,17 +86,19 @@ export class TestRunner extends EventEmitter<EventMap> {
       this.started = true;
       this.startTime = Date.now();
 
-      for (const browser of this.browsers) {
-        if (browser.initialize) {
-          await browser.initialize(this.config, this.testFiles);
-        }
-      }
-      // the browser names can be updated after initialize
-      this.browserNames = Array.from(new Set(this.browsers.map(b => b.name)));
-
       await this.server.start();
 
-      this.runTests(this.sessions.all());
+      if (!this.config.manual) {
+        for (const browser of this.browsers) {
+          if (browser.initialize) {
+            await browser.initialize(this.config, this.testFiles);
+          }
+        }
+
+        // the browser names can be updated after initialize
+        this.browserNames = Array.from(new Set(this.browsers.map(b => b.name)));
+        this.runTests(this.sessions.all());
+      }
     } catch (error) {
       this.stop(error);
     }

packages/test-runner/web-test-runner.config.mjs

@@ -2,4 +2,5 @@ export default {
   rootDir: '../../',
   preserveSymlinks: true,
   nodeResolve: true,
+  browsers: [],
 };