From e8e9abcbdb3742211116f600350117fef6a32180 Mon Sep 17 00:00:00 2001 From: Lars den Bakker Date: Fri, 2 Oct 2020 22:15:53 +0200 Subject: [PATCH 1/2] feat(test-runner): add manual testing and open browser options --- .changeset/orange-carpets-jam.md | 7 +++++ .../docs/test-runner/cli-and-configuration.md | 5 ++++ .../test-runner-cli/src/cli/TestRunnerCli.ts | 28 +++++++++++++------ .../src/cli/getManualDebugMenu.ts | 2 +- .../src/config/readCliArgsConfig.ts | 11 ++++++++ .../src/config/TestRunnerCoreConfig.ts | 5 ++++ .../test-runner-core/src/runner/TestRunner.ts | 25 +++++++++++------ 7 files changed, 64 insertions(+), 19 deletions(-) create mode 100644 .changeset/orange-carpets-jam.md diff --git a/.changeset/orange-carpets-jam.md b/.changeset/orange-carpets-jam.md new file mode 100644 index 000000000..6921966c8 --- /dev/null +++ b/.changeset/orange-carpets-jam.md @@ -0,0 +1,7 @@ +--- +'@web/test-runner-cli': patch +'@web/test-runner-core': patch +'@web/test-runner': patch +--- + +added manual testing and open browser options diff --git a/docs/docs/test-runner/cli-and-configuration.md b/docs/docs/test-runner/cli-and-configuration.md index 38485b6c0..bb936f05c 100644 --- a/docs/docs/test-runner/cli-and-configuration.md +++ b/docs/docs/test-runner/cli-and-configuration.md @@ -170,6 +170,11 @@ interface TestRunnerConfig { // configuration for code coverage coverageConfig?: CoverageConfig; + /** Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL. */ + manual?: boolean; + /** Opens browser for manual testing. Requires the manual option to be set. */ + open?: boolean; + // how long a browser can take to start up before failing. defaults to 30000 browserStartTimeout?: number; // how long a test file can take to load. defaults to 10000 diff --git a/packages/test-runner-cli/src/cli/TestRunnerCli.ts b/packages/test-runner-cli/src/cli/TestRunnerCli.ts index b3bef7487..9298383ee 100644 --- a/packages/test-runner-cli/src/cli/TestRunnerCli.ts +++ b/packages/test-runner-cli/src/cli/TestRunnerCli.ts @@ -19,9 +19,10 @@ import { TestRunnerLogger } from '../logger/TestRunnerLogger'; import { BufferedLogger } from '../reporter/BufferedLogger'; import { getManualDebugMenu } from './getManualDebugMenu'; -export type MenuType = 'overview' | 'focus' | 'debug' | 'manual-debug'; +export type MenuType = 'none' | 'overview' | 'focus' | 'debug' | 'manual-debug'; export const MENUS = { + NONE: 'none' as MenuType, OVERVIEW: 'overview' as MenuType, FOCUS_SELECT_FILE: 'focus' as MenuType, DEBUG_SELECT_FILE: 'debug' as MenuType, @@ -41,7 +42,7 @@ export class TestRunnerCli { private terminal = new DynamicTerminal(); private reportedFilesByTestRun = new Map>(); private sessions: TestSessionManager; - private activeMenu: MenuType = MENUS.OVERVIEW; + private activeMenu: MenuType = MENUS.NONE; private menuSucceededAndPendingFiles: string[] = []; private menuFailedFiles: string[] = []; private testCoverage?: TestCoverage; @@ -67,9 +68,6 @@ export class TestRunnerCli { this.setupRunnerEvents(); this.terminal.start(); - if (this.config.watch) { - this.terminal.clear(); - } for (const reporter of this.config.reporters) { reporter.start?.({ @@ -81,16 +79,19 @@ export class TestRunnerCli { }); } - this.reportTestResults(); - this.reportTestProgress(); + this.switchMenu(this.config.manual ? MENUS.MANUAL_DEBUG : MENUS.OVERVIEW); - if (this.config.watch) { + if (this.config.watch || (this.config.manual && this.terminal.isInteractive)) { this.terminal.observeDirectInput(); } if (this.config.staticLogging || !this.terminal.isInteractive) { this.logger.log(chalk.bold(`Running ${this.runner.testFiles.length} test files...\n`)); } + + if (this.config.open) { + openBrowser(this.localAddress); + } } private setupTerminalEvents() { @@ -107,7 +108,12 @@ export class TestRunnerCli { case KEYCODES.CTRL_C: case KEYCODES.CTRL_D: case 'Q': - this.runner.stop(); + if ( + this.activeMenu === MENUS.OVERVIEW || + (this.config.manual && this.activeMenu === MENUS.MANUAL_DEBUG) + ) { + this.runner.stop(); + } return; case 'D': if (this.activeMenu === MENUS.OVERVIEW) { @@ -297,6 +303,10 @@ export class TestRunnerCli { } private reportTestProgress(final = false) { + if (this.config.manual) { + return; + } + const logStatic = this.config.staticLogging || !this.terminal.isInteractive; if (logStatic && !final) { // print a static progress log only once every 10000ms diff --git a/packages/test-runner-cli/src/cli/getManualDebugMenu.ts b/packages/test-runner-cli/src/cli/getManualDebugMenu.ts index fd7c15fdd..b9f7a8fc2 100644 --- a/packages/test-runner-cli/src/cli/getManualDebugMenu.ts +++ b/packages/test-runner-cli/src/cli/getManualDebugMenu.ts @@ -16,6 +16,6 @@ export function getManualDebugMenu(config: TestRunnerCoreConfig): string[] { `Network address: ${chalk.cyanBright(networkAddress)}`, ' ', `${chalk.gray('Press')} D ${chalk.gray('to open the browser.')}`, - `${chalk.gray('Press')} ESC ${chalk.gray('to exit manual debug.')}`, + `${chalk.gray('Press')} ${config.manual ? 'Q' : 'ESC'} ${chalk.gray('to exit manual debug.')}`, ].filter(_ => !!_); } diff --git a/packages/test-runner-cli/src/config/readCliArgsConfig.ts b/packages/test-runner-cli/src/config/readCliArgsConfig.ts index 87036f61f..f1888c664 100644 --- a/packages/test-runner-cli/src/config/readCliArgsConfig.ts +++ b/packages/test-runner-cli/src/config/readCliArgsConfig.ts @@ -46,6 +46,17 @@ const defaultOptions: OptionDefinition[] = [ type: Boolean, description: 'Disables rendering a progress bar dynamically to the terminal.', }, + { + name: 'manual', + type: Boolean, + description: + 'Starts test runner in manual testing mode. Ignores browsers option and prints manual testing URL.', + }, + { + name: 'open', + type: Boolean, + description: 'Opens browser for manual testing. Requires the manual option to be set.', + }, { name: 'port', type: Number, diff --git a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts index 9e928ee34..dfe922c6c 100644 --- a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts +++ b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts @@ -49,6 +49,11 @@ export interface TestRunnerCoreConfig { testsFinishTimeout: number; staticLogging?: boolean; + /** Ignores browsers option and prints manual testing URL. */ + manual?: boolean; + /** Opens browser for manual testing. Requires the manual option to be set. */ + open?: boolean; + debug?: boolean; mimeTypes?: Record; plugins?: TestRunnerPlugin[]; diff --git a/packages/test-runner-core/src/runner/TestRunner.ts b/packages/test-runner-core/src/runner/TestRunner.ts index a21b2024b..17770da02 100644 --- a/packages/test-runner-core/src/runner/TestRunner.ts +++ b/packages/test-runner-core/src/runner/TestRunner.ts @@ -44,6 +44,11 @@ export class TestRunner extends EventEmitter { groupConfigs, ); this.config = config; + + if (this.config.manual && this.config.watch) { + throw new Error('Cannot combine the manual and watch options.'); + } + this.testFiles = testFiles; this.browsers = browsers; this.browserNames = Array.from(new Set(this.browsers.map(b => b.name))); @@ -73,17 +78,19 @@ export class TestRunner extends EventEmitter { this.started = true; this.startTime = Date.now(); - for (const browser of this.browsers) { - if (browser.initialize) { - await browser.initialize(this.config, this.testFiles); - } - } - // the browser names can be updated after initialize - this.browserNames = Array.from(new Set(this.browsers.map(b => b.name))); - await this.server.start(); - this.runTests(this.sessions.all()); + if (!this.config.manual) { + for (const browser of this.browsers) { + if (browser.initialize) { + await browser.initialize(this.config, this.testFiles); + } + } + + // the browser names can be updated after initialize + this.browserNames = Array.from(new Set(this.browsers.map(b => b.name))); + this.runTests(this.sessions.all()); + } } catch (error) { this.stop(error); } From ed2c5639d47734db226be2b03d6b6379cb8525ba Mon Sep 17 00:00:00 2001 From: Lars den Bakker Date: Fri, 2 Oct 2020 22:19:09 +0200 Subject: [PATCH 2/2] feat(test-runner): improve error message when no browsers are configured --- .changeset/khaki-carpets-tan.md | 6 ++++++ .../test-runner-core/src/runner/TestRunner.ts | 16 ++++++++++++---- packages/test-runner/web-test-runner.config.mjs | 1 + 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 .changeset/khaki-carpets-tan.md diff --git a/.changeset/khaki-carpets-tan.md b/.changeset/khaki-carpets-tan.md new file mode 100644 index 000000000..c42dabbc1 --- /dev/null +++ b/.changeset/khaki-carpets-tan.md @@ -0,0 +1,6 @@ +--- +'@web/test-runner': patch +'@web/test-runner-core': patch +--- + +improve error message when no browsers are configured diff --git a/packages/test-runner-core/src/runner/TestRunner.ts b/packages/test-runner-core/src/runner/TestRunner.ts index 17770da02..aa7aa5dee 100644 --- a/packages/test-runner-core/src/runner/TestRunner.ts +++ b/packages/test-runner-core/src/runner/TestRunner.ts @@ -39,16 +39,24 @@ export class TestRunner extends EventEmitter { constructor(config: TestRunnerCoreConfig, groupConfigs: TestRunnerGroupConfig[] = []) { super(); + if (!config.manual && (!config.browsers || config.browsers.length === 0)) { + throw new Error('No browsers are configured to run tests'); + } + + if (config.manual && config.watch) { + throw new Error('Cannot combine the manual and watch options.'); + } + + if (config.open && !config.manual) { + throw new Error('The open option requires the manual option to be set.'); + } + const { sessionGroups, testFiles, testSessions, browsers } = createTestSessions( config, groupConfigs, ); this.config = config; - if (this.config.manual && this.config.watch) { - throw new Error('Cannot combine the manual and watch options.'); - } - this.testFiles = testFiles; this.browsers = browsers; this.browserNames = Array.from(new Set(this.browsers.map(b => b.name))); diff --git a/packages/test-runner/web-test-runner.config.mjs b/packages/test-runner/web-test-runner.config.mjs index 5c86f32a0..51cd6f605 100644 --- a/packages/test-runner/web-test-runner.config.mjs +++ b/packages/test-runner/web-test-runner.config.mjs @@ -2,4 +2,5 @@ export default { rootDir: '../../', preserveSymlinks: true, nodeResolve: true, + browsers: [], };