From ae25defa3be4861361028fb43ca888bca6dfe98e Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Tue, 5 Sep 2023 12:55:43 -0400 Subject: [PATCH 01/25] report correct number of failures if browser crashes in the middle of a test suite --- packages/server/lib/modes/run.ts | 65 +++++++++++++++---- packages/server/lib/project-base.ts | 13 +++- .../browser_crash_handling_spec.js | 32 +++++---- .../e2e/cypress/e2e/chrome_tab_crash.cy.js | 24 ++++++- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 95558d72a2ee..2f0b88a61446 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -45,7 +45,7 @@ let earlyExitErr: Error let currentSetScreenshotMetadata: SetScreenshotMetadata const debug = Debug('cypress:server:run') - +const eDebug = Debug('cypress:server:run:crash_handling') const DELAY_TO_LET_VIDEO_FINISH_MS = 1000 const relativeSpecPattern = (projectRoot, pattern) => { @@ -411,9 +411,37 @@ function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, return openProject.launch(browser, spec, browserOpts) } +interface ReporterResults { + error?: string + stats: { + failures: number + tests: number + passes: number + pending: number + suites: number + skipped: number + wallClockDuration: number + wallClockStartedAt: string + wallClockEndedAt: string + } + reporter: string + reporterStats: { + suites: number + tests: number + passes: number + pending: number + failures: number + start: string + } + hooks: any[] + tests: any[] +} + function listenForProjectEnd (project, exit): Bluebird { if (globalThis.CY_TEST_MOCK?.listenForProjectEnd) return Bluebird.resolve(globalThis.CY_TEST_MOCK.listenForProjectEnd) + let intermediateStats: ReporterResults | undefined + return new Bluebird((resolve, reject) => { if (exit === false) { resolve = () => { @@ -422,6 +450,7 @@ function listenForProjectEnd (project, exit): Bluebird { } const onEarlyExit = function (err) { + eDebug('onEarlyExit err %O', err) if (err.isFatalApiErr) { return reject(err) } @@ -429,25 +458,35 @@ function listenForProjectEnd (project, exit): Bluebird { console.log('') errors.log(err) - const obj = { - error: errors.stripAnsi(err.message), + // in crash situations, the most recent report will not have the triggering test + // so the results are manually patched, which produces the expected exit=1 and + // terminal output indicating the failed test + const results = { + ...intermediateStats, stats: { - failures: 1, - tests: 0, - passes: 0, - pending: 0, - suites: 0, - skipped: 0, - wallClockDuration: 0, - wallClockStartedAt: new Date().toJSON(), - wallClockEndedAt: new Date().toJSON(), + ...intermediateStats?.stats, + failures: (intermediateStats?.stats?.failures ?? 0) + 1, + skipped: (intermediateStats?.stats?.skipped ?? 1) - 1, + }, + reporterStats: { + ...intermediateStats?.reporterStats, + failures: (intermediateStats?.reporterStats?.failures ?? 0) + 1, }, + error: errors.stripAnsi(err.message), } - return resolve(obj) + debug('patched reporter results: %O', results) + + return resolve(results) } project.once('end', (results) => resolve(results)) + project.on('test:before:run', (results) => { + intermediateStats = results + results.tests.forEach((t) => { + debug('test in results: %O', t) + }) + }) // if we already received a reason to exit early, go ahead and do it if (earlyExitErr) { diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index f9ac70da24e9..98dc568627ac 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -49,6 +49,7 @@ export interface Cfg extends ReceivedCypressOptions { const localCwd = process.cwd() const debug = Debug('cypress:server:project') +const eDebug = Debug('cypress:server:project:emit') type StartWebsocketOptions = Pick @@ -114,6 +115,12 @@ export class ProjectBase extends EE { } } + emit (eventName, ...args) { + eDebug('Project emit %O', eventName, args) + + return super.emit(eventName, ...args) + } + protected ensureProp = ensureProp setOnTestsReceived (fn) { @@ -376,7 +383,7 @@ export class ProjectBase extends EE { }, onMocha: async (event, runnable) => { - debug('onMocha', event) + eDebug('onMocha', event) // bail if we dont have a // reporter instance if (!reporterInstance) { @@ -385,7 +392,9 @@ export class ProjectBase extends EE { reporterInstance.emit(event, runnable) - if (event === 'end') { + if (event === 'test:before:run') { + this.emit('test:before:run', reporterInstance?.results() || {}) + } else if (event === 'end') { const [stats = {}] = await Promise.all([ (reporterInstance != null ? reporterInstance.end() : undefined), this.server.end(), diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index 296a317f8436..d03a6ec8fa97 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -17,6 +17,8 @@ exports['Browser Crash Handling / when the tab crashes in chrome / fails'] = ` Running: chrome_tab_crash.cy.js (1 of 2) + a test suite with a browser crash + ✓ navigates to about:blank We detected that the Chrome Renderer process just crashed. @@ -36,8 +38,8 @@ https://on.cypress.io/renderer-process-crashed (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ + │ Tests: 2 │ + │ Passing: 1 │ │ Failing: 1 │ │ Pending: 0 │ │ Skipped: 0 │ @@ -80,11 +82,11 @@ https://on.cypress.io/renderer-process-crashed Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_tab_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_tab_crash.cy.js XX:XX 2 1 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 3 2 1 - - ` @@ -108,6 +110,8 @@ exports['Browser Crash Handling / when the tab crashes in electron / fails'] = ` Running: chrome_tab_crash.cy.js (1 of 2) + a test suite with a browser crash + ✓ navigates to about:blank We detected that the Electron Renderer process just crashed. @@ -127,8 +131,8 @@ https://on.cypress.io/renderer-process-crashed (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ - │ Passing: 0 │ + │ Tests: 2 │ + │ Passing: 1 │ │ Failing: 1 │ │ Pending: 0 │ │ Skipped: 0 │ @@ -171,11 +175,11 @@ https://on.cypress.io/renderer-process-crashed Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_tab_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_tab_crash.cy.js XX:XX 2 1 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 3 2 1 - - ` @@ -215,7 +219,7 @@ This can happen for many different reasons: (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ + │ Tests: 1 │ │ Passing: 0 │ │ Failing: 1 │ │ Pending: 0 │ @@ -259,11 +263,11 @@ This can happen for many different reasons: Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_process_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_process_crash.cy.js XX:XX 1 - 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 2 1 1 - - ` @@ -303,7 +307,7 @@ This can happen for many different reasons: (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 0 │ + │ Tests: 1 │ │ Passing: 0 │ │ Failing: 1 │ │ Pending: 0 │ @@ -357,11 +361,11 @@ This can happen for many different reasons: Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ chrome_process_crash.cy.js XX:XX - - 1 - - │ + │ ✖ chrome_process_crash.cy.js XX:XX 1 - 1 - - │ ├────────────────────────────────────────────────────────────────────────────────────────────────┤ │ ✔ simple.cy.js XX:XX 1 1 - - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 2 failed (50%) XX:XX 1 1 1 - - + ✖ 1 of 2 failed (50%) XX:XX 2 1 1 - - ` diff --git a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js index 77fc85a8a850..7bb713c86a7e 100644 --- a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js +++ b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js @@ -1,4 +1,22 @@ -it('crashes the chrome tab', () => { - Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) - cy.visit('localhost') +describe('a test suite with a browser crash', function () { + let deferred; let proceed + + before(() => { + deferred = new Promise((res) => proceed = res) + }) + + it('navigates to about:blank', () => { + cy.visit('/index.html').then(() => { + proceed() + }) + }) + + it('crashes the chrome tab', () => { + // make exec of this one dependent on prev, to ensure linear execution of tests for predictable results + cy.wrap(deferred).then((() => { + Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) + + cy.visit('localhost') + })) + }) }) From 8d67daac18816fac9238873dca428d3ef682d678 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Wed, 6 Sep 2023 14:24:48 -0400 Subject: [PATCH 02/25] report failure correctly when browser crashes during test --- packages/server/lib/modes/run.ts | 62 ++++++++- packages/server/lib/project-base.ts | 5 +- system-tests/__snapshots__/record_spec.js | 124 ++++++++++++++++++ .../e2e/cypress/e2e/chrome_tab_crash.cy.js | 14 +- system-tests/test/record_spec.js | 25 ++++ 5 files changed, 217 insertions(+), 13 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 2f0b88a61446..cacba8404f67 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -411,6 +411,23 @@ function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, return openProject.launch(browser, spec, browserOpts) } +interface ReporterTestAttempt { + state: 'skipped' | 'failed' | 'passed' + error: any + timings: any + failedFromHookId: any + wallClockStartedAt: Date + wallClockDuration: number + videoTimestamp: any +} +interface ReporterTest { + testId: string + title: string[] + state: 'skipped' | 'passed' | 'failed' + body: string + displayError: any + attempts: ReporterTestAttempt[] +} interface ReporterResults { error?: string stats: { @@ -432,9 +449,11 @@ interface ReporterResults { pending: number failures: number start: string + end: string + duration: number } hooks: any[] - tests: any[] + tests: ReporterTest[] } function listenForProjectEnd (project, exit): Bluebird { @@ -458,6 +477,11 @@ function listenForProjectEnd (project, exit): Bluebird { console.log('') errors.log(err) + const endTime: number = intermediateStats?.stats?.wallClockEndedAt ? Date.parse(intermediateStats?.stats?.wallClockEndedAt) : new Date().getTime() + const duration = intermediateStats?.stats?.wallClockStartedAt ? + endTime - Date.parse(intermediateStats.stats.wallClockStartedAt) : 0 + const endTimeStamp = new Date(endTime).toJSON() + // in crash situations, the most recent report will not have the triggering test // so the results are manually patched, which produces the expected exit=1 and // terminal output indicating the failed test @@ -465,13 +489,34 @@ function listenForProjectEnd (project, exit): Bluebird { ...intermediateStats, stats: { ...intermediateStats?.stats, + wallClockEndedAt: endTimeStamp, + duration, failures: (intermediateStats?.stats?.failures ?? 0) + 1, skipped: (intermediateStats?.stats?.skipped ?? 1) - 1, }, reporterStats: { ...intermediateStats?.reporterStats, + tests: (intermediateStats?.reporterStats?.tests ?? 0) + 1, // crashed test does not increment this value + end: intermediateStats?.reporterStats?.end || endTimeStamp, + duration, failures: (intermediateStats?.reporterStats?.failures ?? 0) + 1, }, + tests: (intermediateStats?.tests || []).map((test) => { + if (test.testId === pendingRunnable.id) { + return { + ...test, + state: 'failed', + attempts: test.attempts.map((attempt) => { + return { + ...attempt, + state: 'failed', + } + }), + } + } + + return test + }), error: errors.stripAnsi(err.message), } @@ -480,11 +525,18 @@ function listenForProjectEnd (project, exit): Bluebird { return resolve(results) } + let pendingRunnable: any + project.once('end', (results) => resolve(results)) - project.on('test:before:run', (results) => { - intermediateStats = results - results.tests.forEach((t) => { - debug('test in results: %O', t) + project.on('test:before:run', ({ + runnable, + previousResults, + }) => { + intermediateStats = previousResults + eDebug('pending runnable: %O', runnable) + pendingRunnable = runnable + previousResults.tests.forEach((t) => { + eDebug('test in results: %O', t) }) }) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 98dc568627ac..7a069944139c 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -393,7 +393,10 @@ export class ProjectBase extends EE { reporterInstance.emit(event, runnable) if (event === 'test:before:run') { - this.emit('test:before:run', reporterInstance?.results() || {}) + this.emit('test:before:run', { + runnable, + previousResults: reporterInstance?.results() || {}, + }) } else if (event === 'end') { const [stats = {}] = await Promise.all([ (reporterInstance != null ? reporterInstance.end() : undefined), diff --git a/system-tests/__snapshots__/record_spec.js b/system-tests/__snapshots__/record_spec.js index a592cf617c90..bfd8c743aaa9 100644 --- a/system-tests/__snapshots__/record_spec.js +++ b/system-tests/__snapshots__/record_spec.js @@ -3631,3 +3631,127 @@ exports['e2e record capture-protocol enabled protocol runtime errors error in pr ` + +exports['e2e record capture-protocol enabled when the tab crashes in chrome still uploads a test replay 1'] = ` + +==================================================================================================== + + (Run Starting) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Cypress: 1.2.3 │ + │ Browser: FooBrowser 88 │ + │ Specs: 2 found (chrome_tab_crash.cy.js, simple.cy.js) │ + │ Searched: cypress/e2e/chrome_tab_crash.cy.js, cypress/e2e/simple.cy.js │ + │ Params: Tag: false, Group: false, Parallel: false │ + │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: chrome_tab_crash.cy.js (1 of 2) + Estimated: X second(s) + + + a test suite with a browser crash + ✓ navigates to about:blank + +We detected that the Chrome Renderer process just crashed. + +We have failed the current spec but will continue running the next spec. + +This can happen for a number of different reasons. + +If you're running lots of tests on a memory intense application. + - Try increasing the CPU/memory on the machine you're running on. + - Try enabling experimentalMemoryManagement in your config file. + - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. + +You can learn more here: + +https://on.cypress.io/renderer-process-crashed + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: chrome_tab_crash.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay - 1 kB + + (Uploaded Cloud Artifacts) + + - Test Replay - Done Uploading 1 kB 1/1 + +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: simple.cy.js (2 of 2) + Estimated: X second(s) + + + ✓ is true + + 1 passing + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 1 │ + │ Passing: 1 │ + │ Failing: 0 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 0 │ + │ Video: false │ + │ Duration: X seconds │ + │ Estimated: X second(s) │ + │ Spec Ran: simple.cy.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Uploading Cloud Artifacts) + + - Video - Nothing to upload + - Screenshot - Nothing to upload + - Test Replay + + (Uploaded Cloud Artifacts) + + - Test Replay - Done Uploading 1 kB 1/1 + +==================================================================================================== + + (Run Finished) + + + Spec Tests Passing Failing Pending Skipped + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ ✖ chrome_tab_crash.cy.js XX:XX 2 1 1 - - │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✔ simple.cy.js XX:XX 1 1 - - - │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + ✖ 1 of 2 failed (50%) XX:XX 3 2 1 - - + + +─────────────────────────────────────────────────────────────────────────────────────────────────────── + + Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 + + +` diff --git a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js index 7bb713c86a7e..ae5edc1afd37 100644 --- a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js +++ b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js @@ -1,22 +1,22 @@ describe('a test suite with a browser crash', function () { - let deferred; let proceed + // let deferred; let proceed before(() => { - deferred = new Promise((res) => proceed = res) + // deferred = new Promise((res) => proceed = res) }) it('navigates to about:blank', () => { cy.visit('/index.html').then(() => { - proceed() + // proceed() }) }) it('crashes the chrome tab', () => { // make exec of this one dependent on prev, to ensure linear execution of tests for predictable results - cy.wrap(deferred).then((() => { - Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) + // cy.wrap(deferred).then((() => { + Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) - cy.visit('localhost') - })) + cy.visit('localhost') + // })) }) }) diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index 6bbdf75c937f..c91d8bc7ff05 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -2308,6 +2308,31 @@ describe('e2e record', () => { }) }) + describe('when the tab crashes in chrome', () => { + enableCaptureProtocol() + it('still uploads a test replay', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + browser: 'chrome', + spec: 'chrome_tab_crash.cy.js,simple.cy.js', + record: true, + snapshot: true, + expectedExitCode: 1, + }).then(() => { + const urls = getRequestUrls() + const requests = getRequests() + const postResultsRequest = requests.find((r) => r.url === `POST /instances/${instanceId}/results`) + + console.log(JSON.stringify(postResultsRequest, null, 2)) + + console.log(getRequests()) + + expect(urls).to.include.members([`PUT ${CAPTURE_PROTOCOL_UPLOAD_URL}`]) + }) + }) + }) + describe('protocol runtime errors', () => { enableCaptureProtocol() describe('db size too large', () => { From 5b46d7e21fbf1b9f5ca7c933c50b5573089bf175 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Mon, 11 Sep 2023 10:26:41 -0400 Subject: [PATCH 03/25] refactor crash handling --- packages/server/lib/modes/run.ts | 156 ++---------------- .../lib/util/graceful_crash_handling.ts | 152 +++++++++++++++++ 2 files changed, 164 insertions(+), 144 deletions(-) create mode 100644 packages/server/lib/util/graceful_crash_handling.ts diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index cacba8404f67..e3873a3d4b17 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -21,12 +21,13 @@ import random from '../util/random' import system from '../util/system' import chromePolicyCheck from '../util/chrome_policy_check' import type { SpecWithRelativeRoot, SpecFile, TestingType, OpenProjectLaunchOpts, FoundBrowser, BrowserVideoController, VideoRecording, ProcessOptions } from '@packages/types' -import type { Cfg } from '../project-base' +import type { Cfg, ProjectBase } from '../project-base' import type { Browser } from '../browsers/types' import * as printResults from '../util/print-run' import type { ProtocolManager } from '../cloud/protocol' import { telemetry } from '@packages/telemetry' import { CypressRunResult, createPublicBrowser, createPublicConfig, createPublicRunResults, createPublicSpec, createPublicSpecResults } from './results' +import { endAfterError, exitEarly } from '../util/graceful_crash_handling' type SetScreenshotMetadata = (data: TakeScreenshotProps) => void type ScreenshotMetadata = ReturnType @@ -36,16 +37,9 @@ type BeforeSpecRun = any type AfterSpecRun = any type Project = NonNullable> -let exitEarly = (err) => { - debug('set early exit error: %s', err.stack) - - earlyExitErr = err -} -let earlyExitErr: Error let currentSetScreenshotMetadata: SetScreenshotMetadata const debug = Debug('cypress:server:run') -const eDebug = Debug('cypress:server:run:crash_handling') const DELAY_TO_LET_VIDEO_FINISH_MS = 1000 const relativeSpecPattern = (projectRoot, pattern) => { @@ -411,145 +405,19 @@ function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, return openProject.launch(browser, spec, browserOpts) } -interface ReporterTestAttempt { - state: 'skipped' | 'failed' | 'passed' - error: any - timings: any - failedFromHookId: any - wallClockStartedAt: Date - wallClockDuration: number - videoTimestamp: any -} -interface ReporterTest { - testId: string - title: string[] - state: 'skipped' | 'passed' | 'failed' - body: string - displayError: any - attempts: ReporterTestAttempt[] -} -interface ReporterResults { - error?: string - stats: { - failures: number - tests: number - passes: number - pending: number - suites: number - skipped: number - wallClockDuration: number - wallClockStartedAt: string - wallClockEndedAt: string - } - reporter: string - reporterStats: { - suites: number - tests: number - passes: number - pending: number - failures: number - start: string - end: string - duration: number - } - hooks: any[] - tests: ReporterTest[] -} - -function listenForProjectEnd (project, exit): Bluebird { +function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise { if (globalThis.CY_TEST_MOCK?.listenForProjectEnd) return Bluebird.resolve(globalThis.CY_TEST_MOCK.listenForProjectEnd) - let intermediateStats: ReporterResults | undefined - - return new Bluebird((resolve, reject) => { - if (exit === false) { - resolve = () => { - console.log('not exiting due to options.exit being false') - } - } - - const onEarlyExit = function (err) { - eDebug('onEarlyExit err %O', err) - if (err.isFatalApiErr) { - return reject(err) - } - - console.log('') - errors.log(err) - - const endTime: number = intermediateStats?.stats?.wallClockEndedAt ? Date.parse(intermediateStats?.stats?.wallClockEndedAt) : new Date().getTime() - const duration = intermediateStats?.stats?.wallClockStartedAt ? - endTime - Date.parse(intermediateStats.stats.wallClockStartedAt) : 0 - const endTimeStamp = new Date(endTime).toJSON() - - // in crash situations, the most recent report will not have the triggering test - // so the results are manually patched, which produces the expected exit=1 and - // terminal output indicating the failed test - const results = { - ...intermediateStats, - stats: { - ...intermediateStats?.stats, - wallClockEndedAt: endTimeStamp, - duration, - failures: (intermediateStats?.stats?.failures ?? 0) + 1, - skipped: (intermediateStats?.stats?.skipped ?? 1) - 1, - }, - reporterStats: { - ...intermediateStats?.reporterStats, - tests: (intermediateStats?.reporterStats?.tests ?? 0) + 1, // crashed test does not increment this value - end: intermediateStats?.reporterStats?.end || endTimeStamp, - duration, - failures: (intermediateStats?.reporterStats?.failures ?? 0) + 1, - }, - tests: (intermediateStats?.tests || []).map((test) => { - if (test.testId === pendingRunnable.id) { - return { - ...test, - state: 'failed', - attempts: test.attempts.map((attempt) => { - return { - ...attempt, - state: 'failed', - } - }), - } - } - - return test - }), - error: errors.stripAnsi(err.message), - } - - debug('patched reporter results: %O', results) - - return resolve(results) - } - - let pendingRunnable: any - - project.once('end', (results) => resolve(results)) - project.on('test:before:run', ({ - runnable, - previousResults, - }) => { - intermediateStats = previousResults - eDebug('pending runnable: %O', runnable) - pendingRunnable = runnable - previousResults.tests.forEach((t) => { - eDebug('test in results: %O', t) + return Promise.race([ + new Promise((resolve) => { + project.once('end', (results) => { + if (exit) { + resolve(results) + } }) - }) - - // if we already received a reason to exit early, go ahead and do it - if (earlyExitErr) { - return onEarlyExit(earlyExitErr) - } - - // otherwise override exitEarly so we exit as soon as there is a reason - exitEarly = (err) => { - onEarlyExit(err) - } - }) + }), + endAfterError(project, exit), + ]) } async function waitForBrowserToConnect (options: { project: Project, socketId: string, onError: (err: Error) => void, spec: SpecWithRelativeRoot, isFirstSpecInBrowser: boolean, testingType: string, experimentalSingleTabRunMode: boolean, browser: Browser, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, webSecurity: boolean, videoRecording?: VideoRecording, protocolManager?: ProtocolManager }) { diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts new file mode 100644 index 000000000000..fc09b5773ca2 --- /dev/null +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -0,0 +1,152 @@ +import type { ProjectBase } from '../project-base' +import * as errors from '../errors' +import Debug from 'debug' + +const debug = Debug('cypress:server:crash_handling') + +interface ReporterTestAttempt { + state: 'skipped' | 'failed' | 'passed' + error: any + timings: any + failedFromHookId: any + wallClockStartedAt: Date + wallClockDuration: number + videoTimestamp: any +} +interface ReporterTest { + testId: string + title: string[] + state: 'skipped' | 'passed' | 'failed' + body: string + displayError: any + attempts: ReporterTestAttempt[] +} +interface ReporterResults { + error?: string + stats: { + failures: number + tests: number + passes: number + pending: number + suites: number + skipped: number + wallClockDuration: number + wallClockStartedAt: string + wallClockEndedAt: string + } + reporter: string + reporterStats: { + suites: number + tests: number + passes: number + pending: number + failures: number + start: string + end: string + duration: number + } + hooks: any[] + tests: ReporterTest[] +} + +export type CypressRunError = Error & { + isFatalApiErr: boolean +} + +let earlyExitError: CypressRunError + +let earlyExit = (err: CypressRunError) => { + debug('set early exit error: %s', err.stack) + + earlyExitError = err +} + +const patchRunResultsAfterCrash = (error: CypressRunError, reporterResults: ReporterResults, mostRecentRunnable: any): ReporterResults => { + const endTime: number = reporterResults?.stats?.wallClockEndedAt ? Date.parse(reporterResults?.stats?.wallClockEndedAt) : new Date().getTime() + const wallClockDuration = reporterResults?.stats?.wallClockStartedAt ? + endTime - Date.parse(reporterResults.stats.wallClockStartedAt) : 0 + const endTimeStamp = new Date(endTime).toJSON() + + // in crash situations, the most recent report will not have the triggering test + // so the results are manually patched, which produces the expected exit=1 and + // terminal output indicating the failed test + return { + ...reporterResults, + stats: { + ...reporterResults?.stats, + wallClockEndedAt: endTimeStamp, + wallClockDuration, + failures: (reporterResults?.stats?.failures ?? 0) + 1, + skipped: (reporterResults?.stats?.skipped ?? 1) - 1, + }, + reporterStats: { + ...reporterResults?.reporterStats, + tests: (reporterResults?.reporterStats?.tests ?? 0) + 1, // crashed test does not increment this value + end: reporterResults?.reporterStats?.end || endTimeStamp, + duration: wallClockDuration, + failures: (reporterResults?.reporterStats?.failures ?? 0) + 1, + }, + tests: (reporterResults?.tests || []).map((test) => { + if (test.testId === mostRecentRunnable.id) { + return { + ...test, + state: 'failed', + attempts: [ + ...test.attempts.slice(0, -1), + { + ...test.attempts[test.attempts.length - 1], + state: 'failed', + }, + ], + } + } + + return test + }), + error: errors.stripAnsi(error.message), + } +} + +export const endAfterError = (project: ProjectBase, exit: boolean): Promise => { + let pendingRunnable: any + let intermediateStats: ReporterResults + + project.on('test:before:run', ({ + runnable, + previousResults, + }) => { + intermediateStats = previousResults + debug('pending runnable: %O', runnable) + pendingRunnable = runnable + previousResults.tests.forEach((t) => { + debug('test in results: %O', t) + }) + }) + + return new Promise((resolve, reject) => { + const patchedResolve = exit ? resolve : () => { + // eslint-disable-next-line no-console + console.log('not exiting due to options.exit being false') + } + + const handleEarlyExit = (error) => { + if (error.isFatalApiErr) { + reject(error) + } else { + patchedResolve(patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable)) + } + } + + earlyExit = (error) => { + handleEarlyExit(error) + } + + if (earlyExitError) { + handleEarlyExit(earlyExitError) + } + }) +} + +export const exitEarly = (error) => { + return earlyExit(error) +} From f3a2bee64926eb136580de2c97556b84ffa883b1 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Mon, 11 Sep 2023 10:53:09 -0400 Subject: [PATCH 04/25] correct exit option check, clean up debug --- packages/server/lib/util/graceful_crash_handling.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index fc09b5773ca2..0e67f37a9f2d 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -2,7 +2,7 @@ import type { ProjectBase } from '../project-base' import * as errors from '../errors' import Debug from 'debug' -const debug = Debug('cypress:server:crash_handling') +const debug = Debug('cypress:util:crash_handling') interface ReporterTestAttempt { state: 'skipped' | 'failed' | 'passed' @@ -116,18 +116,14 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise previousResults, }) => { intermediateStats = previousResults - debug('pending runnable: %O', runnable) pendingRunnable = runnable - previousResults.tests.forEach((t) => { - debug('test in results: %O', t) - }) }) return new Promise((resolve, reject) => { - const patchedResolve = exit ? resolve : () => { + const patchedResolve = exit === false ? () => { // eslint-disable-next-line no-console console.log('not exiting due to options.exit being false') - } + } : resolve const handleEarlyExit = (error) => { if (error.isFatalApiErr) { @@ -138,6 +134,7 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise } earlyExit = (error) => { + debug('handling early exit with error', error) handleEarlyExit(error) } @@ -148,5 +145,7 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise } export const exitEarly = (error) => { + debug('exit early called', error) + return earlyExit(error) } From 2de2a3842958208f97326811407eb4e10605f5a3 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Mon, 11 Sep 2023 15:18:19 -0400 Subject: [PATCH 05/25] exit on success if exit option !== false --- packages/server/lib/modes/run.ts | 3 +- packages/server/lib/project-base.ts | 1 + .../lib/util/graceful_crash_handling.ts | 9 ++- .../browser_crash_handling_spec.js | 56 +------------------ 4 files changed, 12 insertions(+), 57 deletions(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index e3873a3d4b17..2ce72e293e14 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -411,7 +411,8 @@ function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise return Promise.race([ new Promise((resolve) => { project.once('end', (results) => { - if (exit) { + debug('project ended with results %O', results) + if (exit !== false) { resolve(results) } }) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 7a069944139c..5cb39acf4138 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -360,6 +360,7 @@ export class ProjectBase extends EE { onConnect: (id: string) => { debug('socket:connected') + debug('reporter on socket:connected %O', reporterInstance?.results()) this.emit('socket:connected', id) }, diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index 0e67f37a9f2d..901a93fd16c3 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -115,6 +115,8 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise runnable, previousResults, }) => { + debug('preparing to run test, previous stats reported as %O', previousResults) + intermediateStats = previousResults pendingRunnable = runnable }) @@ -127,9 +129,14 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise const handleEarlyExit = (error) => { if (error.isFatalApiErr) { + debug('handling fatal api error', error) reject(error) } else { - patchedResolve(patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable)) + debug('patching results and resolving') + const results = patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable) + + debug('resolving with patched results %O', results) + patchedResolve(results) } } diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index e73a03aebb52..c718d55cc775 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -20,21 +20,6 @@ exports['Browser Crash Handling / when the tab crashes in chrome / fails'] = ` a test suite with a browser crash ✓ navigates to about:blank -We detected that the Chrome Renderer process just crashed. - -We have failed the current spec but will continue running the next spec. - -This can happen for a number of different reasons. - -If you're running lots of tests on a memory intense application. - - Try increasing the CPU/memory on the machine you're running on. - - Try enabling experimentalMemoryManagement in your config file. - - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. - -You can learn more here: - -https://on.cypress.io/renderer-process-crashed - (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -113,21 +98,6 @@ exports['Browser Crash Handling / when the tab crashes in electron / fails'] = ` a test suite with a browser crash ✓ navigates to about:blank -We detected that the Electron Renderer process just crashed. - -We have failed the current spec but will continue running the next spec. - -This can happen for a number of different reasons. - -If you're running lots of tests on a memory intense application. - - Try increasing the CPU/memory on the machine you're running on. - - Try enabling experimentalMemoryManagement in your config file. - - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. - -You can learn more here: - -https://on.cypress.io/renderer-process-crashed - (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -204,18 +174,6 @@ exports['Browser Crash Handling / when the browser process crashes in chrome / f -We detected that the Chrome process just crashed with code 'null' and signal 'SIGTRAP'. - -We have failed the current spec but will continue running the next spec. - -This can happen for many different reasons: - -- You wrote an endless loop and you must fix your own code -- You are running lots of tests on a memory intense application -- You are running in a memory starved VM environment -- There are problems with your GPU / GPU drivers -- There are browser bugs - (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -292,18 +250,6 @@ exports['Browser Crash Handling / when the browser process crashes in chrome / f -We detected that the Chrome process just crashed with code 'null' and signal 'SIGTRAP'. - -We have failed the current spec but will continue running the next spec. - -This can happen for many different reasons: - -- You wrote an endless loop and you must fix your own code -- You are running lots of tests on a memory intense application -- You are running in a memory starved VM environment -- There are problems with your GPU / GPU drivers -- There are browser bugs - (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -496,7 +442,7 @@ exports['Browser Crash Handling / when the tab closes in chrome / fails'] = ` Running: chrome_tab_close.cy.js (1 of 2) -We detected that the Chrome browser process closed unexpectedly. +We detected that the Chrome tab running Cypress tests closed unexpectedly. We have failed the current spec and aborted the run. From 44d71f19c48b2020b2acbde59d0c82248a6c9331 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Tue, 12 Sep 2023 16:04:28 -0400 Subject: [PATCH 06/25] use default stats when reporter stats are unavailable --- packages/server/lib/project-base.ts | 1 - .../lib/util/graceful_crash_handling.ts | 21 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 5cb39acf4138..7a069944139c 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -360,7 +360,6 @@ export class ProjectBase extends EE { onConnect: (id: string) => { debug('socket:connected') - debug('reporter on socket:connected %O', reporterInstance?.results()) this.emit('socket:connected', id) }, diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index 901a93fd16c3..2a5295ff1e41 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -107,6 +107,23 @@ const patchRunResultsAfterCrash = (error: CypressRunError, reporterResults: Repo } } +const defaultStats = (error: CypressRunError) => { + return { + error: errors.stripAnsi(error.message), + stats: { + failures: 1, + tests: 0, + passes: 0, + pending: 0, + suites: 0, + skipped: 0, + wallClockDuration: 0, + wallClockStartedAt: new Date().toJSON(), + wallClockEndedAt: new Date().toJSON(), + }, + } +} + export const endAfterError = (project: ProjectBase, exit: boolean): Promise => { let pendingRunnable: any let intermediateStats: ReporterResults @@ -133,7 +150,9 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise reject(error) } else { debug('patching results and resolving') - const results = patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable) + const results = (intermediateStats && pendingRunnable) ? + patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable) : + defaultStats(error) debug('resolving with patched results %O', results) patchedResolve(results) From 983bab33d208481c28d4e5331d59fa57fa0800d4 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Wed, 13 Sep 2023 15:17:32 -0400 Subject: [PATCH 07/25] fix error messaging --- .../lib/util/graceful_crash_handling.ts | 3 ++ .../browser_crash_handling_spec.js | 54 +++++++++++++++++++ system-tests/__snapshots__/record_spec.js | 6 +-- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index 2a5295ff1e41..023e8c900c46 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -150,6 +150,9 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise reject(error) } else { debug('patching results and resolving') + // eslint-disable-next-line no-console + console.log('') + errors.log(error) const results = (intermediateStats && pendingRunnable) ? patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable) : defaultStats(error) diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index c718d55cc775..d03a6ec8fa97 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -20,6 +20,21 @@ exports['Browser Crash Handling / when the tab crashes in chrome / fails'] = ` a test suite with a browser crash ✓ navigates to about:blank +We detected that the Chrome Renderer process just crashed. + +We have failed the current spec but will continue running the next spec. + +This can happen for a number of different reasons. + +If you're running lots of tests on a memory intense application. + - Try increasing the CPU/memory on the machine you're running on. + - Try enabling experimentalMemoryManagement in your config file. + - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. + +You can learn more here: + +https://on.cypress.io/renderer-process-crashed + (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -98,6 +113,21 @@ exports['Browser Crash Handling / when the tab crashes in electron / fails'] = ` a test suite with a browser crash ✓ navigates to about:blank +We detected that the Electron Renderer process just crashed. + +We have failed the current spec but will continue running the next spec. + +This can happen for a number of different reasons. + +If you're running lots of tests on a memory intense application. + - Try increasing the CPU/memory on the machine you're running on. + - Try enabling experimentalMemoryManagement in your config file. + - Try lowering numTestsKeptInMemory in your config file during 'cypress open'. + +You can learn more here: + +https://on.cypress.io/renderer-process-crashed + (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -174,6 +204,18 @@ exports['Browser Crash Handling / when the browser process crashes in chrome / f +We detected that the Chrome process just crashed with code 'null' and signal 'SIGTRAP'. + +We have failed the current spec but will continue running the next spec. + +This can happen for many different reasons: + +- You wrote an endless loop and you must fix your own code +- You are running lots of tests on a memory intense application +- You are running in a memory starved VM environment +- There are problems with your GPU / GPU drivers +- There are browser bugs + (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ @@ -250,6 +292,18 @@ exports['Browser Crash Handling / when the browser process crashes in chrome / f +We detected that the Chrome process just crashed with code 'null' and signal 'SIGTRAP'. + +We have failed the current spec but will continue running the next spec. + +This can happen for many different reasons: + +- You wrote an endless loop and you must fix your own code +- You are running lots of tests on a memory intense application +- You are running in a memory starved VM environment +- There are problems with your GPU / GPU drivers +- There are browser bugs + (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ diff --git a/system-tests/__snapshots__/record_spec.js b/system-tests/__snapshots__/record_spec.js index bfd8c743aaa9..63c19ecddaa6 100644 --- a/system-tests/__snapshots__/record_spec.js +++ b/system-tests/__snapshots__/record_spec.js @@ -1209,8 +1209,8 @@ exports['e2e record passing passes 2'] = [ 'skipped': 1, 'failures': 1, 'wallClockStartedAt': '2018-02-01T20:14:19.323Z', - 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', 'wallClockDuration': 1234, + 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', }, 'tests': [ { @@ -1305,8 +1305,8 @@ exports['e2e record passing passes 2'] = [ 'skipped': 0, 'failures': 0, 'wallClockStartedAt': '2018-02-01T20:14:19.323Z', - 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', 'wallClockDuration': 1234, + 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', }, 'tests': [ { @@ -1385,8 +1385,8 @@ exports['e2e record passing passes 2'] = [ 'skipped': 0, 'failures': 1, 'wallClockStartedAt': '2018-02-01T20:14:19.323Z', - 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', 'wallClockDuration': 1234, + 'wallClockEndedAt': '2018-02-01T20:14:19.323Z', }, 'tests': [ { From 93f7e8a8b21507213f5f68fbc3110f28063a075a Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Wed, 13 Sep 2023 15:31:16 -0400 Subject: [PATCH 08/25] move reporter types to an intermediate .ts file, to be combined with reporter.js when migrated --- packages/server/lib/types/reporter.ts | 48 +++++++++++++++ .../lib/util/graceful_crash_handling.ts | 60 ++----------------- 2 files changed, 54 insertions(+), 54 deletions(-) create mode 100644 packages/server/lib/types/reporter.ts diff --git a/packages/server/lib/types/reporter.ts b/packages/server/lib/types/reporter.ts new file mode 100644 index 000000000000..f09d52464afb --- /dev/null +++ b/packages/server/lib/types/reporter.ts @@ -0,0 +1,48 @@ +interface ReporterTestAttempt { + state: 'skipped' | 'failed' | 'passed' + error: any + timings: any + failedFromHookId: any + wallClockStartedAt: Date + wallClockDuration: number + videoTimestamp: any +} +interface ReporterTest { + testId: string + title: string[] + state: 'skipped' | 'passed' | 'failed' + body: string + displayError: any + attempts: ReporterTestAttempt[] +} + +export interface BaseReporterResults { + error?: string + stats: { + failures: number + tests: number + passes: number + pending: number + suites: number + skipped: number + wallClockDuration: number + wallClockStartedAt: string + wallClockEndedAt: string + } +} + +export interface ReporterResults extends BaseReporterResults { + reporter: string + reporterStats: { + suites: number + tests: number + passes: number + pending: number + failures: number + start: string + end: string + duration: number + } + hooks: any[] + tests: ReporterTest[] +} diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index 023e8c900c46..cb132e1ac674 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -1,67 +1,19 @@ import type { ProjectBase } from '../project-base' +import type { BaseReporterResults, ReporterResults } from '../types/reporter' import * as errors from '../errors' import Debug from 'debug' const debug = Debug('cypress:util:crash_handling') -interface ReporterTestAttempt { - state: 'skipped' | 'failed' | 'passed' - error: any - timings: any - failedFromHookId: any - wallClockStartedAt: Date - wallClockDuration: number - videoTimestamp: any -} -interface ReporterTest { - testId: string - title: string[] - state: 'skipped' | 'passed' | 'failed' - body: string - displayError: any - attempts: ReporterTestAttempt[] -} -interface ReporterResults { - error?: string - stats: { - failures: number - tests: number - passes: number - pending: number - suites: number - skipped: number - wallClockDuration: number - wallClockStartedAt: string - wallClockEndedAt: string - } - reporter: string - reporterStats: { - suites: number - tests: number - passes: number - pending: number - failures: number - start: string - end: string - duration: number - } - hooks: any[] - tests: ReporterTest[] -} - -export type CypressRunError = Error & { - isFatalApiErr: boolean -} - -let earlyExitError: CypressRunError +let earlyExitError: Error -let earlyExit = (err: CypressRunError) => { +let earlyExit = (err: Error) => { debug('set early exit error: %s', err.stack) earlyExitError = err } -const patchRunResultsAfterCrash = (error: CypressRunError, reporterResults: ReporterResults, mostRecentRunnable: any): ReporterResults => { +const patchRunResultsAfterCrash = (error: Error, reporterResults: ReporterResults, mostRecentRunnable: any): ReporterResults => { const endTime: number = reporterResults?.stats?.wallClockEndedAt ? Date.parse(reporterResults?.stats?.wallClockEndedAt) : new Date().getTime() const wallClockDuration = reporterResults?.stats?.wallClockStartedAt ? endTime - Date.parse(reporterResults.stats.wallClockStartedAt) : 0 @@ -107,7 +59,7 @@ const patchRunResultsAfterCrash = (error: CypressRunError, reporterResults: Repo } } -const defaultStats = (error: CypressRunError) => { +const defaultStats = (error: Error): BaseReporterResults => { return { error: errors.stripAnsi(error.message), stats: { @@ -144,7 +96,7 @@ export const endAfterError = (project: ProjectBase, exit: boolean): Promise console.log('not exiting due to options.exit being false') } : resolve - const handleEarlyExit = (error) => { + const handleEarlyExit = (error: Error & { isFatalApiErr?: boolean }) => { if (error.isFatalApiErr) { debug('handling fatal api error', error) reject(error) From a117dbf7eba18f12481e7d45441ba59ee3328581 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 10:38:45 -0400 Subject: [PATCH 09/25] debug tab close test in ci --- system-tests/package.json | 2 +- system-tests/test/browser_crash_handling_spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/system-tests/package.json b/system-tests/package.json index fb50fddf1e12..0cffb9e6a9ee 100644 --- a/system-tests/package.json +++ b/system-tests/package.json @@ -14,7 +14,7 @@ "pretest": "yarn gulp ensureCloudValidations", "test": "node ./scripts/run.js --glob-in-dir=\"{test,test-binary}\"", "pretest:ci": "yarn gulp ensureCloudValidations", - "test:ci": "node ./scripts/run.js", + "test:ci": "DEBUG=cypress:server:browsers:browser-cri-client,cypress:util:crash_handling ./scripts/run.js", "update:snapshots": "SNAPSHOT_UPDATE=1 npm run test" }, "devDependencies": { diff --git a/system-tests/test/browser_crash_handling_spec.js b/system-tests/test/browser_crash_handling_spec.js index 9762dbb9f5bb..bff98c43d589 100644 --- a/system-tests/test/browser_crash_handling_spec.js +++ b/system-tests/test/browser_crash_handling_spec.js @@ -28,7 +28,7 @@ describe('Browser Crash Handling', () => { }) // It should fail the chrome_tab_close spec, and exit early, do not move onto the next spec - context('when the tab closes in chrome', () => { + context.only('when the tab closes in chrome', () => { // const outputPath = path.join(e2ePath, 'output.json') systemTests.it('fails', { From c9c5ebbcdeb3637d823658c06a1e2ab02583ceab Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 11:03:57 -0400 Subject: [PATCH 10/25] move debug env from pkg to ci yml --- .circleci/workflows.yml | 1 + system-tests/package.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml index 3400182b7dfd..d6243fedc1a1 100644 --- a/.circleci/workflows.yml +++ b/.circleci/workflows.yml @@ -652,6 +652,7 @@ commands: done SPECS=`echo $SPECS | xargs -n 1 | circleci tests split --split-by=timings` echo SPECS=$SPECS + DEBUG=cypress:server:browsers:browser-cri-client,cypress:util:crash_handling yarn workspace @tooling/system-tests test:ci $SPECS --browser <> - verify-mocha-results - store_test_results: diff --git a/system-tests/package.json b/system-tests/package.json index 0cffb9e6a9ee..8a791e74bd8d 100644 --- a/system-tests/package.json +++ b/system-tests/package.json @@ -14,7 +14,7 @@ "pretest": "yarn gulp ensureCloudValidations", "test": "node ./scripts/run.js --glob-in-dir=\"{test,test-binary}\"", "pretest:ci": "yarn gulp ensureCloudValidations", - "test:ci": "DEBUG=cypress:server:browsers:browser-cri-client,cypress:util:crash_handling ./scripts/run.js", + "test:ci": "./scripts/run.js", "update:snapshots": "SNAPSHOT_UPDATE=1 npm run test" }, "devDependencies": { From ef74379f26e60f30775235eb72912fdc17e77023 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 11:28:45 -0400 Subject: [PATCH 11/25] set debug env in spec --- .circleci/workflows.yml | 1 - system-tests/test/browser_crash_handling_spec.js | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml index d6243fedc1a1..3400182b7dfd 100644 --- a/.circleci/workflows.yml +++ b/.circleci/workflows.yml @@ -652,7 +652,6 @@ commands: done SPECS=`echo $SPECS | xargs -n 1 | circleci tests split --split-by=timings` echo SPECS=$SPECS - DEBUG=cypress:server:browsers:browser-cri-client,cypress:util:crash_handling yarn workspace @tooling/system-tests test:ci $SPECS --browser <> - verify-mocha-results - store_test_results: diff --git a/system-tests/test/browser_crash_handling_spec.js b/system-tests/test/browser_crash_handling_spec.js index bff98c43d589..813227aa1a73 100644 --- a/system-tests/test/browser_crash_handling_spec.js +++ b/system-tests/test/browser_crash_handling_spec.js @@ -1,5 +1,7 @@ const systemTests = require('../lib/system-tests').default +process.env.DEBUG = 'cypress:server:browsers:browser-cri-client,cypress:util:crash_handling' + describe('Browser Crash Handling', () => { systemTests.setup({ settings: { @@ -28,7 +30,7 @@ describe('Browser Crash Handling', () => { }) // It should fail the chrome_tab_close spec, and exit early, do not move onto the next spec - context.only('when the tab closes in chrome', () => { + context('when the tab closes in chrome', () => { // const outputPath = path.join(e2ePath, 'output.json') systemTests.it('fails', { From e259a69ffb59fb50391e4411dd2ee9af28306e12 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 11:49:51 -0400 Subject: [PATCH 12/25] fix pckg --- system-tests/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/package.json b/system-tests/package.json index 8a791e74bd8d..fb50fddf1e12 100644 --- a/system-tests/package.json +++ b/system-tests/package.json @@ -14,7 +14,7 @@ "pretest": "yarn gulp ensureCloudValidations", "test": "node ./scripts/run.js --glob-in-dir=\"{test,test-binary}\"", "pretest:ci": "yarn gulp ensureCloudValidations", - "test:ci": "./scripts/run.js", + "test:ci": "node ./scripts/run.js", "update:snapshots": "SNAPSHOT_UPDATE=1 npm run test" }, "devDependencies": { From 0896f7f31b0e47a6fa94176439134fa1f72495d1 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 13:54:09 -0400 Subject: [PATCH 13/25] adds some logging to cri-client --- packages/server/lib/browsers/browser-cri-client.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/server/lib/browsers/browser-cri-client.ts b/packages/server/lib/browsers/browser-cri-client.ts index 4d6d536731a6..bf06b0315eb7 100644 --- a/packages/server/lib/browsers/browser-cri-client.ts +++ b/packages/server/lib/browsers/browser-cri-client.ts @@ -158,6 +158,7 @@ export class BrowserCriClient { closing: browserCriClient.closing, closed: browserCriClient.closed, resettingBrowserTargets: browserCriClient.resettingBrowserTargets, + currentlyAttachedTarget: browserCriClient.currentlyAttachedTarget, }) // we may have gotten a delayed "Target.targetDestroyed" even for a page that we @@ -202,6 +203,10 @@ export class BrowserCriClient { }) .timeout(500) .then((expectedDestroyedEvent) => { + debug({ + expectedDestroyedEvent, + }) + if (expectedDestroyedEvent === true) { return } From b8be7fda8322ce5f73609d9d4dc433a1164840d0 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 13:54:36 -0400 Subject: [PATCH 14/25] remove event emit logging from project-base --- packages/server/lib/project-base.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 7a069944139c..5a99821b8e7e 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -115,12 +115,6 @@ export class ProjectBase extends EE { } } - emit (eventName, ...args) { - eDebug('Project emit %O', eventName, args) - - return super.emit(eventName, ...args) - } - protected ensureProp = ensureProp setOnTestsReceived (fn) { From 8ddeb41b8f9f0ffc0c4fb4f7bab6924d7e30a42e Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 14 Sep 2023 15:17:36 -0400 Subject: [PATCH 15/25] revert snapshot for tab close system test --- packages/server/lib/browsers/browser-cri-client.ts | 5 ----- system-tests/__snapshots__/browser_crash_handling_spec.js | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/server/lib/browsers/browser-cri-client.ts b/packages/server/lib/browsers/browser-cri-client.ts index bf06b0315eb7..4d6d536731a6 100644 --- a/packages/server/lib/browsers/browser-cri-client.ts +++ b/packages/server/lib/browsers/browser-cri-client.ts @@ -158,7 +158,6 @@ export class BrowserCriClient { closing: browserCriClient.closing, closed: browserCriClient.closed, resettingBrowserTargets: browserCriClient.resettingBrowserTargets, - currentlyAttachedTarget: browserCriClient.currentlyAttachedTarget, }) // we may have gotten a delayed "Target.targetDestroyed" even for a page that we @@ -203,10 +202,6 @@ export class BrowserCriClient { }) .timeout(500) .then((expectedDestroyedEvent) => { - debug({ - expectedDestroyedEvent, - }) - if (expectedDestroyedEvent === true) { return } diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index d03a6ec8fa97..e73a03aebb52 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -496,7 +496,7 @@ exports['Browser Crash Handling / when the tab closes in chrome / fails'] = ` Running: chrome_tab_close.cy.js (1 of 2) -We detected that the Chrome tab running Cypress tests closed unexpectedly. +We detected that the Chrome browser process closed unexpectedly. We have failed the current spec and aborted the run. From 2bb5df44a0de2ba72fc1d456c6c52e6b919ac733 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Fri, 15 Sep 2023 13:27:53 -0400 Subject: [PATCH 16/25] fixes console output for no exit on success --- packages/server/lib/modes/run.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 2ce72e293e14..761007966090 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -412,7 +412,9 @@ function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise new Promise((resolve) => { project.once('end', (results) => { debug('project ended with results %O', results) - if (exit !== false) { + if (exit === false) { + console.log('not exiting due to options.exit being false') + } else { resolve(results) } }) From e69f9bdb5902db3745c7365d4b58aa77e5bdbcd5 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Mon, 18 Sep 2023 15:59:41 -0400 Subject: [PATCH 17/25] changelog --- cli/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 6418a505b6e8..eb2d55071d7a 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -7,6 +7,10 @@ _Released 09/19/2023 (PENDING)_ - Introduces new layout for Runs page providing additional run information. Addresses [#27203](https://github.com/cypress-io/cypress/issues/27203). + **Bugfixes:** + + - Enables test replay for executed specs in runs that have a spec that causes a browser crash. Addressed in [#27786](https://github.com/cypress-io/cypress/pull/27786) + ## 13.2.0 _Released 09/12/2023_ From bc44078f043f26dfa8b2bf3521fd95749cd2cad3 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Mon, 18 Sep 2023 16:23:14 -0400 Subject: [PATCH 18/25] changelog wsp --- cli/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index eb2d55071d7a..0200da5a45c9 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -7,7 +7,7 @@ _Released 09/19/2023 (PENDING)_ - Introduces new layout for Runs page providing additional run information. Addresses [#27203](https://github.com/cypress-io/cypress/issues/27203). - **Bugfixes:** +**Bugfixes:** - Enables test replay for executed specs in runs that have a spec that causes a browser crash. Addressed in [#27786](https://github.com/cypress-io/cypress/pull/27786) From 47f03d4d66a1c453cdfa075641fd3bd79a822b4b Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Tue, 19 Sep 2023 10:30:50 -0400 Subject: [PATCH 19/25] cleanup --- packages/server/lib/project-base.ts | 2 -- .../e2e/cypress/e2e/chrome_tab_crash.cy.js | 14 +------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 5a99821b8e7e..e8172918c40c 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -49,7 +49,6 @@ export interface Cfg extends ReceivedCypressOptions { const localCwd = process.cwd() const debug = Debug('cypress:server:project') -const eDebug = Debug('cypress:server:project:emit') type StartWebsocketOptions = Pick @@ -377,7 +376,6 @@ export class ProjectBase extends EE { }, onMocha: async (event, runnable) => { - eDebug('onMocha', event) // bail if we dont have a // reporter instance if (!reporterInstance) { diff --git a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js index ae5edc1afd37..893f2a751a81 100644 --- a/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js +++ b/system-tests/projects/e2e/cypress/e2e/chrome_tab_crash.cy.js @@ -1,22 +1,10 @@ describe('a test suite with a browser crash', function () { - // let deferred; let proceed - - before(() => { - // deferred = new Promise((res) => proceed = res) - }) - it('navigates to about:blank', () => { - cy.visit('/index.html').then(() => { - // proceed() - }) + cy.visit('/index.html') }) it('crashes the chrome tab', () => { - // make exec of this one dependent on prev, to ensure linear execution of tests for predictable results - // cy.wrap(deferred).then((() => { Cypress.automation('remote:debugger:protocol', { command: 'Page.navigate', params: { url: 'chrome://crash', transitionType: 'typed' } }) - cy.visit('localhost') - // })) }) }) From dd4f1f7edf8766d2f2a8fea23767c402026a85d7 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Tue, 19 Sep 2023 14:30:52 -0400 Subject: [PATCH 20/25] clean up tests --- system-tests/test/browser_crash_handling_spec.js | 2 -- system-tests/test/record_spec.js | 10 +++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/system-tests/test/browser_crash_handling_spec.js b/system-tests/test/browser_crash_handling_spec.js index 813227aa1a73..9762dbb9f5bb 100644 --- a/system-tests/test/browser_crash_handling_spec.js +++ b/system-tests/test/browser_crash_handling_spec.js @@ -1,7 +1,5 @@ const systemTests = require('../lib/system-tests').default -process.env.DEBUG = 'cypress:server:browsers:browser-cri-client,cypress:util:crash_handling' - describe('Browser Crash Handling', () => { systemTests.setup({ settings: { diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index c91d8bc7ff05..ccb04c67f1c0 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -2324,9 +2324,13 @@ describe('e2e record', () => { const requests = getRequests() const postResultsRequest = requests.find((r) => r.url === `POST /instances/${instanceId}/results`) - console.log(JSON.stringify(postResultsRequest, null, 2)) - - console.log(getRequests()) + expect(postResultsRequest.body.exception).to.include('Chrome Renderer process just crashed') + expect(postResultsRequest.body.tests).to.have.length(2) + expect(postResultsRequest.body.stats.suites).to.equal(1) + expect(postResultsRequest.body.stats.tests).to.equal(2) + expect(postResultsRequest.body.stats.passes).to.equal(1) + expect(postResultsRequest.body.stats.failures).to.equal(1) + expect(postResultsRequest.body.stats.skipped).to.equal(0) expect(urls).to.include.members([`PUT ${CAPTURE_PROTOCOL_UPLOAD_URL}`]) }) From 0c09b1dbc2e76e9300e6d61e8bf1a7ba94519063 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Wed, 20 Sep 2023 16:20:41 -0400 Subject: [PATCH 21/25] refactor to more straightforward control flow --- packages/server/lib/modes/results.ts | 2 +- packages/server/lib/modes/run.ts | 59 ++++++++---- .../lib/util/graceful_crash_handling.ts | 95 ++++++++----------- .../browser_crash_handling_spec.js | 2 +- 4 files changed, 83 insertions(+), 75 deletions(-) diff --git a/packages/server/lib/modes/results.ts b/packages/server/lib/modes/results.ts index 9da4b263aa83..e92e7520feec 100644 --- a/packages/server/lib/modes/results.ts +++ b/packages/server/lib/modes/results.ts @@ -56,7 +56,7 @@ interface ScreenshotInformation { width: pixels } -interface RunResult { +export interface RunResult { error: string | null hooks: HookInformation[] reporter: string diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 761007966090..2dbee21338a6 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -27,7 +27,7 @@ import * as printResults from '../util/print-run' import type { ProtocolManager } from '../cloud/protocol' import { telemetry } from '@packages/telemetry' import { CypressRunResult, createPublicBrowser, createPublicConfig, createPublicRunResults, createPublicSpec, createPublicSpecResults } from './results' -import { endAfterError, exitEarly } from '../util/graceful_crash_handling' +import { EarlyExitTerminator } from '../util/graceful_crash_handling' type SetScreenshotMetadata = (data: TakeScreenshotProps) => void type ScreenshotMetadata = ReturnType @@ -42,6 +42,8 @@ let currentSetScreenshotMetadata: SetScreenshotMetadata const debug = Debug('cypress:server:run') const DELAY_TO_LET_VIDEO_FINISH_MS = 1000 +let earlyExitTerminator = new EarlyExitTerminator() + const relativeSpecPattern = (projectRoot, pattern) => { if (typeof pattern === 'string') { return pattern.replace(`${projectRoot}/`, '') @@ -405,22 +407,31 @@ function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot, return openProject.launch(browser, spec, browserOpts) } -function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise { +async function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise { if (globalThis.CY_TEST_MOCK?.listenForProjectEnd) return Bluebird.resolve(globalThis.CY_TEST_MOCK.listenForProjectEnd) - return Promise.race([ - new Promise((resolve) => { - project.once('end', (results) => { - debug('project ended with results %O', results) - if (exit === false) { - console.log('not exiting due to options.exit being false') - } else { - resolve(results) - } - }) - }), - endAfterError(project, exit), - ]) + // if exit is false, we need to intercept the resolution of tests - whether + // an early exit with intermediate results, or a full run. + return new Promise((resolve, reject) => { + Promise.race([ + new Promise((res) => { + project.once('end', (results) => { + debug('project ended with results %O', results) + res(results) + }) + }), + earlyExitTerminator.waitForEarlyExit(project, exit), + ]).then((results) => { + if (exit === false) { + // eslint-disable-next-line no-console + console.log('not exiting due to options.exit being false') + } else { + resolve(results) + } + }).catch((err) => { + reject(err) + }) + }) } async function waitForBrowserToConnect (options: { project: Project, socketId: string, onError: (err: Error) => void, spec: SpecWithRelativeRoot, isFirstSpecInBrowser: boolean, testingType: string, experimentalSingleTabRunMode: boolean, browser: Browser, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, webSecurity: boolean, videoRecording?: VideoRecording, protocolManager?: ProtocolManager }) { @@ -692,6 +703,13 @@ async function waitForTestsToFinishRunning (options: { project: Project, screens results.video = null } + // the early exit terminator persists between specs, + // so if this spec crashed, the next one will report as + // a crash too unless it is reset. Would like to not rely + // on closure, but threading through fn props via options is also not + // great. + earlyExitTerminator = new EarlyExitTerminator() + return results } @@ -967,7 +985,11 @@ async function ready (options: ReadyOptions) { // this needs to be a closure over `exitEarly` and not a reference // because `exitEarly` gets overwritten in `listenForProjectEnd` // TODO: refactor this so we don't need to extend options - const onError = options.onError = (err) => exitEarly(err) + + const onError = options.onError = (err) => { + debug('onError') + earlyExitTerminator.exitEarly(err) + } // alias and coerce to null let specPatternFromCli = options.spec || null @@ -1102,6 +1124,7 @@ async function ready (options: ReadyOptions) { } export async function run (options, loading: Promise) { + debug('run start') // Check if running as electron process if (require('../util/electron-app').isRunningAsElectronProcess({ debug })) { const app = require('electron').app @@ -1123,6 +1146,8 @@ export async function run (options, loading: Promise) { try { return ready(options) } catch (e) { - return exitEarly(e) + debug('caught outer error', e) + + return earlyExitTerminator.exitEarly(e) } } diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index cb132e1ac674..38d42718fc49 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -2,17 +2,10 @@ import type { ProjectBase } from '../project-base' import type { BaseReporterResults, ReporterResults } from '../types/reporter' import * as errors from '../errors' import Debug from 'debug' +import pDefer, { DeferredPromise } from 'p-defer' const debug = Debug('cypress:util:crash_handling') -let earlyExitError: Error - -let earlyExit = (err: Error) => { - debug('set early exit error: %s', err.stack) - - earlyExitError = err -} - const patchRunResultsAfterCrash = (error: Error, reporterResults: ReporterResults, mostRecentRunnable: any): ReporterResults => { const endTime: number = reporterResults?.stats?.wallClockEndedAt ? Date.parse(reporterResults?.stats?.wallClockEndedAt) : new Date().getTime() const wallClockDuration = reporterResults?.stats?.wallClockStartedAt ? @@ -76,57 +69,47 @@ const defaultStats = (error: Error): BaseReporterResults => { } } -export const endAfterError = (project: ProjectBase, exit: boolean): Promise => { - let pendingRunnable: any - let intermediateStats: ReporterResults - - project.on('test:before:run', ({ - runnable, - previousResults, - }) => { - debug('preparing to run test, previous stats reported as %O', previousResults) - - intermediateStats = previousResults - pendingRunnable = runnable - }) - - return new Promise((resolve, reject) => { - const patchedResolve = exit === false ? () => { - // eslint-disable-next-line no-console - console.log('not exiting due to options.exit being false') - } : resolve - - const handleEarlyExit = (error: Error & { isFatalApiErr?: boolean }) => { - if (error.isFatalApiErr) { - debug('handling fatal api error', error) - reject(error) - } else { - debug('patching results and resolving') - // eslint-disable-next-line no-console - console.log('') - errors.log(error) - const results = (intermediateStats && pendingRunnable) ? - patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable) : - defaultStats(error) - - debug('resolving with patched results %O', results) - patchedResolve(results) - } - } +export class EarlyExitTerminator { + private terminator: DeferredPromise - earlyExit = (error) => { - debug('handling early exit with error', error) - handleEarlyExit(error) - } + private pendingRunnable: any + private intermediateStats: ReporterResults | undefined + + constructor () { + this.terminator = pDefer() + } - if (earlyExitError) { - handleEarlyExit(earlyExitError) + waitForEarlyExit (project: ProjectBase, exit?: boolean) { + debug('waiting for early exit') + + project.on('test:before:run', ({ + runnable, + previousResults, + }) => { + debug('preparing to run test, previous stats reported as %O', previousResults) + + this.intermediateStats = previousResults + this.pendingRunnable = runnable + }) + + return this.terminator.promise + } + + exitEarly (error) { + if (error.isFatalApiErr) { + this.terminator.reject(error) + + return } - }) -} -export const exitEarly = (error) => { - debug('exit early called', error) + // eslint-disable-next-line no-console + console.log('') + errors.log(error) - return earlyExit(error) + const runResults: BaseReporterResults = (this.intermediateStats && this.pendingRunnable) ? + patchRunResultsAfterCrash(error, this.intermediateStats, this.pendingRunnable) : + defaultStats(error) + + this.terminator.resolve(runResults) + } } diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index e73a03aebb52..d03a6ec8fa97 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -496,7 +496,7 @@ exports['Browser Crash Handling / when the tab closes in chrome / fails'] = ` Running: chrome_tab_close.cy.js (1 of 2) -We detected that the Chrome browser process closed unexpectedly. +We detected that the Chrome tab running Cypress tests closed unexpectedly. We have failed the current spec and aborted the run. From f7e145f61855122ec26e59049968c0328090f70e Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 21 Sep 2023 10:44:53 -0400 Subject: [PATCH 22/25] rm export for unused type --- packages/server/lib/modes/results.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/lib/modes/results.ts b/packages/server/lib/modes/results.ts index e92e7520feec..9da4b263aa83 100644 --- a/packages/server/lib/modes/results.ts +++ b/packages/server/lib/modes/results.ts @@ -56,7 +56,7 @@ interface ScreenshotInformation { width: pixels } -export interface RunResult { +interface RunResult { error: string | null hooks: HookInformation[] reporter: string From c688ca66397668124998fcdba0107350b33c2801 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 21 Sep 2023 13:28:20 -0400 Subject: [PATCH 23/25] correct tab close snapshot for ci --- system-tests/__snapshots__/browser_crash_handling_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index d03a6ec8fa97..e73a03aebb52 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -496,7 +496,7 @@ exports['Browser Crash Handling / when the tab closes in chrome / fails'] = ` Running: chrome_tab_close.cy.js (1 of 2) -We detected that the Chrome tab running Cypress tests closed unexpectedly. +We detected that the Chrome browser process closed unexpectedly. We have failed the current spec and aborted the run. From a043e40f6f48947a9ae6021277602f52a0f4e9dc Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Wed, 4 Oct 2023 16:19:34 -0400 Subject: [PATCH 24/25] new system test for mid-test config crash --- system-tests/__snapshots__/record_spec.js | 57 +++++++++---------- .../cypress-with-project-id.config.js | 15 +++++ .../cypress/e2e/simple_multiple.cy.js | 13 +++++ system-tests/test/record_spec.js | 31 +++++++++- 4 files changed, 82 insertions(+), 34 deletions(-) create mode 100644 system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js create mode 100644 system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js diff --git a/system-tests/__snapshots__/record_spec.js b/system-tests/__snapshots__/record_spec.js index 143d24419abe..a6136f786010 100644 --- a/system-tests/__snapshots__/record_spec.js +++ b/system-tests/__snapshots__/record_spec.js @@ -3468,7 +3468,7 @@ exports['e2e record capture-protocol enabled protocol runtime errors error in pr ` -exports['capture-protocol api errors upload 500 - retries 8 times continues 1'] = ` +exports['capture-protocol api errors upload 500 - retries 7 times and succeeds on the last call continues 1'] = ` ==================================================================================================== @@ -3524,12 +3524,12 @@ exports['capture-protocol api errors upload 500 - retries 8 times continues 1'] - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay + - Test Replay - 1 kB (Uploaded Cloud Artifacts) - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Failed Uploading 2/2 - Internal Server Error + - Test Replay - Done Uploading 1 kB 2/2 ==================================================================================================== @@ -3550,7 +3550,7 @@ exports['capture-protocol api errors upload 500 - retries 8 times continues 1'] ` -exports['capture-protocol api errors upload 500 - retries 7 times and succeeds on the last call continues 1'] = ` +exports['capture-protocol api errors upload 500 - retries 8 times and fails continues 1'] = ` ==================================================================================================== @@ -3606,12 +3606,12 @@ exports['capture-protocol api errors upload 500 - retries 7 times and succeeds o - Video - Nothing to upload - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - 1 kB + - Test Replay (Uploaded Cloud Artifacts) - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Done Uploading 1 kB 2/2 + - Test Replay - Failed Uploading 2/2 - Internal Server Error ==================================================================================================== @@ -3632,7 +3632,7 @@ exports['capture-protocol api errors upload 500 - retries 7 times and succeeds o ` -exports['capture-protocol api errors upload 500 - retries 8 times and fails continues 1'] = ` +exports['e2e record capture-protocol enabled when there is an async error thrown from config file posts accurate test results 1'] = ` ==================================================================================================== @@ -3641,8 +3641,8 @@ exports['capture-protocol api errors upload 500 - retries 8 times and fails cont ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ + │ Specs: 1 found (simple_multiple.cy.js) │ + │ Searched: cypress/e2e/simple_multiple.cy.js │ │ Params: Tag: false, Group: false, Parallel: false │ │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -3650,50 +3650,45 @@ exports['capture-protocol api errors upload 500 - retries 8 times and fails cont ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: record_pass.cy.js (1 of 1) + Running: simple_multiple.cy.js (1 of 1) Estimated: X second(s) - record pass - ✓ passes - - is pending + suite + ✓ is true +Your configFile threw an error from: cypress-with-project-id.config.js - 1 passing - 1 pending +We stopped running your tests because your config file crashed. +Error: Async error from plugins file + [stack trace lines] (Results) ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Tests: 2 │ │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 1 │ + │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ + │ Spec Ran: simple_multiple.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - (Uploading Cloud Artifacts) - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay + - Screenshot - Nothing to upload + - Test Replay - 1 kB (Uploaded Cloud Artifacts) - - Screenshot - Done Uploading 1 kB 1/2 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Test Replay - Failed Uploading 2/2 - Internal Server Error + - Test Replay - Done Uploading 1 kB 1/1 ==================================================================================================== @@ -3702,9 +3697,9 @@ exports['capture-protocol api errors upload 500 - retries 8 times and fails cont Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ + │ ✖ simple_multiple.cy.js XX:XX 2 1 1 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - + ✖ 1 of 1 failed (100%) XX:XX 2 1 1 - - ─────────────────────────────────────────────────────────────────────────────────────────────────────── @@ -3714,7 +3709,7 @@ exports['capture-protocol api errors upload 500 - retries 8 times and fails cont ` -exports['e2e record capture-protocol enabled when the tab crashes in chrome still uploads a test replay 1'] = ` +exports['e2e record capture-protocol enabled when the tab crashes in chrome posts accurate test results 1'] = ` ==================================================================================================== diff --git a/system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js b/system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js new file mode 100644 index 000000000000..135e9da2ecc1 --- /dev/null +++ b/system-tests/projects/config-with-crashing-plugin/cypress-with-project-id.config.js @@ -0,0 +1,15 @@ +module.exports = { + 'projectId': 'pid123', + 'e2e': { + 'supportFile': false, + setupNodeEvents (on, config) { + on('before:spec', () => { + setTimeout(() => { + throw new Error('Async error from plugins file') + }, 1000) + }) + + return config + }, + }, +} diff --git a/system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js b/system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js new file mode 100644 index 000000000000..8c710948ce29 --- /dev/null +++ b/system-tests/projects/config-with-crashing-plugin/cypress/e2e/simple_multiple.cy.js @@ -0,0 +1,13 @@ +describe('suite', function () { + it('is true', () => { + expect(true).to.be.true + cy.wait(150) + }) + + it('is still true', () => { + // the config should crash before this test completes; + // this is a long wait in order to improve predictability + cy.wait(10000) + expect(true).to.be.true + }) +}) diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index 06e381887023..cdf51f60805f 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -2310,7 +2310,7 @@ describe('e2e record', () => { describe('when the tab crashes in chrome', () => { enableCaptureProtocol() - it('still uploads a test replay', function () { + it('posts accurate test results', function () { return systemTests.exec(this, { key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', configFile: 'cypress-with-project-id.config.js', @@ -2320,7 +2320,6 @@ describe('e2e record', () => { snapshot: true, expectedExitCode: 1, }).then(() => { - const urls = getRequestUrls() const requests = getRequests() const postResultsRequest = requests.find((r) => r.url === `POST /instances/${instanceId}/results`) @@ -2331,8 +2330,34 @@ describe('e2e record', () => { expect(postResultsRequest.body.stats.passes).to.equal(1) expect(postResultsRequest.body.stats.failures).to.equal(1) expect(postResultsRequest.body.stats.skipped).to.equal(0) + }) + }) + }) - expect(urls).to.include.members([`PUT ${CAPTURE_PROTOCOL_UPLOAD_URL}`]) + describe('when there is an async error thrown from config file', () => { + enableCaptureProtocol() + it('posts accurate test results', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + browser: 'chrome', + project: 'config-with-crashing-plugin', + spec: 'simple_multiple.cy.js', + configFile: 'cypress-with-project-id.config.js', + record: true, + snapshot: true, + expectedExitCode: 1, + }).then(() => { + const requests = getRequests() + const postResultsRequest = requests.find((r) => r.url === `POST /instances/${instanceId}/results`) + + console.log(postResultsRequest) + expect(postResultsRequest?.body.exception).to.include('Your configFile threw an error') + expect(postResultsRequest?.body.tests).to.have.length(2) + expect(postResultsRequest?.body.stats.suites).to.equal(1) + expect(postResultsRequest?.body.stats.tests).to.equal(2) + expect(postResultsRequest?.body.stats.passes).to.equal(1) + expect(postResultsRequest?.body.stats.failures).to.equal(1) + expect(postResultsRequest?.body.stats.skipped).to.equal(0) }) }) }) From 9bf8025386982a4580022ec75ca60027f392f402 Mon Sep 17 00:00:00 2001 From: Cacie Prins Date: Thu, 5 Oct 2023 14:57:14 -0400 Subject: [PATCH 25/25] update snapshots --- .../__snapshots__/browser_crash_handling_spec.js | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/system-tests/__snapshots__/browser_crash_handling_spec.js b/system-tests/__snapshots__/browser_crash_handling_spec.js index 4b52a5024801..e73a03aebb52 100644 --- a/system-tests/__snapshots__/browser_crash_handling_spec.js +++ b/system-tests/__snapshots__/browser_crash_handling_spec.js @@ -43,19 +43,13 @@ https://on.cypress.io/renderer-process-crashed │ Failing: 1 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 1 │ + │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: chrome_tab_crash.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (1280x720) - (failed).png - - ──────────────────────────────────────────────────────────────────────────────────────────────────── Running: simple.cy.js (2 of 2) @@ -142,19 +136,13 @@ https://on.cypress.io/renderer-process-crashed │ Failing: 1 │ │ Pending: 0 │ │ Skipped: 0 │ - │ Screenshots: 1 │ + │ Screenshots: 0 │ │ Video: false │ │ Duration: X seconds │ │ Spec Ran: chrome_tab_crash.cy.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/chrome_tab_crash.cy.js/navigates to about html (1280x720) - (failed).png - - ──────────────────────────────────────────────────────────────────────────────────────────────────── Running: simple.cy.js (2 of 2)