diff --git a/.github/workflows/end2end-test.yml b/.github/workflows/end2end-test.yml index b3d2ba9b12e5d..7f24b774c6968 100644 --- a/.github/workflows/end2end-test.yml +++ b/.github/workflows/end2end-test.yml @@ -51,6 +51,13 @@ jobs: $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests ) + - name: Archive debug artifacts (screenshots, HTML snapshots) + uses: actions/upload-artifact@v2 + if: always() + with: + name: failures-artifacts + path: artifacts + admin-2: name: Admin - 2 @@ -92,6 +99,13 @@ jobs: $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests ) + - name: Archive debug artifacts (screenshots, HTML snapshots) + uses: actions/upload-artifact@v2 + if: always() + with: + name: failures-artifacts + path: artifacts + admin-3: name: Admin - 3 @@ -133,6 +147,13 @@ jobs: $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests ) + - name: Archive debug artifacts (screenshots, HTML snapshots) + uses: actions/upload-artifact@v2 + if: always() + with: + name: failures-artifacts + path: artifacts + admin-4: name: Admin - 4 @@ -173,3 +194,11 @@ jobs: run: | $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests ) + + - name: Archive debug artifacts (screenshots, HTML snapshots) + uses: actions/upload-artifact@v2 + if: always() + with: + name: failures-artifacts + path: artifacts + diff --git a/packages/e2e-tests/config/setup-debug-artifacts.js b/packages/e2e-tests/config/setup-debug-artifacts.js new file mode 100644 index 0000000000000..577fc0f5130cd --- /dev/null +++ b/packages/e2e-tests/config/setup-debug-artifacts.js @@ -0,0 +1,54 @@ +/** + * External dependencies + */ +const fs = require( 'fs' ); +const util = require( 'util' ); +const root = process.env.GITHUB_WORKSPACE || __dirname + '/../../../'; +const ARTIFACTS_PATH = root + '/artifacts'; + +const writeFile = util.promisify( fs.writeFile ); + +if ( ! fs.existsSync( ARTIFACTS_PATH ) ) { + fs.mkdirSync( ARTIFACTS_PATH ); +} + +/** + * Gutenberg uses the default jest-jasmine2 test runner that comes with Jest. + * Unfortunately, version 2 of jasmine doesn't support async reporters. It + * does support async before and after hooks though, so the workaround here + * works by making each test wait for the artifacts before starting. + * + * Kudos to Tom Esterez (@testerez) for sharing this idea in https://github.com/smooth-code/jest-puppeteer/issues/131#issuecomment-424073620 + */ +let artifactsPromise; +// eslint-disable-next-line jest/no-jasmine-globals +jasmine.getEnv().addReporter( { + specDone: ( result ) => { + if ( result.status === 'failed' ) { + artifactsPromise = storeArtifacts( result.fullName ); + } + }, +} ); + +beforeEach( () => artifactsPromise ); +afterAll( () => artifactsPromise ); + +async function storeArtifacts( testName ) { + const slug = slugify( testName ); + await writeFile( + `${ ARTIFACTS_PATH }/${ slug }-snapshot.html`, + await page.content() + ); + await page.screenshot( { path: `${ ARTIFACTS_PATH }/${ slug }.jpg` } ); +} + +function slugify( testName ) { + const datetime = new Date().toISOString().split( '.' )[ 0 ]; + const readableName = `${ testName } ${ datetime }`; + const slug = readableName + .toLowerCase() + .replace( /:/g, '-' ) + .replace( /[^0-9a-zA-Z \-\(\)]/g, '' ) + .replace( / /g, '-' ); + return slug; +} diff --git a/packages/e2e-tests/jest.config.js b/packages/e2e-tests/jest.config.js index 4d2add7ca8542..eeceeb7cc470b 100644 --- a/packages/e2e-tests/jest.config.js +++ b/packages/e2e-tests/jest.config.js @@ -2,6 +2,7 @@ module.exports = { ...require( '@wordpress/scripts/config/jest-e2e.config' ), setupFiles: [ '/config/gutenberg-phase.js' ], setupFilesAfterEnv: [ + '/config/setup-debug-artifacts.js', '/config/setup-test-framework.js', '@wordpress/jest-console', '@wordpress/jest-puppeteer-axe',