From ddd962cef25113aac2e87d1957502d3d853cb83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 1 Nov 2021 15:22:02 +0100 Subject: [PATCH 1/5] [APM] E2E script improvements --- .../src/get_server_watch_paths.ts | 1 + x-pack/plugins/apm/ftr_e2e/cypress_open.ts | 20 ------------- x-pack/plugins/apm/ftr_e2e/cypress_run.ts | 22 -------------- x-pack/plugins/apm/ftr_e2e/cypress_start.ts | 23 ++++---------- .../apm/ftr_e2e/{config.ts => ftr_config.ts} | 0 x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts | 26 ++++++++++++++++ x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts | 30 +++++++++++++++++++ x-pack/plugins/apm/scripts/test/e2e.js | 21 ++++++++++--- 8 files changed, 79 insertions(+), 64 deletions(-) delete mode 100644 x-pack/plugins/apm/ftr_e2e/cypress_open.ts delete mode 100644 x-pack/plugins/apm/ftr_e2e/cypress_run.ts rename x-pack/plugins/apm/ftr_e2e/{config.ts => ftr_config.ts} (100%) create mode 100644 x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts create mode 100644 x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts diff --git a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts index 53f52279c8be8..e1bd431d280a4 100644 --- a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts +++ b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts @@ -59,6 +59,7 @@ export function getServerWatchPaths({ pluginPaths, pluginScanDirs }: Options) { fromRoot('x-pack/plugins/reporting/chromium'), fromRoot('x-pack/plugins/security_solution/cypress'), fromRoot('x-pack/plugins/apm/scripts'), + fromRoot('x-pack/plugins/apm/ftr_e2e'), // prevents restarts for APM cypress tests fromRoot('x-pack/plugins/canvas/canvas_plugin_src'), // prevents server from restarting twice for Canvas plugin changes, fromRoot('x-pack/plugins/cases/server/scripts'), fromRoot('x-pack/plugins/lists/scripts'), diff --git a/x-pack/plugins/apm/ftr_e2e/cypress_open.ts b/x-pack/plugins/apm/ftr_e2e/cypress_open.ts deleted file mode 100644 index 3f7758b40b90d..0000000000000 --- a/x-pack/plugins/apm/ftr_e2e/cypress_open.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrConfigProviderContext } from '@kbn/test'; -import { cypressOpenTests } from './cypress_start'; - -async function openE2ETests({ readConfigFile }: FtrConfigProviderContext) { - const kibanaConfig = await readConfigFile(require.resolve('./config.ts')); - return { - ...kibanaConfig.getAll(), - testRunner: cypressOpenTests, - }; -} - -// eslint-disable-next-line import/no-default-export -export default openE2ETests; diff --git a/x-pack/plugins/apm/ftr_e2e/cypress_run.ts b/x-pack/plugins/apm/ftr_e2e/cypress_run.ts deleted file mode 100644 index 16f93b39910f3..0000000000000 --- a/x-pack/plugins/apm/ftr_e2e/cypress_run.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { argv } from 'yargs'; -import { FtrConfigProviderContext } from '@kbn/test'; -import { cypressRunTests } from './cypress_start'; - -const specArg = argv.spec as string | undefined; - -async function runE2ETests({ readConfigFile }: FtrConfigProviderContext) { - const kibanaConfig = await readConfigFile(require.resolve('./config.ts')); - return { - ...kibanaConfig.getAll(), - testRunner: cypressRunTests(specArg), - }; -} - -// eslint-disable-next-line import/no-default-export -export default runE2ETests; diff --git a/x-pack/plugins/apm/ftr_e2e/cypress_start.ts b/x-pack/plugins/apm/ftr_e2e/cypress_start.ts index 2d0be8c007089..c8ab216cbce5c 100644 --- a/x-pack/plugins/apm/ftr_e2e/cypress_start.ts +++ b/x-pack/plugins/apm/ftr_e2e/cypress_start.ts @@ -7,30 +7,16 @@ /* eslint-disable no-console */ +import { argv } from 'yargs'; import Url from 'url'; import cypress from 'cypress'; import { FtrProviderContext } from './ftr_provider_context'; import { createApmUsersAndRoles } from '../scripts/create-apm-users-and-roles/create_apm_users_and_roles'; import { esArchiverLoad, esArchiverUnload } from './cypress/tasks/es_archiver'; -export function cypressRunTests(spec?: string) { - return async ({ getService }: FtrProviderContext) => { - const result = await cypressStart(getService, cypress.run, spec); - - if (result && (result.status === 'failed' || result.totalFailed > 0)) { - throw new Error(`APM Cypress tests failed`); - } - }; -} - -export async function cypressOpenTests({ getService }: FtrProviderContext) { - await cypressStart(getService, cypress.open); -} - -async function cypressStart( +export async function cypressStart( getService: FtrProviderContext['getService'], - cypressExecution: typeof cypress.run | typeof cypress.open, - spec?: string + cypressExecution: typeof cypress.run | typeof cypress.open ) { const config = getService('config'); @@ -68,8 +54,9 @@ async function cypressStart( console.log(`Loading ES archive "${archiveName}"`); await esArchiverLoad(archiveName); + const spec = argv.grep as string | undefined; const res = await cypressExecution({ - ...(spec !== undefined ? { spec } : {}), + ...(spec ? { spec } : {}), config: { baseUrl: kibanaUrl }, env: { KIBANA_URL: kibanaUrl, diff --git a/x-pack/plugins/apm/ftr_e2e/config.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config.ts similarity index 100% rename from x-pack/plugins/apm/ftr_e2e/config.ts rename to x-pack/plugins/apm/ftr_e2e/ftr_config.ts diff --git a/x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts new file mode 100644 index 0000000000000..92f605d328789 --- /dev/null +++ b/x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import cypress from 'cypress'; +import { FtrProviderContext } from './ftr_provider_context'; +import { cypressStart } from './cypress_start'; + +async function ftrConfigOpen({ readConfigFile }: FtrConfigProviderContext) { + const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); + return { + ...kibanaConfig.getAll(), + testRunner, + }; +} + +export async function testRunner({ getService }: FtrProviderContext) { + await cypressStart(getService, cypress.open); +} + +// eslint-disable-next-line import/no-default-export +export default ftrConfigOpen; diff --git a/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts new file mode 100644 index 0000000000000..51c859a8477f2 --- /dev/null +++ b/x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import cypress from 'cypress'; +import { cypressStart } from './cypress_start'; +import { FtrProviderContext } from './ftr_provider_context'; + +async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) { + const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts')); + return { + ...kibanaConfig.getAll(), + testRunner, + }; +} + +async function testRunner({ getService }: FtrProviderContext) { + const result = await cypressStart(getService, cypress.run); + + if (result && (result.status === 'failed' || result.totalFailed > 0)) { + throw new Error(`APM Cypress tests failed`); + } +} + +// eslint-disable-next-line import/no-default-export +export default ftrConfigRun; diff --git a/x-pack/plugins/apm/scripts/test/e2e.js b/x-pack/plugins/apm/scripts/test/e2e.js index b3ce510a8e569..2f6ff09ccf89e 100644 --- a/x-pack/plugins/apm/scripts/test/e2e.js +++ b/x-pack/plugins/apm/scripts/test/e2e.js @@ -20,7 +20,7 @@ const { argv } = yargs(process.argv.slice(2)) .option('server', { default: false, type: 'boolean', - description: 'Start Elasticsearch and kibana', + description: 'Start Elasticsearch and Kibana', }) .option('runner', { default: false, @@ -28,14 +28,25 @@ const { argv } = yargs(process.argv.slice(2)) description: 'Run all tests (an instance of Elasticsearch and kibana are needs to be available)', }) + .option('spec', { + default: false, + type: 'string', + description: + 'Specify the spec files to run (use doublequotes for glob matching)', + }) .option('open', { default: false, type: 'boolean', description: 'Opens the Cypress Test Runner', }) + .option('bail', { + default: false, + type: 'boolean', + description: 'stop tests after the first failure', + }) .help(); -const { server, runner, open, kibanaInstallDir } = argv; +const { server, runner, open, spec, bail, kibanaInstallDir } = argv; const e2eDir = path.join(__dirname, '../../ftr_e2e'); @@ -46,9 +57,11 @@ if (server) { ftrScript = 'functional_test_runner'; } -const config = open ? './cypress_open.ts' : './cypress_run.ts'; +const config = open ? './ftr_config_open.ts' : './ftr_config_run.ts'; +const grepArg = spec ? `--grep ${spec}` : ''; +const bailArg = bail ? `--bail` : ''; childProcess.execSync( - `node ../../../../scripts/${ftrScript} --config ${config} --kibana-install-dir '${kibanaInstallDir}'`, + `node ../../../../scripts/${ftrScript} --config ${config} ${grepArg} ${bailArg} --kibana-install-dir '${kibanaInstallDir}'`, { cwd: e2eDir, stdio: 'inherit' } ); From a3f6e70d20b94c40f12ef7abbf958fc137e7ad09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Nov 2021 23:25:26 +0100 Subject: [PATCH 2/5] Fix snapshot --- packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts index 9cb902882ffd7..9fa13b013f195 100644 --- a/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts +++ b/packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts @@ -68,6 +68,7 @@ it('produces the right watch and ignore list', () => { /x-pack/plugins/reporting/chromium, /x-pack/plugins/security_solution/cypress, /x-pack/plugins/apm/scripts, + /x-pack/plugins/apm/ftr_e2e, /x-pack/plugins/canvas/canvas_plugin_src, /x-pack/plugins/cases/server/scripts, /x-pack/plugins/lists/scripts, From 7d2db2c8ed0669746619bda950188e4099e998e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Nov 2021 23:29:20 +0100 Subject: [PATCH 3/5] Add grep arg --- x-pack/plugins/apm/scripts/test/api.js | 13 +++++++++++-- x-pack/plugins/apm/scripts/test/e2e.js | 7 ++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/scripts/test/api.js b/x-pack/plugins/apm/scripts/test/api.js index 4f0d82d0c1163..68e211a530876 100644 --- a/x-pack/plugins/apm/scripts/test/api.js +++ b/x-pack/plugins/apm/scripts/test/api.js @@ -33,9 +33,15 @@ const { argv } = yargs(process.argv.slice(2)) description: 'Run all tests (an instance of Elasticsearch and kibana are needs to be available)', }) + .option('grep', { + alias: 'spec', + default: false, + type: 'string', + description: 'Specify the spec files to run', + }) .help(); -const { trial, server, runner } = argv; +const { trial, server, runner, grep } = argv; const license = trial ? 'trial' : 'basic'; console.log(`License: ${license}`); @@ -46,7 +52,10 @@ if (server) { } else if (runner) { ftrScript = 'functional_test_runner'; } + +const grepArg = grep ? `--grep ${grep}` : ''; + childProcess.execSync( - `node ../../../../scripts/${ftrScript} --config ../../../../test/apm_api_integration/${license}/config.ts`, + `node ../../../../scripts/${ftrScript} ${grepArg} --config ../../../../test/apm_api_integration/${license}/config.ts`, { cwd: path.join(__dirname), stdio: 'inherit' } ); diff --git a/x-pack/plugins/apm/scripts/test/e2e.js b/x-pack/plugins/apm/scripts/test/e2e.js index 2f6ff09ccf89e..7e13777f42fd6 100644 --- a/x-pack/plugins/apm/scripts/test/e2e.js +++ b/x-pack/plugins/apm/scripts/test/e2e.js @@ -28,7 +28,8 @@ const { argv } = yargs(process.argv.slice(2)) description: 'Run all tests (an instance of Elasticsearch and kibana are needs to be available)', }) - .option('spec', { + .option('grep', { + alias: 'spec', default: false, type: 'string', description: @@ -46,7 +47,7 @@ const { argv } = yargs(process.argv.slice(2)) }) .help(); -const { server, runner, open, spec, bail, kibanaInstallDir } = argv; +const { server, runner, open, grep, bail, kibanaInstallDir } = argv; const e2eDir = path.join(__dirname, '../../ftr_e2e'); @@ -58,7 +59,7 @@ if (server) { } const config = open ? './ftr_config_open.ts' : './ftr_config_run.ts'; -const grepArg = spec ? `--grep ${spec}` : ''; +const grepArg = grep ? `--grep ${grep}` : ''; const bailArg = bail ? `--bail` : ''; childProcess.execSync( From 8f897f2457210cd0d158a1cac557468ad7f464e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 4 Nov 2021 12:18:19 +0100 Subject: [PATCH 4/5] Log command --- x-pack/plugins/apm/scripts/test/api.js | 10 +++++----- x-pack/plugins/apm/scripts/test/e2e.js | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/apm/scripts/test/api.js b/x-pack/plugins/apm/scripts/test/api.js index 68e211a530876..1905c8eb7c2dd 100644 --- a/x-pack/plugins/apm/scripts/test/api.js +++ b/x-pack/plugins/apm/scripts/test/api.js @@ -53,9 +53,9 @@ if (server) { ftrScript = 'functional_test_runner'; } -const grepArg = grep ? `--grep ${grep}` : ''; +const grepArg = grep ? `--grep "${grep}"` : ''; +const cmd = `node ../../../../scripts/${ftrScript} ${grepArg} --config ../../../../test/apm_api_integration/${license}/config.ts`; -childProcess.execSync( - `node ../../../../scripts/${ftrScript} ${grepArg} --config ../../../../test/apm_api_integration/${license}/config.ts`, - { cwd: path.join(__dirname), stdio: 'inherit' } -); +console.log(`Running ${cmd}`); + +childProcess.execSync(cmd, { cwd: path.join(__dirname), stdio: 'inherit' }); diff --git a/x-pack/plugins/apm/scripts/test/e2e.js b/x-pack/plugins/apm/scripts/test/e2e.js index 7e13777f42fd6..13055dce2fec5 100644 --- a/x-pack/plugins/apm/scripts/test/e2e.js +++ b/x-pack/plugins/apm/scripts/test/e2e.js @@ -59,10 +59,9 @@ if (server) { } const config = open ? './ftr_config_open.ts' : './ftr_config_run.ts'; -const grepArg = grep ? `--grep ${grep}` : ''; +const grepArg = grep ? `--grep "${grep}"` : ''; const bailArg = bail ? `--bail` : ''; +const cmd = `node ../../../../scripts/${ftrScript} --config ${config} ${grepArg} ${bailArg} --kibana-install-dir '${kibanaInstallDir}'`; -childProcess.execSync( - `node ../../../../scripts/${ftrScript} --config ${config} ${grepArg} ${bailArg} --kibana-install-dir '${kibanaInstallDir}'`, - { cwd: e2eDir, stdio: 'inherit' } -); +console.log(`Running ${cmd}`); +childProcess.execSync(cmd, { cwd: e2eDir, stdio: 'inherit' }); From 79e9849ebf4cc07dffce981d0b154953bc85a86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 4 Nov 2021 13:07:42 +0100 Subject: [PATCH 5/5] Improve readme --- x-pack/plugins/apm/dev_docs/testing.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/apm/dev_docs/testing.md b/x-pack/plugins/apm/dev_docs/testing.md index 95ba2467befcd..2a7533402ecca 100644 --- a/x-pack/plugins/apm/dev_docs/testing.md +++ b/x-pack/plugins/apm/dev_docs/testing.md @@ -27,7 +27,7 @@ API tests are separated in two suites: node scripts/test/api [--trial] [--help] ``` -The API tests are located in `x-pack/test/apm_api_integration/`. +The API tests are located in [`x-pack/test/apm_api_integration/`](/x-pack/test/apm_api_integration/). **API Test tips** @@ -43,11 +43,12 @@ The API tests are located in `x-pack/test/apm_api_integration/`. node scripts/test/e2e [--trial] [--help] ``` -The E2E tests are located [here](../ftr_e2e) +The E2E tests are located in [`x-pack/plugins/apm/ftr_e2e`](../ftr_e2e) --- ## Functional tests (Security and Correlations tests) + TODO: We could try moving this tests to the new e2e tests located at `x-pack/plugins/apm/ftr_e2e`. **Start server** @@ -66,10 +67,10 @@ APM tests are located in `x-pack/test/functional/apps/apm`. For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme) diff --git a/x-pack/plugins/apm/scripts/test/README.md b/x-pack/plugins/apm/scripts/test/README.md - ## Storybook ### Start + ``` yarn storybook apm ``` @@ -77,6 +78,7 @@ yarn storybook apm All files with a .stories.tsx extension will be loaded. You can access the development environment at http://localhost:9001. ## Data generation + For end-to-end (e.g. agent -> apm server -> elasticsearch <- kibana) development and testing of Elastic APM please check the the [APM Integration Testing repository](https://github.com/elastic/apm-integration-testing). -Data can also be generated using the [elastic-apm-synthtrace](../../../../packages/elastic-apm-synthtrace/README.md) CLI. \ No newline at end of file +Data can also be generated using the [elastic-apm-synthtrace](../../../../packages/elastic-apm-synthtrace/README.md) CLI.