[APM] E2E script improvements (#116972) (#117526)

Co-authored-by: Søren Louv-Jansen <[email protected]>

packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts

@@ -68,6 +68,7 @@ it('produces the right watch and ignore list', () => {
       <absolute path>/x-pack/plugins/reporting/chromium,
       <absolute path>/x-pack/plugins/security_solution/cypress,
       <absolute path>/x-pack/plugins/apm/scripts,
+      <absolute path>/x-pack/plugins/apm/ftr_e2e,
       <absolute path>/x-pack/plugins/canvas/canvas_plugin_src,
       <absolute path>/x-pack/plugins/cases/server/scripts,
       <absolute path>/x-pack/plugins/lists/scripts,

packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts

@@ -59,6 +59,7 @@ export function getServerWatchPaths({ pluginPaths, pluginScanDirs }: Options) {
     fromRoot('x-pack/plugins/reporting/chromium'),
     fromRoot('x-pack/plugins/security_solution/cypress'),
     fromRoot('x-pack/plugins/apm/scripts'),
+    fromRoot('x-pack/plugins/apm/ftr_e2e'), // prevents restarts for APM cypress tests
     fromRoot('x-pack/plugins/canvas/canvas_plugin_src'), // prevents server from restarting twice for Canvas plugin changes,
     fromRoot('x-pack/plugins/cases/server/scripts'),
     fromRoot('x-pack/plugins/lists/scripts'),

x-pack/plugins/apm/dev_docs/testing.md

@@ -27,7 +27,7 @@ API tests are separated in two suites:
 node scripts/test/api [--trial] [--help]
 ```
 
-The API tests are located in `x-pack/test/apm_api_integration/`.
+The API tests are located in [`x-pack/test/apm_api_integration/`](/x-pack/test/apm_api_integration/).
 
 **API Test tips**
 
@@ -43,11 +43,12 @@ The API tests are located in `x-pack/test/apm_api_integration/`.
 node scripts/test/e2e [--trial] [--help]
 ```
 
-The E2E tests are located [here](../ftr_e2e)
+The E2E tests are located in [`x-pack/plugins/apm/ftr_e2e`](../ftr_e2e)
 
 ---
 
 ## Functional tests (Security and Correlations tests)
+
 TODO: We could try moving this tests to the new e2e tests located at `x-pack/plugins/apm/ftr_e2e`.
 
 **Start server**
@@ -66,17 +67,18 @@ APM tests are located in `x-pack/test/functional/apps/apm`.
 For debugging access Elasticsearch on http://localhost:9220` (elastic/changeme)
 diff --git a/x-pack/plugins/apm/scripts/test/README.md b/x-pack/plugins/apm/scripts/test/README.md
 
-
 ## Storybook
 
 ### Start
+
 ```
 yarn storybook apm
 ```
 
 All files with a .stories.tsx extension will be loaded. You can access the development environment at http://localhost:9001.
 
 ## Data generation
+
 For end-to-end (e.g. agent -> apm server -> elasticsearch <- kibana) development and testing of Elastic APM please check the the [APM Integration Testing repository](https://github.com/elastic/apm-integration-testing).
 
-Data can also be generated using the [elastic-apm-synthtrace](../../../../packages/elastic-apm-synthtrace/README.md) CLI.
+Data can also be generated using the [elastic-apm-synthtrace](../../../../packages/elastic-apm-synthtrace/README.md) CLI.

x-pack/plugins/apm/ftr_e2e/cypress_start.ts

@@ -7,30 +7,16 @@
 
 /* eslint-disable no-console */
 
+import { argv } from 'yargs';
 import Url from 'url';
 import cypress from 'cypress';
 import { FtrProviderContext } from './ftr_provider_context';
 import { createApmUsersAndRoles } from '../scripts/create-apm-users-and-roles/create_apm_users_and_roles';
 import { esArchiverLoad, esArchiverUnload } from './cypress/tasks/es_archiver';
 
-export function cypressRunTests(spec?: string) {
-  return async ({ getService }: FtrProviderContext) => {
-    const result = await cypressStart(getService, cypress.run, spec);
-
-    if (result && (result.status === 'failed' || result.totalFailed > 0)) {
-      throw new Error(`APM Cypress tests failed`);
-    }
-  };
-}
-
-export async function cypressOpenTests({ getService }: FtrProviderContext) {
-  await cypressStart(getService, cypress.open);
-}
-
-async function cypressStart(
+export async function cypressStart(
   getService: FtrProviderContext['getService'],
-  cypressExecution: typeof cypress.run | typeof cypress.open,
-  spec?: string
+  cypressExecution: typeof cypress.run | typeof cypress.open
 ) {
   const config = getService('config');
 
@@ -68,8 +54,9 @@ async function cypressStart(
   console.log(`Loading ES archive "${archiveName}"`);
   await esArchiverLoad(archiveName);
 
+  const spec = argv.grep as string | undefined;
   const res = await cypressExecution({
-    ...(spec !== undefined ? { spec } : {}),
+    ...(spec ? { spec } : {}),
     config: { baseUrl: kibanaUrl },
     env: {
       KIBANA_URL: kibanaUrl,

x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import cypress from 'cypress';
+import { FtrProviderContext } from './ftr_provider_context';
+import { cypressStart } from './cypress_start';
+
+async function ftrConfigOpen({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
+  return {
+    ...kibanaConfig.getAll(),
+    testRunner,
+  };
+}
+
+export async function testRunner({ getService }: FtrProviderContext) {
+  await cypressStart(getService, cypress.open);
+}
+
+// eslint-disable-next-line import/no-default-export
+export default ftrConfigOpen;

x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import cypress from 'cypress';
+import { cypressStart } from './cypress_start';
+import { FtrProviderContext } from './ftr_provider_context';
+
+async function ftrConfigRun({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaConfig = await readConfigFile(require.resolve('./ftr_config.ts'));
+  return {
+    ...kibanaConfig.getAll(),
+    testRunner,
+  };
+}
+
+async function testRunner({ getService }: FtrProviderContext) {
+  const result = await cypressStart(getService, cypress.run);
+
+  if (result && (result.status === 'failed' || result.totalFailed > 0)) {
+    throw new Error(`APM Cypress tests failed`);
+  }
+}
+
+// eslint-disable-next-line import/no-default-export
+export default ftrConfigRun;

x-pack/plugins/apm/scripts/test/api.js

@@ -33,9 +33,15 @@ const { argv } = yargs(process.argv.slice(2))
     description:
       'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
   })
+  .option('grep', {
+    alias: 'spec',
+    default: false,
+    type: 'string',
+    description: 'Specify the spec files to run',
+  })
   .help();
 
-const { trial, server, runner } = argv;
+const { trial, server, runner, grep } = argv;
 
 const license = trial ? 'trial' : 'basic';
 console.log(`License: ${license}`);
@@ -46,7 +52,10 @@ if (server) {
 } else if (runner) {
   ftrScript = 'functional_test_runner';
 }
-childProcess.execSync(
-  `node ../../../../scripts/${ftrScript} --config ../../../../test/apm_api_integration/${license}/config.ts`,
-  { cwd: path.join(__dirname), stdio: 'inherit' }
-);
+
+const grepArg = grep ? `--grep "${grep}"` : '';
+const cmd = `node ../../../../scripts/${ftrScript} ${grepArg} --config ../../../../test/apm_api_integration/${license}/config.ts`;
+
+console.log(`Running ${cmd}`);
+
+childProcess.execSync(cmd, { cwd: path.join(__dirname), stdio: 'inherit' });

x-pack/plugins/apm/scripts/test/e2e.js

@@ -20,22 +20,34 @@ const { argv } = yargs(process.argv.slice(2))
   .option('server', {
     default: false,
     type: 'boolean',
-    description: 'Start Elasticsearch and kibana',
+    description: 'Start Elasticsearch and Kibana',
   })
   .option('runner', {
     default: false,
     type: 'boolean',
     description:
       'Run all tests (an instance of Elasticsearch and kibana are needs to be available)',
   })
+  .option('grep', {
+    alias: 'spec',
+    default: false,
+    type: 'string',
+    description:
+      'Specify the spec files to run (use doublequotes for glob matching)',
+  })
   .option('open', {
     default: false,
     type: 'boolean',
     description: 'Opens the Cypress Test Runner',
   })
+  .option('bail', {
+    default: false,
+    type: 'boolean',
+    description: 'stop tests after the first failure',
+  })
   .help();
 
-const { server, runner, open, kibanaInstallDir } = argv;
+const { server, runner, open, grep, bail, kibanaInstallDir } = argv;
 
 const e2eDir = path.join(__dirname, '../../ftr_e2e');
 
@@ -46,9 +58,10 @@ if (server) {
   ftrScript = 'functional_test_runner';
 }
 
-const config = open ? './cypress_open.ts' : './cypress_run.ts';
+const config = open ? './ftr_config_open.ts' : './ftr_config_run.ts';
+const grepArg = grep ? `--grep "${grep}"` : '';
+const bailArg = bail ? `--bail` : '';
+const cmd = `node ../../../../scripts/${ftrScript} --config ${config} ${grepArg} ${bailArg} --kibana-install-dir '${kibanaInstallDir}'`;
 
-childProcess.execSync(
-  `node ../../../../scripts/${ftrScript} --config ${config} --kibana-install-dir '${kibanaInstallDir}'`,
-  { cwd: e2eDir, stdio: 'inherit' }
-);
+console.log(`Running ${cmd}`);
+childProcess.execSync(cmd, { cwd: e2eDir, stdio: 'inherit' });