diff --git a/x-pack/plugins/observability_solution/apm/dev_docs/testing.md b/x-pack/plugins/observability_solution/apm/dev_docs/testing.md index 02ff43d37bf75..aeb9435a2e113 100644 --- a/x-pack/plugins/observability_solution/apm/dev_docs/testing.md +++ b/x-pack/plugins/observability_solution/apm/dev_docs/testing.md @@ -3,7 +3,9 @@ We've got three ways of testing our code: - Unit testing with Jest -- API testing +- Integration Tests + - Deployment specific (stateful or serverless) API testing + - Deployment-agnostic (both stateful and serverless) testing - End-to-end testing (with Cypress) API tests are usually preferred. They're stable and reasonably quick, and give a good approximation of real-world usage. @@ -73,7 +75,59 @@ node x-pack/plugins/observability_solution/apm/scripts/test/api --runner --basic #### API Test tips -- For data generation in API tests have a look at the [kbn-apm-synthtrace](../../../../packages/kbn-apm-synthtrace/README.md) package +- For data generation in API tests have a look at the [kbn-apm-synthtrace](../../../../../packages/kbn-apm-synthtrace/README.md) package +- For debugging access Elasticsearch on http://localhost:9220 and Kibana on http://localhost:5620 (`elastic` / `changeme`) + +--- + +## Deployment-agnostic Tests (dat) + +| Option | Description | +| ------------ | ----------------------------------------------- | +| --serverless | Loads serverless configuration | +| --stateful | Loads stateful configuration | +| --server | Only start ES and Kibana | +| --runner | Only run tests | +| --grep | Specify the specs to run | +| --grep-files | Specify the files to run | +| --inspect | Add --inspect-brk flag to the ftr for debugging | +| --times | Repeat the test n number of times | + +Deployment-agnostic tests are located in [`x-pack/test/deployment_agnostic/apis/observability/apm/index.ts`](../../../../test/api_integration/deployment_agnostic/apis/observability/apm/index.ts). + +#### Start server and run test (single process) + +``` +node x-pack/plugins/observability_solution/apm/scripts/test/dat [--serverless/--stateful] [--help] +``` + +The above command will start an ES instance on http://localhost:9220, a Kibana instance on http://localhost:5620 and run the api tests. +Once the tests finish, the instances will be terminated. + +#### Start server and run test (separate processes) + +```sh + +# start server +node x-pack/plugins/observability_solution/apm/scripts/test/dat --server --stateful + +# run tests +node x-pack/plugins/observability_solution/apm/scripts/test/dat --runner --stateful --grep-files=error_group_list +``` + +### Update snapshots (from Kibana root) + +To update snapshots append `--updateSnapshots` to the `--runner` command: + +``` +node x-pack/plugins/observability_solution/apm/scripts/test/dat --runner --stateful --updateSnapshots +``` + +(The test server needs to be running) + +#### API Test tips + +- For data generation in Deployment-agnostic tests have a look at the [kbn-apm-synthtrace](../../../../../packages/kbn-apm-synthtrace/README.md) package - For debugging access Elasticsearch on http://localhost:9220 and Kibana on http://localhost:5620 (`elastic` / `changeme`) --- diff --git a/x-pack/plugins/observability_solution/apm/scripts/test/dat.js b/x-pack/plugins/observability_solution/apm/scripts/test/dat.js new file mode 100644 index 0000000000000..b457cd4da5e76 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/scripts/test/dat.js @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable no-console */ +const { times } = require('lodash'); +const yargs = require('yargs'); +const path = require('path'); +const childProcess = require('child_process'); +const { REPO_ROOT } = require('@kbn/repo-info'); + +const { argv } = yargs(process.argv.slice(2)) + .option('serverless', { + default: false, + type: 'boolean', + description: 'Loads serverless configuration', + }) + .option('stateful', { + default: false, + type: 'boolean', + description: 'Loads stateful configuration', + }) + .option('server', { + default: false, + type: 'boolean', + description: 'Only start ES and Kibana', + }) + .option('runner', { + default: false, + type: 'boolean', + description: 'Only run tests', + }) + .option('grep', { + alias: 'spec', + type: 'string', + description: 'Specify the specs to run', + }) + .option('grep-files', { + alias: 'files', + type: 'array', + string: true, + description: 'Specify the files to run', + }) + .option('inspect', { + default: false, + type: 'boolean', + description: 'Add --inspect-brk flag to the ftr for debugging', + }) + .option('times', { + type: 'number', + description: 'Repeat the test n number of times', + }) + .option('updateSnapshots', { + default: false, + type: 'boolean', + description: 'Update snapshots', + }) + .option('bail', { + default: false, + type: 'boolean', + description: 'Stop the test run at the first failure', + }) + .check((argv) => { + const { inspect, runner } = argv; + if (inspect && !runner) { + throw new Error('--inspect can only be used with --runner'); + } else { + return true; + } + }) + .help(); + +const { serverless, stateful, bail, server, runner, grep, grepFiles, inspect, updateSnapshots } = + argv; + +if ((serverless === false && stateful === false) || (serverless && stateful)) { + throw new Error('Please specify either --stateful or --serverless'); +} + +let ftrScript = 'functional_tests'; +if (server) { + ftrScript = 'functional_tests_server'; +} else if (runner) { + ftrScript = 'functional_test_runner'; +} + +const environment = serverless ? 'serverless' : 'stateful'; + +const cmd = [ + 'node', + ...(inspect ? ['--inspect-brk'] : []), + `${REPO_ROOT}/scripts/${ftrScript}`, + ...(grep ? [`--grep "${grep}"`] : []), + ...(updateSnapshots ? [`--updateSnapshots`] : []), + ...(bail ? [`--bail`] : []), + `--config ${REPO_ROOT}/x-pack/test/api_integration/deployment_agnostic/configs/${environment}/oblt.apm.${environment}.config.ts`, +].join(' '); + +console.log(`Running: "${cmd}"`); + +function runTests() { + childProcess.execSync(cmd, { + cwd: path.join(__dirname), + stdio: 'inherit', + env: { ...process.env, APM_TEST_GREP_FILES: JSON.stringify(grepFiles) }, + }); +} + +if (argv.times) { + const runCounter = { succeeded: 0, failed: 0, remaining: argv.times }; + let exitStatus = 0; + times(argv.times, () => { + try { + runTests(); + runCounter.succeeded++; + } catch (e) { + if (bail) { + throw e; + } + + exitStatus = 1; + runCounter.failed++; + } + runCounter.remaining--; + if (argv.times > 1) { + console.log(runCounter); + } + }); + process.exit(exitStatus); +} else { + runTests(); +}