From 2498f5719fcea591a3cd592ccac3bc9aa3576464 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 1 Feb 2021 22:33:50 +0100 Subject: [PATCH 1/3] [load testing] add env vars to pass simulations and repo rootPath (#89544) * [load testing] add env vars to pass simulations and repo rootPath * pass simulation to sript as argument * export GATLING_SIMULATIONS * fix export * add validation --- test/scripts/jenkins_build_load_testing.sh | 13 ++++- x-pack/test/load/runner.ts | 63 +++++++++++++++++++--- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/test/scripts/jenkins_build_load_testing.sh b/test/scripts/jenkins_build_load_testing.sh index aeb584b106498..659321f1d3975 100755 --- a/test/scripts/jenkins_build_load_testing.sh +++ b/test/scripts/jenkins_build_load_testing.sh @@ -1,5 +1,13 @@ #!/usr/bin/env bash +while getopts s: flag +do + case "${flag}" in + s) simulations=${OPTARG};; + esac +done +echo "Simulation classes: $simulations"; + cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh @@ -25,6 +33,7 @@ echo " -> test setup" source test/scripts/jenkins_test_setup_xpack.sh echo " -> run gatling load testing" +export GATLING_SIMULATIONS="$simulations" node scripts/functional_tests \ - --kibana-install-dir "$KIBANA_INSTALL_DIR" \ - --config test/load/config.ts + --kibana-install-dir "$KIBANA_INSTALL_DIR" \ + --config test/load/config.ts diff --git a/x-pack/test/load/runner.ts b/x-pack/test/load/runner.ts index e9a1cadfddc55..52448a6b32a99 100644 --- a/x-pack/test/load/runner.ts +++ b/x-pack/test/load/runner.ts @@ -7,23 +7,57 @@ import { withProcRunner } from '@kbn/dev-utils'; import { resolve } from 'path'; import { REPO_ROOT } from '@kbn/utils'; +import Fs from 'fs'; +import { createFlagError } from '@kbn/dev-utils'; import { FtrProviderContext } from './../functional/ftr_provider_context'; +const baseSimulationPath = 'src/test/scala/org/kibanaLoadTest/simulation'; +const simulationPackage = 'org.kibanaLoadTest.simulation'; +const simulationFIleExtension = '.scala'; +const gatlingProjectRootPath: string = + process.env.GATLING_PROJECT_PATH || resolve(REPO_ROOT, '../kibana-load-testing'); +const simulationEntry: string = process.env.GATLING_SIMULATIONS || 'DemoJourney'; + +if (!Fs.existsSync(gatlingProjectRootPath)) { + throw createFlagError( + `Incorrect path to load testing project: '${gatlingProjectRootPath}'\n + Clone 'elastic/kibana-load-testing' and set path using 'GATLING_PROJECT_PATH' env var` + ); +} + +const dropEmptyLines = (s: string) => s.split(',').filter((i) => i.length > 0); +const simulationClasses = dropEmptyLines(simulationEntry); +const simulationsRootPath = resolve(gatlingProjectRootPath, baseSimulationPath); + +simulationClasses.map((className) => { + const simulationClassPath = resolve( + simulationsRootPath, + className.replace('.', '/') + simulationFIleExtension + ); + if (!Fs.existsSync(simulationClassPath)) { + throw createFlagError(`Simulation class is not found: '${simulationClassPath}'`); + } +}); + +/** + * + * GatlingTestRunner is used to run load simulation against local Kibana instance + * + * Use GATLING_SIMULATIONS to pass comma-separated class names + * Use GATLING_PROJECT_PATH to override path to 'kibana-load-testing' project + */ export async function GatlingTestRunner({ getService }: FtrProviderContext) { const log = getService('log'); - const gatlingProjectRootPath = resolve(REPO_ROOT, '../kibana-load-testing'); await withProcRunner(log, async (procs) => { - await procs.run('gatling', { + await procs.run('mvn: clean compile', { cmd: 'mvn', args: [ - 'clean', - '-q', + '-Dmaven.wagon.http.retryHandler.count=3', '-Dmaven.test.failure.ignore=true', - 'compile', - 'gatling:test', '-q', - '-Dgatling.simulationClass=org.kibanaLoadTest.simulation.DemoJourney', + 'clean', + 'compile', ], cwd: gatlingProjectRootPath, env: { @@ -31,5 +65,20 @@ export async function GatlingTestRunner({ getService }: FtrProviderContext) { }, wait: true, }); + for (const simulationClass of simulationClasses) { + await procs.run('gatling: test', { + cmd: 'mvn', + args: [ + 'gatling:test', + '-q', + `-Dgatling.simulationClass=${simulationPackage}.${simulationClass}`, + ], + cwd: gatlingProjectRootPath, + env: { + ...process.env, + }, + wait: true, + }); + } }); } From da28bd214cce0f397018b3ece97bf5e65f932bc3 Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Mon, 1 Feb 2021 15:05:42 -0700 Subject: [PATCH 2/3] TS project refs: Migrates grokdebugger (#89652) --- x-pack/plugins/grokdebugger/tsconfig.json | 24 +++++++++++++++++++++++ x-pack/test/tsconfig.json | 1 + x-pack/tsconfig.json | 2 ++ x-pack/tsconfig.refs.json | 1 + 4 files changed, 28 insertions(+) create mode 100644 x-pack/plugins/grokdebugger/tsconfig.json diff --git a/x-pack/plugins/grokdebugger/tsconfig.json b/x-pack/plugins/grokdebugger/tsconfig.json new file mode 100644 index 0000000000000..34cf8d74c0024 --- /dev/null +++ b/x-pack/plugins/grokdebugger/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + "common/**/*", + "public/**/*", + "server/**/*", + "../../typings/**/*", + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/dev_tools/tsconfig.json"}, + { "path": "../../../src/plugins/home/tsconfig.json"}, + { "path": "../../../src/plugins/kibana_react/tsconfig.json" }, + { "path": "../../../src/plugins/es_ui_shared/tsconfig.json" }, + { "path": "../licensing/tsconfig.json" } + ] +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 4c77a12c5c01a..a4d7b1a870c38 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -64,6 +64,7 @@ { "path": "../plugins/global_search_bar/tsconfig.json" }, { "path": "../plugins/ingest_pipelines/tsconfig.json" }, { "path": "../plugins/license_management/tsconfig.json" }, + { "path": "../plugins/grokdebugger/tsconfig.json" }, { "path": "../plugins/painless_lab/tsconfig.json" }, { "path": "../plugins/watcher/tsconfig.json" } ] diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 7ff1e10e8a58b..19633b6a887e8 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -44,6 +44,7 @@ "plugins/license_management/**/*", "plugins/painless_lab/**/*", "plugins/watcher/**/*", + "plugins/grokdebugger/**/*", "test/**/*" ], "compilerOptions": { @@ -119,6 +120,7 @@ { "path": "./plugins/translations/tsconfig.json" }, { "path": "./plugins/triggers_actions_ui/tsconfig.json"}, { "path": "./plugins/stack_alerts/tsconfig.json"}, + { "path": "./plugins/grokdebugger/tsconfig.json" }, { "path": "./plugins/ingest_pipelines/tsconfig.json"}, { "path": "./plugins/license_management/tsconfig.json" }, { "path": "./plugins/painless_lab/tsconfig.json" }, diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index eff35147a1da9..b6c4f63f34c11 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -44,6 +44,7 @@ { "path": "./plugins/cloud/tsconfig.json" }, { "path": "./plugins/saved_objects_tagging/tsconfig.json" }, { "path": "./plugins/global_search_bar/tsconfig.json" }, + { "path": "./plugins/grokdebugger/tsconfig.json" }, { "path": "./plugins/ingest_pipelines/tsconfig.json" }, { "path": "./plugins/license_management/tsconfig.json" }, { "path": "./plugins/painless_lab/tsconfig.json" }, From 3c0d30cbfd6864ec7ceca60c8b5ef59d76ce2917 Mon Sep 17 00:00:00 2001 From: Kevin Qualters <56408403+kqualters-elastic@users.noreply.github.com> Date: Mon, 1 Feb 2021 17:34:57 -0500 Subject: [PATCH 3/3] Add --ssl flag to make resolver generator use ssl with kbn and elasticsearch clients (#89873) --- .../endpoint/resolver_generator_script.ts | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index 9ad094086b632..de0cec3c06033 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -5,9 +5,10 @@ */ /* eslint-disable no-console */ import yargs from 'yargs'; +import fs from 'fs'; import { Client, ClientOptions } from '@elastic/elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; -import { KbnClient, ToolingLog } from '@kbn/dev-utils'; +import { KbnClient, ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils'; import { AxiosResponse } from 'axios'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data'; @@ -202,15 +203,41 @@ async function main() { type: 'boolean', default: false, }, + ssl: { + alias: 'ssl', + describe: 'Use https for elasticsearch and kbn clients', + type: 'boolean', + default: false, + }, }).argv; + let ca: Buffer; + let kbnClient: KbnClientWithApiKeySupport; + let clientOptions: ClientOptions; - const kbnClient = new KbnClientWithApiKeySupport({ - log: new ToolingLog({ - level: 'info', - writeTo: process.stdout, - }), - url: argv.kibana, - }); + if (argv.ssl) { + ca = fs.readFileSync(CA_CERT_PATH); + const url = argv.kibana.replace('http:', 'https:'); + const node = argv.node.replace('http:', 'https:'); + kbnClient = new KbnClientWithApiKeySupport({ + log: new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }), + url, + certificateAuthorities: [ca], + }); + clientOptions = { node, ssl: { ca: [ca] } }; + } else { + kbnClient = new KbnClientWithApiKeySupport({ + log: new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }), + url: argv.kibana, + }); + clientOptions = { node: argv.node }; + } + const client = new Client(clientOptions); try { await doIngestSetup(kbnClient); @@ -219,9 +246,6 @@ async function main() { process.exit(1); } - const clientOptions: ClientOptions = { node: argv.node }; - const client = new Client(clientOptions); - if (argv.delete) { await deleteIndices( [argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex],