[8.7] [ftr] fix scripts/functional_tests to respect cli flags (#155734)…

… (#155907)

# Backport

This will backport the following commits from `main` to `8.7`:
- [[ftr] fix scripts/functional_tests to respect cli flags
(#155734)](#155734)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Dzmitry
Lemechko","email":"[email protected]"},"sourceCommit":{"committedDate":"2023-04-26T16:33:54Z","message":"[ftr]
fix scripts/functional_tests to respect cli flags (#155734)\n\n##
Summary\r\n\r\nThis\r\n[failure](https://buildkite.com/elastic/kibana-on-merge/builds/29091#01879988-ecd2-4e4f-bfb4-108939f145a1)\r\nclearly
shows that `--bail` flag is ignored when passed
to\r\n`scripts/functional_tests.js` script, and
since\r\n`scripts/functional_tests.js --help` list these flags I think
we need to\r\nfix it:\r\n\r\n```\r\n --include-tag Tags that suites must
include to be run, can be included multiple times\r\n --exclude-tag Tags
that suites must NOT include to be run, can be included multiple
times\r\n --include Files that must included to be run, can be included
multiple times\r\n --exclude Files that must NOT be included to be run,
can be included multiple times\r\n --grep Pattern to select which tests
to run\r\n --bail Stop the test run at the first failure\r\n --dry-run
Report tests without executing them\r\n --updateBaselines Replace
baseline screenshots with whatever is generated from the test\r\n
--updateSnapshots Replace inline and file snapshots with whatever is
generated from the test\r\n```\r\n\r\nI was able to reproduce it
locally:\r\n1.
Break\r\n[test/functional/apps/console/_console.ts](test/functional/apps/console/_console.ts)\r\nby
adding `expect(1).to.be(2);` in the first `it` function\r\n2. Run `node
scripts/functional_tests.js --bail
--config\r\ntest/functional/apps/console/config.ts`\r\nActual: Tests
continue to run after failure\r\nExpected: Stop tests after first
failure\r\n\r\nIt turned out `scripts/functional_test_runner.js`
respects the flags so\r\nI just copied the logic
from\r\n[packages/kbn-test/src/functional_test_runner/cli.ts](https://github.com/elastic/kibana/blob/main/packages/kbn-test/src/functional_test_runner/cli.ts#L41-L63)\r\n\r\nLet
me know if you think we need to add jest
tests.\r\n\r\n\r\nTested:\r\n\r\n```\r\nnode scripts/functional_tests.js
--bail --config test/functional/apps/console/config.ts --grep \"multiple
requests
output\"\r\n```","sha":"4211e03a5f7651afeea492e44116eb614b54f00c","branchLabelMapping":{"^v8.8.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v7.17.10","v8.8.0","v8.7.2"],"number":155734,"url":"https://github.com/elastic/kibana/pull/155734","mergeCommit":{"message":"[ftr]
fix scripts/functional_tests to respect cli flags (#155734)\n\n##
Summary\r\n\r\nThis\r\n[failure](https://buildkite.com/elastic/kibana-on-merge/builds/29091#01879988-ecd2-4e4f-bfb4-108939f145a1)\r\nclearly
shows that `--bail` flag is ignored when passed
to\r\n`scripts/functional_tests.js` script, and
since\r\n`scripts/functional_tests.js --help` list these flags I think
we need to\r\nfix it:\r\n\r\n```\r\n --include-tag Tags that suites must
include to be run, can be included multiple times\r\n --exclude-tag Tags
that suites must NOT include to be run, can be included multiple
times\r\n --include Files that must included to be run, can be included
multiple times\r\n --exclude Files that must NOT be included to be run,
can be included multiple times\r\n --grep Pattern to select which tests
to run\r\n --bail Stop the test run at the first failure\r\n --dry-run
Report tests without executing them\r\n --updateBaselines Replace
baseline screenshots with whatever is generated from the test\r\n
--updateSnapshots Replace inline and file snapshots with whatever is
generated from the test\r\n```\r\n\r\nI was able to reproduce it
locally:\r\n1.
Break\r\n[test/functional/apps/console/_console.ts](test/functional/apps/console/_console.ts)\r\nby
adding `expect(1).to.be(2);` in the first `it` function\r\n2. Run `node
scripts/functional_tests.js --bail
--config\r\ntest/functional/apps/console/config.ts`\r\nActual: Tests
continue to run after failure\r\nExpected: Stop tests after first
failure\r\n\r\nIt turned out `scripts/functional_test_runner.js`
respects the flags so\r\nI just copied the logic
from\r\n[packages/kbn-test/src/functional_test_runner/cli.ts](https://github.com/elastic/kibana/blob/main/packages/kbn-test/src/functional_test_runner/cli.ts#L41-L63)\r\n\r\nLet
me know if you think we need to add jest
tests.\r\n\r\n\r\nTested:\r\n\r\n```\r\nnode scripts/functional_tests.js
--bail --config test/functional/apps/console/config.ts --grep \"multiple
requests
output\"\r\n```","sha":"4211e03a5f7651afeea492e44116eb614b54f00c"}},"sourceBranch":"main","suggestedTargetBranches":["7.17","8.7"],"targetPullRequestStates":[{"branch":"7.17","label":"v7.17.10","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.8.0","labelRegex":"^v8.8.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/155734","number":155734,"mergeCommit":{"message":"[ftr]
fix scripts/functional_tests to respect cli flags (#155734)\n\n##
Summary\r\n\r\nThis\r\n[failure](https://buildkite.com/elastic/kibana-on-merge/builds/29091#01879988-ecd2-4e4f-bfb4-108939f145a1)\r\nclearly
shows that `--bail` flag is ignored when passed
to\r\n`scripts/functional_tests.js` script, and
since\r\n`scripts/functional_tests.js --help` list these flags I think
we need to\r\nfix it:\r\n\r\n```\r\n --include-tag Tags that suites must
include to be run, can be included multiple times\r\n --exclude-tag Tags
that suites must NOT include to be run, can be included multiple
times\r\n --include Files that must included to be run, can be included
multiple times\r\n --exclude Files that must NOT be included to be run,
can be included multiple times\r\n --grep Pattern to select which tests
to run\r\n --bail Stop the test run at the first failure\r\n --dry-run
Report tests without executing them\r\n --updateBaselines Replace
baseline screenshots with whatever is generated from the test\r\n
--updateSnapshots Replace inline and file snapshots with whatever is
generated from the test\r\n```\r\n\r\nI was able to reproduce it
locally:\r\n1.
Break\r\n[test/functional/apps/console/_console.ts](test/functional/apps/console/_console.ts)\r\nby
adding `expect(1).to.be(2);` in the first `it` function\r\n2. Run `node
scripts/functional_tests.js --bail
--config\r\ntest/functional/apps/console/config.ts`\r\nActual: Tests
continue to run after failure\r\nExpected: Stop tests after first
failure\r\n\r\nIt turned out `scripts/functional_test_runner.js`
respects the flags so\r\nI just copied the logic
from\r\n[packages/kbn-test/src/functional_test_runner/cli.ts](https://github.com/elastic/kibana/blob/main/packages/kbn-test/src/functional_test_runner/cli.ts#L41-L63)\r\n\r\nLet
me know if you think we need to add jest
tests.\r\n\r\n\r\nTested:\r\n\r\n```\r\nnode scripts/functional_tests.js
--bail --config test/functional/apps/console/config.ts --grep \"multiple
requests
output\"\r\n```","sha":"4211e03a5f7651afeea492e44116eb614b54f00c"}},{"branch":"8.7","label":"v8.7.2","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Dzmitry Lemechko <[email protected]>

packages/kbn-test/src/functional_tests/run_tests/flags.ts

@@ -78,12 +78,12 @@ export function parseFlags(flags: FlagsReader) {
     installDir: flags.path('kibana-install-dir'),
     grep: flags.string('grep'),
     suiteTags: {
-      include: flags.arrayOfStrings('include-tag'),
-      exclude: flags.arrayOfStrings('exclude-tag'),
+      include: flags.arrayOfStrings('include-tag') ?? [],
+      exclude: flags.arrayOfStrings('exclude-tag') ?? [],
     },
     suiteFilters: {
-      include: flags.arrayOfPaths('include'),
-      exclude: flags.arrayOfPaths('exclude'),
+      include: flags.arrayOfPaths('include') ?? [],
+      exclude: flags.arrayOfPaths('exclude') ?? [],
     },
   };
 }

packages/kbn-test/src/functional_tests/run_tests/run_tests.ts

@@ -36,14 +36,35 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
     log.warning('❗️❗️❗️');
   }
 
+  const settingOverrides = {
+    mochaOpts: {
+      bail: options.bail,
+      dryRun: options.dryRun,
+      grep: options.grep,
+    },
+    kbnTestServer: {
+      installDir: options.installDir,
+    },
+    suiteFiles: {
+      include: options.suiteFilters.include,
+      exclude: options.suiteFilters.exclude,
+    },
+    suiteTags: {
+      include: options.suiteTags.include,
+      exclude: options.suiteTags.exclude,
+    },
+    updateBaselines: options.updateBaselines,
+    updateSnapshots: options.updateSnapshots,
+  };
+
   for (const [i, path] of options.configs.entries()) {
     await log.indent(0, async () => {
       if (options.configs.length > 1) {
         const progress = `${i + 1}/${options.configs.length}`;
         log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, path)}`);
       }
 
-      const config = await readConfigFile(log, options.esVersion, path);
+      const config = await readConfigFile(log, options.esVersion, path, settingOverrides);
 
       const hasTests = await checkForEnabledTestsInFtrConfig({
         config,