From 4da7d988e41e30c7b2db1eece39695e4de37ec17 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 19 Aug 2021 09:40:23 -0700 Subject: [PATCH] [ftr] rework ciGroup validation to remove JOBS.yml and avoid duplication (#109149) Co-authored-by: spalger # Conflicts: # .ci/jobs.yml # test/scripts/jenkins_code_coverage.sh --- .ci/ci_groups.yml | 29 +++++++++++ .ci/jobs.yml | 41 --------------- .../lib/config/schema.ts | 1 + .../lib/mocha/decorate_mocha_ui.js | 10 +++- .../lib/mocha/load_test_files.js | 1 + .../src/functional_tests/lib/run_ftr.js | 4 +- .../kbn-test/src/functional_tests/tasks.js | 2 +- scripts/ensure_all_tests_in_ci_group.js | 2 +- src/dev/ensure_all_tests_in_ci_group.ts | 52 +++++++++++++++++++ src/dev/run_ensure_all_tests_in_ci_group.js | 50 ------------------ test/api_integration/config.js | 1 + test/examples/config.js | 1 + test/interpreter_functional/config.ts | 1 + test/plugin_functional/config.ts | 1 + test/scripts/jenkins_build_kibana.sh | 19 +------ 15 files changed, 100 insertions(+), 115 deletions(-) create mode 100644 .ci/ci_groups.yml delete mode 100644 .ci/jobs.yml create mode 100644 src/dev/ensure_all_tests_in_ci_group.ts delete mode 100644 src/dev/run_ensure_all_tests_in_ci_group.js diff --git a/.ci/ci_groups.yml b/.ci/ci_groups.yml new file mode 100644 index 0000000000000..6d1fb2234406c --- /dev/null +++ b/.ci/ci_groups.yml @@ -0,0 +1,29 @@ +root: + - ciGroup1 + - ciGroup2 + - ciGroup3 + - ciGroup4 + - ciGroup5 + - ciGroup6 + - ciGroup7 + - ciGroup8 + - ciGroup9 + - ciGroup10 + - ciGroup11 + - ciGroup12 + +xpack: + - ciGroup1 + - ciGroup2 + - ciGroup3 + - ciGroup4 + - ciGroup5 + - ciGroup6 + - ciGroup7 + - ciGroup8 + - ciGroup9 + - ciGroup10 + - ciGroup11 + - ciGroup12 + - ciGroup13 + - ciGroupDocker diff --git a/.ci/jobs.yml b/.ci/jobs.yml deleted file mode 100644 index 481029f8f76aa..0000000000000 --- a/.ci/jobs.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This file is needed by node scripts/ensure_all_tests_in_ci_group for the list of ciGroups. That must be changed before this file can be removed - -JOB: - - kibana-intake - - kibana-firefoxSmoke - - kibana-ciGroup1 - - kibana-ciGroup2 - - kibana-ciGroup3 - - kibana-ciGroup4 - - kibana-ciGroup5 - - kibana-ciGroup6 - - kibana-ciGroup7 - - kibana-ciGroup8 - - kibana-ciGroup9 - - kibana-ciGroup10 - - kibana-ciGroup11 - - kibana-ciGroup12 - - kibana-accessibility - - kibana-visualRegression - - # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh - - x-pack-firefoxSmoke - - x-pack-ciGroup1 - - x-pack-ciGroup2 - - x-pack-ciGroup3 - - x-pack-ciGroup4 - - x-pack-ciGroup5 - - x-pack-ciGroup6 - - x-pack-ciGroup7 - - x-pack-ciGroup8 - - x-pack-ciGroup9 - - x-pack-ciGroup10 - - x-pack-ciGroup11 - - x-pack-ciGroup12 - - x-pack-ciGroup13 - - x-pack-ciGroupDocker - - x-pack-accessibility - - x-pack-visualRegression - -# `~` is yaml for `null` -exclude: ~ \ No newline at end of file diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index f9877447840da..011a716317fd2 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -71,6 +71,7 @@ const defaultRelativeToConfigPath = (path: string) => { export const schema = Joi.object() .keys({ + rootTags: Joi.array().items(Joi.string()), testFiles: Joi.array().items(Joi.string()), testRunner: Joi.func(), diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js index 3832dc7c59e19..c6693245da28b 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js @@ -21,7 +21,7 @@ function split(arr, fn) { return [a, b]; } -export function decorateMochaUi(log, lifecycle, context, { isDockerGroup }) { +export function decorateMochaUi(log, lifecycle, context, { isDockerGroup, rootTags }) { // incremented at the start of each suite, decremented after // so that in each non-suite call we can know if we are within // a suite, or that when a suite is defined it is within a suite @@ -62,7 +62,13 @@ export function decorateMochaUi(log, lifecycle, context, { isDockerGroup }) { }); const relativeFilePath = relative(REPO_ROOT, this.file); - this._tags = isDockerGroup ? ['ciGroupDocker', relativeFilePath] : [relativeFilePath]; + this._tags = [ + ...(isDockerGroup ? ['ciGroupDocker', relativeFilePath] : [relativeFilePath]), + // we attach the "root tags" to all the child suites of the root suite, so that if they + // need to be excluded they can be removed from the root suite without removing the entire + // root suite + ...(this.parent.root ? [...(rootTags ?? [])] : []), + ]; this.suiteTag = relativeFilePath; // The tag that uniquely targets this suite/file this.tags = (tags) => { const newTags = Array.isArray(tags) ? tags : [tags]; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js index b6cc73cdb08c8..59f8f003004b6 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js @@ -62,6 +62,7 @@ export const loadTestFiles = ({ const context = decorateMochaUi(log, lifecycle, global, { isDockerGroup, + rootTags: config.get('rootTags'), }); mocha.suite.emit('pre-require', context, path, mocha); diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js index b84d01fbebbe1..40937b8b4fc2d 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -52,9 +52,9 @@ export async function assertNoneExcluded({ configPath, options }) { throw new CliError(` ${stats.excludedTests.length} tests in the ${configPath} config are excluded when filtering by the tags run on CI. Make sure that all suites are - tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh + tagged with one of the following tags: - tags: ${JSON.stringify(options.suiteTags)} + ${JSON.stringify(options.suiteTags)} - ${stats.excludedTests.join('\n - ')} `); diff --git a/packages/kbn-test/src/functional_tests/tasks.js b/packages/kbn-test/src/functional_tests/tasks.js index f4dfdcb93fb2b..1e51906203f09 100644 --- a/packages/kbn-test/src/functional_tests/tasks.js +++ b/packages/kbn-test/src/functional_tests/tasks.js @@ -55,7 +55,7 @@ const makeSuccessMessage = (options) => { * @property {string} options.esFrom Optionally run from source instead of snapshot */ export async function runTests(options) { - if (!process.env.KBN_NP_PLUGINS_BUILT) { + if (!process.env.KBN_NP_PLUGINS_BUILT && !options.assertNoneExcluded) { const log = options.createLogger(); log.warning('❗️❗️❗️'); log.warning('❗️❗️❗️'); diff --git a/scripts/ensure_all_tests_in_ci_group.js b/scripts/ensure_all_tests_in_ci_group.js index 757e2b28c75e3..7e382c4a20d17 100644 --- a/scripts/ensure_all_tests_in_ci_group.js +++ b/scripts/ensure_all_tests_in_ci_group.js @@ -7,4 +7,4 @@ */ require('../src/setup_node_env'); -require('../src/dev/run_ensure_all_tests_in_ci_group'); +require('../src/dev/ensure_all_tests_in_ci_group').runEnsureAllTestsInCiGroupsCli(); diff --git a/src/dev/ensure_all_tests_in_ci_group.ts b/src/dev/ensure_all_tests_in_ci_group.ts new file mode 100644 index 0000000000000..aeccefae05d2c --- /dev/null +++ b/src/dev/ensure_all_tests_in_ci_group.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs/promises'; + +import execa from 'execa'; +import { safeLoad } from 'js-yaml'; + +import { run, REPO_ROOT } from '@kbn/dev-utils'; +import { schema } from '@kbn/config-schema'; + +const RELATIVE_JOBS_YAML_PATH = '.ci/ci_groups.yml'; +const JOBS_YAML_PATH = Path.resolve(REPO_ROOT, RELATIVE_JOBS_YAML_PATH); +const SCHEMA = schema.object({ + root: schema.arrayOf(schema.string()), + xpack: schema.arrayOf(schema.string()), +}); + +export function runEnsureAllTestsInCiGroupsCli() { + run(async ({ log }) => { + const { root, xpack } = SCHEMA.validate(safeLoad(await Fs.readFile(JOBS_YAML_PATH, 'utf-8'))); + + log.info( + 'validating root tests directory contains all "root" ciGroups from', + RELATIVE_JOBS_YAML_PATH + ); + await execa(process.execPath, [ + 'scripts/functional_tests', + ...root.map((tag) => `--include-tag=${tag}`), + '--include-tag=runOutsideOfCiGroups', + '--assert-none-excluded', + ]); + + log.info( + 'validating x-pack/tests directory contains all "xpack" ciGroups from', + RELATIVE_JOBS_YAML_PATH + ); + await execa(process.execPath, [ + 'x-pack/scripts/functional_tests', + ...xpack.map((tag) => `--include-tag=${tag}`), + '--assert-none-excluded', + ]); + + log.success('all tests are in a valid ciGroup'); + }); +} diff --git a/src/dev/run_ensure_all_tests_in_ci_group.js b/src/dev/run_ensure_all_tests_in_ci_group.js deleted file mode 100644 index b5e4798d8d800..0000000000000 --- a/src/dev/run_ensure_all_tests_in_ci_group.js +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { readFileSync } from 'fs'; -import { resolve } from 'path'; - -import execa from 'execa'; -import { safeLoad } from 'js-yaml'; - -import { run } from '@kbn/dev-utils'; - -const JOBS_YAML = readFileSync(resolve(__dirname, '../../.ci/jobs.yml'), 'utf8'); -const TEST_TAGS = safeLoad(JOBS_YAML) - .JOB.filter((id) => id.startsWith('kibana-ciGroup')) - .map((id) => id.replace(/^kibana-/, '')); - -run(async ({ log }) => { - try { - const result = await execa(process.execPath, [ - 'scripts/functional_test_runner', - ...TEST_TAGS.map((tag) => `--include-tag=${tag}`), - '--config', - 'test/functional/config.js', - '--test-stats', - ]); - const stats = JSON.parse(result.stderr); - - if (stats.excludedTests.length > 0) { - log.error(` - ${stats.excludedTests.length} tests are excluded by the ciGroup tags, make sure that - all test suites have a "ciGroup{X}" tag and that "tasks/functional_test_groups.js" - knows about the tag that you are using. - - tags: ${JSON.stringify({ include: TEST_TAGS })} - - - ${stats.excludedTests.join('\n - ')} - `); - process.exitCode = 1; - return; - } - } catch (error) { - log.error(error.stack); - process.exitCode = 1; - } -}); diff --git a/test/api_integration/config.js b/test/api_integration/config.js index 84fb0b7907c3b..4988094dad7a2 100644 --- a/test/api_integration/config.js +++ b/test/api_integration/config.js @@ -13,6 +13,7 @@ export default async function ({ readConfigFile }) { const functionalConfig = await readConfigFile(require.resolve('../functional/config')); return { + rootTags: ['runOutsideOfCiGroups'], testFiles: [require.resolve('./apis')], services, servers: commonConfig.get('servers'), diff --git a/test/examples/config.js b/test/examples/config.js index d47748e5f22a9..5530794c7525f 100644 --- a/test/examples/config.js +++ b/test/examples/config.js @@ -21,6 +21,7 @@ export default async function ({ readConfigFile }) { ); return { + rootTags: ['runOutsideOfCiGroups'], testFiles: [ require.resolve('./embeddables'), require.resolve('./bfetch_explorer'), diff --git a/test/interpreter_functional/config.ts b/test/interpreter_functional/config.ts index c0ec982fb98b6..3f9c846a51429 100644 --- a/test/interpreter_functional/config.ts +++ b/test/interpreter_functional/config.ts @@ -20,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { ); return { + rootTags: ['runOutsideOfCiGroups'], testFiles: [require.resolve('./test_suites/run_pipeline')], services: functionalConfig.get('services'), pageObjects: functionalConfig.get('pageObjects'), diff --git a/test/plugin_functional/config.ts b/test/plugin_functional/config.ts index 3791e663fb68c..e2b7457ffec78 100644 --- a/test/plugin_functional/config.ts +++ b/test/plugin_functional/config.ts @@ -20,6 +20,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { ); return { + rootTags: ['runOutsideOfCiGroups'], testFiles: [ require.resolve('./test_suites/usage_collection'), require.resolve('./test_suites/telemetry'), diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index f165a3de0621b..7c75f18180298 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -11,24 +11,7 @@ fi export KBN_NP_PLUGINS_BUILT=true echo " -> Ensuring all functional tests are in a ciGroup" -node scripts/ensure_all_tests_in_ci_group; - -echo " -> Ensuring all x-pack functional tests are in a ciGroup" -node x-pack/scripts/functional_tests --assert-none-excluded \ - --include-tag ciGroup1 \ - --include-tag ciGroup2 \ - --include-tag ciGroup3 \ - --include-tag ciGroup4 \ - --include-tag ciGroup5 \ - --include-tag ciGroup6 \ - --include-tag ciGroup7 \ - --include-tag ciGroup8 \ - --include-tag ciGroup9 \ - --include-tag ciGroup10 \ - --include-tag ciGroup11 \ - --include-tag ciGroup12 \ - --include-tag ciGroup13 \ - --include-tag ciGroupDocker +node scripts/ensure_all_tests_in_ci_group echo " -> building and extracting default Kibana distributable for use in functional tests" node scripts/build --debug