From a387c0f80e81a1ddc57a216572e41829f7e9a0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Thu, 27 Apr 2023 17:19:35 +0200 Subject: [PATCH] [8.8] [Serverless] Select project type via config (#155754) (#155997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.8`: - [[Serverless] Select project type via config (#155754)](https://github.com/elastic/kibana/pull/155754) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- config/serverless.oblt.yml | 1 + .../src/bootstrap.test.mocks.ts | 25 ++++++ .../src/bootstrap.test.ts | 61 +++++++++++++ .../src/bootstrap.ts | 46 +++++++++- .../src/register_service_config.ts | 2 + .../src/root/serverless_config.ts | 34 +++++++ .../serverless_config_flag.test.ts | 88 +++++++++++++++++++ src/cli/serve/serve.js | 74 +++++++++++----- 8 files changed, 307 insertions(+), 24 deletions(-) create mode 100644 packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts create mode 100644 packages/core/root/core-root-server-internal/src/bootstrap.test.ts create mode 100644 packages/core/root/core-root-server-internal/src/root/serverless_config.ts create mode 100644 src/cli/serve/integration_tests/serverless_config_flag.test.ts diff --git a/config/serverless.oblt.yml b/config/serverless.oblt.yml index ba76648238348..945142d48d8db 100644 --- a/config/serverless.oblt.yml +++ b/config/serverless.oblt.yml @@ -1 +1,2 @@ +uiSettings.overrides.defaultRoute: /app/observability/overview xpack.infra.logs.app_target: discover diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts b/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts new file mode 100644 index 0000000000000..07277d565f694 --- /dev/null +++ b/packages/core/root/core-root-server-internal/src/bootstrap.test.mocks.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Env } from '@kbn/config'; +import { rawConfigServiceMock, configServiceMock } from '@kbn/config-mocks'; + +export const mockConfigService = configServiceMock.create(); +export const mockRawConfigService = rawConfigServiceMock.create(); +export const mockRawConfigServiceConstructor = jest.fn(() => mockRawConfigService); +jest.doMock('@kbn/config', () => ({ + ConfigService: jest.fn(() => mockConfigService), + Env, + RawConfigService: jest.fn(mockRawConfigServiceConstructor), +})); + +jest.doMock('./root', () => ({ + Root: jest.fn(() => ({ + shutdown: jest.fn(), + })), +})); diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.test.ts b/packages/core/root/core-root-server-internal/src/bootstrap.test.ts new file mode 100644 index 0000000000000..1bd413314aa98 --- /dev/null +++ b/packages/core/root/core-root-server-internal/src/bootstrap.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { of } from 'rxjs'; +import type { CliArgs } from '@kbn/config'; + +import { mockRawConfigService, mockRawConfigServiceConstructor } from './bootstrap.test.mocks'; + +jest.mock('@kbn/core-logging-server-internal'); + +import { bootstrap } from './bootstrap'; + +const bootstrapCfg = { + configs: ['config/kibana.yml'], + cliArgs: {} as unknown as CliArgs, + applyConfigOverrides: () => ({}), +}; + +describe('bootstrap', () => { + describe('serverless', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('should load additional serverless files for a valid project', async () => { + mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'es' })); + await bootstrap(bootstrapCfg); + expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(2); + expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith( + 1, + bootstrapCfg.configs, + bootstrapCfg.applyConfigOverrides + ); + expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith( + 2, + [ + expect.stringContaining('config/serverless.yml'), + expect.stringContaining('config/serverless.es.yml'), + ...bootstrapCfg.configs, + ], + bootstrapCfg.applyConfigOverrides + ); + }); + + test('should skip loading the serverless files for an invalid project', async () => { + mockRawConfigService.getConfig$.mockReturnValue(of({ serverless: 'not-valid' })); + await bootstrap(bootstrapCfg); + expect(mockRawConfigServiceConstructor).toHaveBeenCalledTimes(1); + expect(mockRawConfigServiceConstructor).toHaveBeenNthCalledWith( + 1, + bootstrapCfg.configs, + bootstrapCfg.applyConfigOverrides + ); + }); + }); +}); diff --git a/packages/core/root/core-root-server-internal/src/bootstrap.ts b/packages/core/root/core-root-server-internal/src/bootstrap.ts index 3b5a340a7b79c..3f55b6493a6bd 100644 --- a/packages/core/root/core-root-server-internal/src/bootstrap.ts +++ b/packages/core/root/core-root-server-internal/src/bootstrap.ts @@ -7,9 +7,14 @@ */ import chalk from 'chalk'; +import { firstValueFrom } from 'rxjs'; import { getPackages } from '@kbn/repo-packages'; import { CliArgs, Env, RawConfigService } from '@kbn/config'; import { CriticalError } from '@kbn/core-base-server-internal'; +import { resolve } from 'path'; +import { getConfigDirectory } from '@kbn/utils'; +import { statSync } from 'fs'; +import { VALID_SERVERLESS_PROJECT_TYPES } from './root/serverless_config'; import { Root } from './root'; import { MIGRATION_EXCEPTION_CODE } from './constants'; @@ -38,15 +43,40 @@ export async function bootstrap({ configs, cliArgs, applyConfigOverrides }: Boot // eslint-disable-next-line @typescript-eslint/no-var-requires const { REPO_ROOT } = require('@kbn/repo-info'); - const env = Env.createDefault(REPO_ROOT, { + let env = Env.createDefault(REPO_ROOT, { configs, cliArgs, repoPackages: getPackages(REPO_ROOT), }); - const rawConfigService = new RawConfigService(env.configs, applyConfigOverrides); + let rawConfigService = new RawConfigService(env.configs, applyConfigOverrides); rawConfigService.loadConfig(); + // Hack to load the extra serverless config files if `serverless: {projectType}` is found in it. + const rawConfig = await firstValueFrom(rawConfigService.getConfig$()); + const serverlessProjectType = rawConfig?.serverless; + if ( + typeof serverlessProjectType === 'string' && + VALID_SERVERLESS_PROJECT_TYPES.includes(serverlessProjectType) + ) { + const extendedConfigs = [ + ...['serverless.yml', `serverless.${serverlessProjectType}.yml`] + .map((name) => resolve(getConfigDirectory(), name)) + .filter(configFileExists), + ...configs, + ]; + + env = Env.createDefault(REPO_ROOT, { + configs: extendedConfigs, + cliArgs: { ...cliArgs, serverless: true }, + repoPackages: getPackages(REPO_ROOT), + }); + + rawConfigService.stop(); + rawConfigService = new RawConfigService(env.configs, applyConfigOverrides); + rawConfigService.loadConfig(); + } + const root = new Root(rawConfigService, env, onRootShutdown); process.on('SIGHUP', () => reloadConfiguration()); @@ -128,3 +158,15 @@ function onRootShutdown(reason?: any) { process.exit(0); } + +function configFileExists(path: string) { + try { + return statSync(path).isFile(); + } catch (err) { + if (err.code === 'ENOENT') { + return false; + } + + throw err; + } +} diff --git a/packages/core/root/core-root-server-internal/src/register_service_config.ts b/packages/core/root/core-root-server-internal/src/register_service_config.ts index a22ea56f25ee9..f646f9e538ae8 100644 --- a/packages/core/root/core-root-server-internal/src/register_service_config.ts +++ b/packages/core/root/core-root-server-internal/src/register_service_config.ts @@ -28,6 +28,7 @@ import { uiSettingsConfig } from '@kbn/core-ui-settings-server-internal'; import { config as pluginsConfig } from '@kbn/core-plugins-server-internal'; import { elasticApmConfig } from './root/elastic_config'; +import { serverlessConfig } from './root/serverless_config'; const rootConfigPath = ''; @@ -49,6 +50,7 @@ export function registerServiceConfig(configService: ConfigService) { pluginsConfig, savedObjectsConfig, savedObjectsMigrationConfig, + serverlessConfig, statusConfig, uiSettingsConfig, ]; diff --git a/packages/core/root/core-root-server-internal/src/root/serverless_config.ts b/packages/core/root/core-root-server-internal/src/root/serverless_config.ts new file mode 100644 index 0000000000000..351065f7d83df --- /dev/null +++ b/packages/core/root/core-root-server-internal/src/root/serverless_config.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema, TypeOf, Type } from '@kbn/config-schema'; +import { ServiceConfigDescriptor } from '@kbn/core-base-server-internal'; + +// Config validation for how to run Kibana in Serverless mode. +// Clients need to specify the project type to run in. +// Going for a simple `serverless` string because it serves as +// a direct replacement to the legacy --serverless CLI flag. +// If we even decide to extend this further, and converting it into an object, +// BWC can be ensured by adding the object definition as another alternative to `schema.oneOf`. + +export const VALID_SERVERLESS_PROJECT_TYPES = ['es', 'oblt', 'security']; + +const serverlessConfigSchema = schema.maybe( + schema.oneOf( + VALID_SERVERLESS_PROJECT_TYPES.map((projectName) => schema.literal(projectName)) as [ + Type // This cast is needed because it's different to Type[] :sight: + ] + ) +); + +export type ServerlessConfigType = TypeOf; + +export const serverlessConfig: ServiceConfigDescriptor = { + path: 'serverless', + schema: serverlessConfigSchema, +}; diff --git a/src/cli/serve/integration_tests/serverless_config_flag.test.ts b/src/cli/serve/integration_tests/serverless_config_flag.test.ts new file mode 100644 index 0000000000000..6c67ba6261eb4 --- /dev/null +++ b/src/cli/serve/integration_tests/serverless_config_flag.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { spawn, spawnSync } from 'child_process'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; +import { filter, firstValueFrom, from, take, concatMap } from 'rxjs'; + +import { REPO_ROOT } from '@kbn/repo-info'; +import { getConfigDirectory } from '@kbn/utils'; + +describe('cli serverless project type', () => { + it( + 'exits with statusCode 1 and logs an error when serverless project type is invalid', + () => { + const { error, status, stdout } = spawnSync( + process.execPath, + ['scripts/kibana', '--serverless=non-existing-project-type'], + { + cwd: REPO_ROOT, + } + ); + expect(error).toBe(undefined); + + expect(stdout.toString('utf8')).toContain( + 'FATAL CLI ERROR Error: invalid --serverless value, must be one of es, oblt, security' + ); + + expect(status).toBe(1); + }, + 20 * 1000 + ); + + // Skipping this one because on CI it fails to read the config file + it.skip.each(['es', 'oblt', 'security'])( + 'writes the serverless project type %s in config/serverless.recent.yml', + async (mode) => { + // Making sure `--serverless` translates into the `serverless` config entry, and validates against the accepted values + const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], { + cwd: REPO_ROOT, + }); + + // Wait for 5 lines in the logs + await firstValueFrom(from(child.stdout).pipe(take(5))); + + expect( + readFileSync(resolve(getConfigDirectory(), 'serverless.recent.yml'), 'utf-8') + ).toContain(`serverless: ${mode}\n`); + + child.kill('SIGKILL'); + } + ); + + it.each(['es', 'oblt', 'security'])( + 'Kibana does not crash when running project type %s', + async (mode) => { + const child = spawn(process.execPath, ['scripts/kibana', `--serverless=${mode}`], { + cwd: REPO_ROOT, + }); + + // Wait until Kibana starts listening to the port + let leftover = ''; + const found = await firstValueFrom( + from(child.stdout).pipe( + concatMap((chunk: Buffer) => { + const data = leftover + chunk.toString('utf-8'); + const msgs = data.split('\n'); + leftover = msgs.pop() ?? ''; + return msgs; + }), + filter( + (msg) => + msg.includes('http server running at http://localhost:5601') || msg.includes('FATAL') + ) + ) + ); + + child.kill('SIGKILL'); + + expect(found).not.toContain('FATAL'); + } + ); +}); diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 9facf94408235..c1b9f04fd8d81 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -8,7 +8,7 @@ import { set as lodashSet } from '@kbn/safer-lodash-set'; import _ from 'lodash'; -import { statSync } from 'fs'; +import { statSync, existsSync, readFileSync, writeFileSync } from 'fs'; import { resolve } from 'path'; import url from 'url'; @@ -22,14 +22,14 @@ const VALID_SERVERLESS_PROJECT_MODE = ['es', 'oblt', 'security']; /** * @param {Record} opts - * @returns {ServerlessProjectMode | null} + * @returns {ServerlessProjectMode | true | null} */ function getServerlessProjectMode(opts) { if (!opts.serverless) { return null; } - if (VALID_SERVERLESS_PROJECT_MODE.includes(opts.serverless)) { + if (VALID_SERVERLESS_PROJECT_MODE.includes(opts.serverless) || opts.serverless === true) { return opts.serverless; } @@ -94,16 +94,6 @@ function configFileExists(name) { } } -/** - * @returns {boolean} Whether the distribution can run in Serverless mode - */ -function isServerlessCapableDistribution() { - // For now, checking if the `serverless.yml` config file exists should be enough - // We could also check the following as well, but I don't think it's necessary: - // VALID_SERVERLESS_PROJECT_MODE.some((projectType) => configFileExists(`serverless.${projectType}.yml`)) - return configFileExists('serverless.yml'); -} - /** * @param {string} name * @param {string[]} configs @@ -115,6 +105,48 @@ function maybeAddConfig(name, configs, method) { } } +/** + * @param {string} file + * @param {'es' | 'security' | 'oblt' | true} projectType + * @param {boolean} isDevMode + * @param {string[]} configs + * @param {'push' | 'unshift'} method + */ +function maybeSetRecentConfig(file, projectType, isDevMode, configs, method) { + const path = resolve(getConfigDirectory(), file); + + function writeMode(selectedProjectType) { + writeFileSync( + path, + `${ + isDevMode ? 'xpack.serverless.plugin.developer.projectSwitcher.enabled: true\n' : '' + }serverless: ${selectedProjectType}\n` + ); + } + + try { + if (projectType === true) { + if (!existsSync(path)) { + writeMode('es'); + } + } else { + const data = readFileSync(path, 'utf-8'); + const match = data.match(/serverless: (\w+)\n/); + if (!match || match[1] !== projectType) { + writeMode(projectType); + } + } + + configs[method](path); + } catch (err) { + if (err.code === 'ENOENT') { + return; + } + + throw err; + } +} + /** * @returns {string[]} */ @@ -251,13 +283,14 @@ export default function (program) { .option( '--run-examples', 'Adds plugin paths for all the Kibana example plugins and runs with no base path' + ) + .option( + '--serverless [oblt|security|es]', + 'Start Kibana in a specific serverless project mode. ' + + 'If no mode is provided, it starts Kibana in the most recent serverless project mode (default is es)' ); } - if (isServerlessCapableDistribution()) { - command.option('--serverless ', 'Start Kibana in a serverless project mode'); - } - if (DEV_MODE_SUPPORTED) { command .option('--dev', 'Run the server with development mode defaults') @@ -282,10 +315,8 @@ export default function (program) { const configs = [getConfigPath(), ...getEnvConfigs(), ...(opts.config || [])]; const serverlessMode = getServerlessProjectMode(opts); - // we "unshift" .serverless. config so that it only overrides defaults if (serverlessMode) { - maybeAddConfig(`serverless.yml`, configs, 'push'); - maybeAddConfig(`serverless.${serverlessMode}.yml`, configs, 'unshift'); + maybeSetRecentConfig('serverless.recent.yml', serverlessMode, opts.dev, configs, 'push'); } // .dev. configs are "pushed" so that they override all other config files @@ -293,7 +324,7 @@ export default function (program) { maybeAddConfig('kibana.dev.yml', configs, 'push'); if (serverlessMode) { maybeAddConfig(`serverless.dev.yml`, configs, 'push'); - maybeAddConfig(`serverless.${serverlessMode}.dev.yml`, configs, 'push'); + maybeAddConfig('serverless.recent.dev.yml', configs, 'push'); } } @@ -315,7 +346,6 @@ export default function (program) { oss: !!opts.oss, cache: !!opts.cache, dist: !!opts.dist, - serverless: !!opts.serverless, }; // In development mode, the main process uses the @kbn/dev-cli-mode