[OAS] Capture and commit serverless bundle

packages/core/test-helpers/core-test-helpers-kbn-server/src/create_serverless_root.ts

@@ -38,13 +38,18 @@ const projectType: ServerlessProjectType = 'es';
  */
 export function createTestServerlessInstances({
   adjustTimeout,
+  kibana = {},
 }: {
-  adjustTimeout: (timeout: number) => void;
-}): TestServerlessUtils {
+  kibana?: {
+    settings?: {};
+    cliArgs?: Partial<CliArgs>;
+  };
+  adjustTimeout?: (timeout: number) => void;
+} = {}): TestServerlessUtils {
   adjustTimeout?.(150_000);
 
   const esUtils = createServerlessES();
-  const kbUtils = createServerlessKibana();
+  const kbUtils = createServerlessKibana(kibana.settings, kibana.cliArgs);
 
   return {
     startES: async () => {

packages/kbn-capture-oas-snapshot-cli/src/capture_oas_snapshot.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import fs from 'node:fs/promises';
+import { encode } from 'node:querystring';
+import type { ChildProcess } from 'node:child_process';
+import fetch from 'node-fetch';
+import * as Rx from 'rxjs';
+import { startTSWorker } from '@kbn/dev-utils';
+import { createTestEsCluster } from '@kbn/test';
+import type { ToolingLog } from '@kbn/tooling-log';
+import { createTestServerlessInstances } from '@kbn/core-test-helpers-kbn-server';
+import type { Result } from './kibana_worker';
+import { sortAndPrettyPrint } from './run_capture_oas_snapshot_cli';
+import { buildFlavourEnvArgName } from './common';
+
+interface CaptureOasSnapshotArgs {
+  log: ToolingLog;
+  buildFlavour: 'serverless' | 'traditional';
+  outputFile: string;
+  update: boolean;
+  filters?: {
+    pathStartsWith?: string[];
+    excludePathsMatching?: string[];
+  };
+}
+
+const MB = 1024 * 1024;
+const twoDeci = (num: number) => Math.round(num * 100) / 100;
+
+export async function captureOasSnapshot({
+  log,
+  filters = {},
+  buildFlavour,
+  update,
+  outputFile,
+}: CaptureOasSnapshotArgs): Promise<void> {
+  const { excludePathsMatching = [], pathStartsWith } = filters;
+  // internal consts
+  const port = 5622;
+  // We are only including /api/status for now
+  excludePathsMatching.push(
+    '/{path*}',
+    // Our internal asset paths
+    '/XXXXXXXXXXXX/'
+  );
+
+  let esCluster: undefined | { stop(): Promise<void> };
+  let kbWorker: undefined | ChildProcess;
+
+  try {
+    log.info('Starting es...');
+    esCluster = await log.indent(4, async () => {
+      if (buildFlavour === 'serverless') {
+        const { startES } = createTestServerlessInstances();
+        return await startES();
+      }
+      const cluster = createTestEsCluster({ log });
+      await cluster.start();
+      return { stop: () => cluster.cleanup() };
+    });
+
+    log.info('Starting Kibana...');
+    kbWorker = await log.indent(4, async () => {
+      log.info('Loading core with all plugins enabled so that we can capture OAS for all...');
+      const { msg$, proc } = startTSWorker<Result>({
+        log,
+        src: require.resolve('./kibana_worker'),
+        env: { ...process.env, [buildFlavourEnvArgName]: buildFlavour },
+      });
+      await Rx.firstValueFrom(
+        msg$.pipe(
+          Rx.map((msg) => {
+            if (msg !== 'ready')
+              throw new Error(`received unexpected message from worker (expected "ready"): ${msg}`);
+          })
+        )
+      );
+      return proc;
+    });
+
+    const qs = encode({
+      access: 'public',
+      version: '2023-10-31', // hard coded for now, we can make this configurable later
+      pathStartsWith,
+      excludePathsMatching,
+    });
+    const url = `http://localhost:${port}/api/oas?${qs}`;
+    log.info(`Fetching OAS at ${url}...`);
+    const result = await fetch(url, {
+      headers: {
+        'kbn-xsrf': 'kbn-oas-snapshot',
+        authorization: `Basic ${Buffer.from('elastic:changeme').toString('base64')}`,
+      },
+    });
+    if (result.status !== 200) {
+      log.error(`Failed to fetch OAS: ${JSON.stringify(result, null, 2)}`);
+      throw new Error(`Failed to fetch OAS: ${result.status}`);
+    }
+    const currentOas = await result.json();
+    log.info(`Recieved OAS, writing to ${outputFile}...`);
+    if (update) {
+      await fs.writeFile(outputFile, sortAndPrettyPrint(currentOas));
+      const { size: sizeBytes } = await fs.stat(outputFile);
+      log.success(`OAS written to ${outputFile}. File size ~${twoDeci(sizeBytes / MB)} MB.`);
+    } else {
+      log.success(
+        `OAS recieved, not writing to file. Got OAS for ${
+          Object.keys(currentOas.paths).length
+        } paths.`
+      );
+    }
+  } catch (err) {
+    log.error(`Failed to capture OAS: ${JSON.stringify(err, null, 2)}`);
+    throw err;
+  } finally {
+    kbWorker?.kill('SIGILL');
+    await esCluster?.stop();
+  }
+}

packages/kbn-capture-oas-snapshot-cli/src/common.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const buildFlavourEnvArgName = 'CAPTURE_OAS_SNAPSHOT_WORKER_BUILD_FLAVOR';

packages/kbn-capture-oas-snapshot-cli/src/kibana_worker.ts

@@ -6,16 +6,24 @@
  * Side Public License, v 1.
  */
 
-import { createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
+import {
+  createRootWithCorePlugins,
+  createTestServerlessInstances,
+} from '@kbn/core-test-helpers-kbn-server';
 import { set } from '@kbn/safer-lodash-set';
 import { PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH } from '@kbn/core-plugins-server-internal/src/constants';
+import { buildFlavourEnvArgName } from './common';
 
 export type Result = 'ready';
 
 (async () => {
   if (!process.send) {
     throw new Error('worker must be run in a node.js fork');
   }
+  const buildFlavour = process.env[buildFlavourEnvArgName];
+  if (!buildFlavour) throw new Error(`env arg ${buildFlavourEnvArgName} must be provided`);
+
+  const serverless = buildFlavour === 'serverless';
 
   const settings = {
     logging: {
@@ -30,7 +38,8 @@ export type Result = 'ready';
   };
   set(settings, PLUGIN_SYSTEM_ENABLE_ALL_PLUGINS_CONFIG_PATH, true);
 
-  const root = createRootWithCorePlugins(settings, {
+  const cliArgs = {
+    serverless,
     basePath: false,
     cache: false,
     dev: true,
@@ -40,11 +49,21 @@ export type Result = 'ready';
     oss: false,
     runExamples: false,
     watch: false,
-  });
+  };
 
-  await root.preboot();
-  await root.setup();
-  await root.start();
+  if (serverless) {
+    // Satisfy spaces config for serverless:
+    set(settings, 'xpack.spaces.allowFeatureVisibility', false);
+    const { startKibana } = createTestServerlessInstances({
+      kibana: { settings, cliArgs },
+    });
+    const {} = await startKibana();
+  } else {
+    const root = createRootWithCorePlugins(settings, cliArgs);
+    await root.preboot();
+    await root.setup();
+    await root.start();
+  }
 
   const result: Result = 'ready';
 

packages/kbn-capture-oas-snapshot-cli/src/run_capture_oas_snapshot_cli.ts

@@ -7,18 +7,10 @@
  */
 
 import path from 'node:path';
-import fs from 'node:fs/promises';
-import { encode } from 'node:querystring';
-import fetch from 'node-fetch';
 import { run } from '@kbn/dev-cli-runner';
-import { startTSWorker } from '@kbn/dev-utils';
-import { createTestEsCluster } from '@kbn/test';
-import * as Rx from 'rxjs';
 import { REPO_ROOT } from '@kbn/repo-info';
 import chalk from 'chalk';
-import type { Result } from './kibana_worker';
-
-const OAS_FILE_PATH = path.resolve(REPO_ROOT, './oas_docs/bundle.json');
+import { captureOasSnapshot } from './capture_oas_snapshot';
 
 export const sortAndPrettyPrint = (object: object) => {
   const keys = new Set<string>();
@@ -29,100 +21,65 @@ export const sortAndPrettyPrint = (object: object) => {
   return JSON.stringify(object, Array.from(keys).sort(), 2);
 };
 
-const MB = 1024 * 1024;
-const twoDeci = (num: number) => Math.round(num * 100) / 100;
+const OAS_OUTPUT_DIR = path.resolve(REPO_ROOT, './oas_docs');
 
 run(
-  async ({ log, flagsReader, addCleanupTask }) => {
+  async ({ log, flagsReader }) => {
+    const serverless = flagsReader.boolean('serverless');
+    const traditional = flagsReader.boolean('traditional');
+    if (!serverless && !traditional) {
+      log.error(
+        'Not capturing any OAS, remove one or both of `--no-serverless` or `--no-traditional` flags to run this CLI'
+      );
+      process.exit(1);
+    }
+
     const update = flagsReader.boolean('update');
     const pathStartsWith = flagsReader.arrayOfStrings('include-path');
     const excludePathsMatching = flagsReader.arrayOfStrings('exclude-path') ?? [];
 
-    // internal consts
-    const port = 5622;
-    // We are only including /api/status for now
-    excludePathsMatching.push(
-      '/{path*}',
-      // Our internal asset paths
-      '/XXXXXXXXXXXX/'
-    );
-
-    log.info('Starting es...');
-    await log.indent(4, async () => {
-      const cluster = createTestEsCluster({ log });
-      await cluster.start();
-      addCleanupTask(() => cluster.cleanup());
-    });
-
-    log.info('Starting Kibana...');
-    await log.indent(4, async () => {
-      log.info('Loading core with all plugins enabled so that we can capture OAS for all...');
-      const { msg$, proc } = startTSWorker<Result>({
+    if (traditional) {
+      log.info('Capturing OAS for traditional Kibana...');
+      await captureOasSnapshot({
         log,
-        src: require.resolve('./kibana_worker'),
+        buildFlavour: 'traditional',
+        outputFile: path.resolve(OAS_OUTPUT_DIR, 'bundle.json'),
+        filters: { pathStartsWith, excludePathsMatching },
+        update,
       });
-      await Rx.firstValueFrom(
-        msg$.pipe(
-          Rx.map((msg) => {
-            if (msg !== 'ready')
-              throw new Error(`received unexpected message from worker (expected "ready"): ${msg}`);
-          })
-        )
-      );
-      addCleanupTask(() => proc.kill('SIGILL'));
-    });
+      log.success('Captured OAS for traditional Kibana.');
+    }
 
-    try {
-      const qs = encode({
-        access: 'public',
-        version: '2023-10-31', // hard coded for now, we can make this configurable later
-        pathStartsWith,
-        excludePathsMatching,
-      });
-      const url = `http://localhost:${port}/api/oas?${qs}`;
-      log.info(`Fetching OAS at ${url}...`);
-      const result = await fetch(url, {
-        headers: {
-          'kbn-xsrf': 'kbn-oas-snapshot',
-          authorization: `Basic ${Buffer.from('elastic:changeme').toString('base64')}`,
-        },
+    if (serverless) {
+      log.info('Capturing OAS for serverless Kibana...');
+      await captureOasSnapshot({
+        log,
+        buildFlavour: 'serverless',
+        outputFile: path.resolve(OAS_OUTPUT_DIR, 'bundle.serverless.json'),
+        filters: { pathStartsWith, excludePathsMatching },
+        update,
       });
-      if (result.status !== 200) {
-        log.error(`Failed to fetch OAS: ${JSON.stringify(result, null, 2)}`);
-        throw new Error(`Failed to fetch OAS: ${result.status}`);
-      }
-      const currentOas = await result.json();
-      log.info(`Recieved OAS, writing to ${OAS_FILE_PATH}...`);
-      if (update) {
-        await fs.writeFile(OAS_FILE_PATH, sortAndPrettyPrint(currentOas));
-        const { size: sizeBytes } = await fs.stat(OAS_FILE_PATH);
-        log.success(`OAS written to ${OAS_FILE_PATH}. File size ~${twoDeci(sizeBytes / MB)} MB.`);
-      } else {
-        log.success(
-          `OAS recieved, not writing to file. Got OAS for ${
-            Object.keys(currentOas.paths).length
-          } paths.`
-        );
-      }
-    } catch (err) {
-      log.error(`Failed to capture OAS: ${JSON.stringify(err, null, 2)}`);
-      throw err;
+      log.success('Captured OAS for serverless Kibana.');
     }
   },
   {
     description: `
       Get the current OAS from Kibana's /api/oas API
     `,
     flags: {
-      boolean: ['update'],
-      string: ['include-path', 'exclude-path'],
+      boolean: ['update', 'serverless', 'traditional'],
+      string: ['include-path', 'exclude-path', 'build-flavor'],
       default: {
         fix: false,
+        serverless: true,
+        traditional: true,
       },
       help: `
         --include-path            Path to include. Path must start with provided value. Can be passed multiple times.
         --exclude-path            Path to exclude. Path must NOT start with provided value. Can be passed multiple times.
-        --update                  Write the current OAS to ${chalk.cyan(OAS_FILE_PATH)}.
+        --update                  Write the current OAS bundles to ${chalk.cyan(OAS_OUTPUT_DIR)}.
+        --no-serverless           Whether to skip OAS for serverless Kibana. Defaults to false.
+        --no-traditional          Whether to skip OAS for traditional Kibana. Defaults to false.
       `,
     },
   }

packages/kbn-capture-oas-snapshot-cli/tsconfig.json

@@ -21,5 +21,6 @@
     "@kbn/dev-cli-runner",
     "@kbn/test",
     "@kbn/dev-utils",
+    "@kbn/tooling-log",
   ]
 }