From fe53ea0c37f24c40bce0ef529ea32f052386189f Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 1 Mar 2021 17:29:39 -0700 Subject: [PATCH] [kbn/optimizer] allow customizing the limits path from the script --- packages/kbn-optimizer/src/cli.ts | 318 ++++++++++-------- packages/kbn-optimizer/src/index.ts | 1 + packages/kbn-optimizer/src/limits.ts | 27 +- .../src/optimizer/optimizer_config.ts | 5 +- scripts/build_kibana_platform_plugins.js | 4 +- .../tasks/build_kibana_platform_plugins.ts | 1 + 6 files changed, 199 insertions(+), 157 deletions(-) diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 8fb906aa4603e..8f82f34646e60 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -12,164 +12,190 @@ import Path from 'path'; import { REPO_ROOT } from '@kbn/utils'; import { lastValueFrom } from '@kbn/std'; -import { run, createFlagError } from '@kbn/dev-utils'; +import { run, createFlagError, Flags } from '@kbn/dev-utils'; import { logOptimizerState } from './log_optimizer_state'; import { OptimizerConfig } from './optimizer'; import { runOptimizer } from './run_optimizer'; import { validateLimitsForAllBundles, updateBundleLimits } from './limits'; -run( - async ({ log, flags }) => { - const watch = flags.watch ?? false; - if (typeof watch !== 'boolean') { - throw createFlagError('expected --watch to have no value'); +function getLimitsPath(flags: Flags, defaultPath: string) { + if (flags.limits) { + if (typeof flags.limits !== 'string') { + throw createFlagError('expected --limits to be a string'); } - const oss = flags.oss ?? false; - if (typeof oss !== 'boolean') { - throw createFlagError('expected --oss to have no value'); - } - - const cache = flags.cache ?? true; - if (typeof cache !== 'boolean') { - throw createFlagError('expected --cache to have no value'); - } - - const includeCoreBundle = flags.core ?? true; - if (typeof includeCoreBundle !== 'boolean') { - throw createFlagError('expected --core to have no value'); - } - - const dist = flags.dist ?? false; - if (typeof dist !== 'boolean') { - throw createFlagError('expected --dist to have no value'); - } - - const examples = flags.examples ?? false; - if (typeof examples !== 'boolean') { - throw createFlagError('expected --no-examples to have no value'); - } - - const profileWebpack = flags.profile ?? false; - if (typeof profileWebpack !== 'boolean') { - throw createFlagError('expected --profile to have no value'); - } - - const inspectWorkers = flags['inspect-workers'] ?? false; - if (typeof inspectWorkers !== 'boolean') { - throw createFlagError('expected --no-inspect-workers to have no value'); - } - - const maxWorkerCount = flags.workers ? Number.parseInt(String(flags.workers), 10) : undefined; - if (maxWorkerCount !== undefined && (!Number.isFinite(maxWorkerCount) || maxWorkerCount < 1)) { - throw createFlagError('expected --workers to be a number greater than 0'); - } - - const extraPluginScanDirs = ([] as string[]) - .concat((flags['scan-dir'] as string | string[]) || []) - .map((p) => Path.resolve(p)); - if (!extraPluginScanDirs.every((s) => typeof s === 'string')) { - throw createFlagError('expected --scan-dir to be a string'); - } - - const reportStats = flags['report-stats'] ?? false; - if (typeof reportStats !== 'boolean') { - throw createFlagError('expected --report-stats to have no value'); - } - - const filter = typeof flags.filter === 'string' ? [flags.filter] : flags.filter; - if (!Array.isArray(filter) || !filter.every((f) => typeof f === 'string')) { - throw createFlagError('expected --filter to be one or more strings'); - } - - const focus = typeof flags.focus === 'string' ? [flags.focus] : flags.focus; - if (!Array.isArray(focus) || !focus.every((f) => typeof f === 'string')) { - throw createFlagError('expected --focus to be one or more strings'); - } + return Path.resolve(flags.limits); + } - const validateLimits = flags['validate-limits'] ?? false; - if (typeof validateLimits !== 'boolean') { - throw createFlagError('expected --validate-limits to have no value'); - } + if (process.env.KBN_OPTIMIZER_LIMITS_PATH) { + return Path.resolve(process.env.KBN_OPTIMIZER_LIMITS_PATH); + } - const updateLimits = flags['update-limits'] ?? false; - if (typeof updateLimits !== 'boolean') { - throw createFlagError('expected --update-limits to have no value'); - } + return defaultPath; +} + +export function runKbnOptimizerCli(options: { defaultLimitsPath: string }) { + run( + async ({ log, flags }) => { + const watch = flags.watch ?? false; + if (typeof watch !== 'boolean') { + throw createFlagError('expected --watch to have no value'); + } + + const oss = flags.oss ?? false; + if (typeof oss !== 'boolean') { + throw createFlagError('expected --oss to have no value'); + } + + const cache = flags.cache ?? true; + if (typeof cache !== 'boolean') { + throw createFlagError('expected --cache to have no value'); + } + + const includeCoreBundle = flags.core ?? true; + if (typeof includeCoreBundle !== 'boolean') { + throw createFlagError('expected --core to have no value'); + } + + const dist = flags.dist ?? false; + if (typeof dist !== 'boolean') { + throw createFlagError('expected --dist to have no value'); + } + + const examples = flags.examples ?? false; + if (typeof examples !== 'boolean') { + throw createFlagError('expected --no-examples to have no value'); + } + + const profileWebpack = flags.profile ?? false; + if (typeof profileWebpack !== 'boolean') { + throw createFlagError('expected --profile to have no value'); + } + + const inspectWorkers = flags['inspect-workers'] ?? false; + if (typeof inspectWorkers !== 'boolean') { + throw createFlagError('expected --no-inspect-workers to have no value'); + } + + const maxWorkerCount = flags.workers ? Number.parseInt(String(flags.workers), 10) : undefined; + if ( + maxWorkerCount !== undefined && + (!Number.isFinite(maxWorkerCount) || maxWorkerCount < 1) + ) { + throw createFlagError('expected --workers to be a number greater than 0'); + } + + const extraPluginScanDirs = ([] as string[]) + .concat((flags['scan-dir'] as string | string[]) || []) + .map((p) => Path.resolve(p)); + if (!extraPluginScanDirs.every((s) => typeof s === 'string')) { + throw createFlagError('expected --scan-dir to be a string'); + } + + const reportStats = flags['report-stats'] ?? false; + if (typeof reportStats !== 'boolean') { + throw createFlagError('expected --report-stats to have no value'); + } + + const filter = typeof flags.filter === 'string' ? [flags.filter] : flags.filter; + if (!Array.isArray(filter) || !filter.every((f) => typeof f === 'string')) { + throw createFlagError('expected --filter to be one or more strings'); + } + + const focus = typeof flags.focus === 'string' ? [flags.focus] : flags.focus; + if (!Array.isArray(focus) || !focus.every((f) => typeof f === 'string')) { + throw createFlagError('expected --focus to be one or more strings'); + } + + const limitsPath = getLimitsPath(flags, options.defaultLimitsPath); + + const validateLimits = flags['validate-limits'] ?? false; + if (typeof validateLimits !== 'boolean') { + throw createFlagError('expected --validate-limits to have no value'); + } + + const updateLimits = flags['update-limits'] ?? false; + if (typeof updateLimits !== 'boolean') { + throw createFlagError('expected --update-limits to have no value'); + } + + const config = OptimizerConfig.create({ + repoRoot: REPO_ROOT, + watch, + maxWorkerCount, + oss: oss && !(validateLimits || updateLimits), + dist: dist || updateLimits, + cache, + examples: examples && !(validateLimits || updateLimits), + profileWebpack, + extraPluginScanDirs, + inspectWorkers, + includeCoreBundle, + filter, + focus, + limitsPath, + }); - const config = OptimizerConfig.create({ - repoRoot: REPO_ROOT, - watch, - maxWorkerCount, - oss: oss && !(validateLimits || updateLimits), - dist: dist || updateLimits, - cache, - examples: examples && !(validateLimits || updateLimits), - profileWebpack, - extraPluginScanDirs, - inspectWorkers, - includeCoreBundle, - filter, - focus, - }); - - if (validateLimits) { - validateLimitsForAllBundles(log, config); - return; - } + if (validateLimits) { + validateLimitsForAllBundles(log, config, limitsPath); + return; + } - const update$ = runOptimizer(config); + const update$ = runOptimizer(config); - await lastValueFrom(update$.pipe(logOptimizerState(log, config))); + await lastValueFrom(update$.pipe(logOptimizerState(log, config))); - if (updateLimits) { - updateBundleLimits({ - log, - config, - dropMissing: !(focus || filter), - }); - } - }, - { - flags: { - boolean: [ - 'core', - 'watch', - 'oss', - 'examples', - 'dist', - 'cache', - 'profile', - 'inspect-workers', - 'validate-limits', - 'update-limits', - ], - string: ['workers', 'scan-dir', 'filter'], - default: { - core: true, - examples: true, - cache: true, - 'inspect-workers': true, - filter: [], - focus: [], - }, - help: ` - --watch run the optimizer in watch mode - --workers max number of workers to use - --oss only build oss plugins - --profile profile the webpack builds and write stats.json files to build outputs - --no-core disable generating the core bundle - --no-cache disable the cache - --focus just like --filter, except dependencies are automatically included, --filter applies to result - --filter comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported - --no-examples don't build the example plugins - --dist create bundles that are suitable for inclusion in the Kibana distributable, enabled when running with --update-limits - --scan-dir add a directory to the list of directories scanned for plugins (specify as many times as necessary) - --no-inspect-workers when inspecting the parent process, don't inspect the workers - --validate-limits validate the limits.yml config to ensure that there are limits defined for every bundle - --update-limits run a build and rewrite the limits file to include the current bundle sizes +5kb - `, + if (updateLimits) { + updateBundleLimits({ + log, + config, + dropMissing: !(focus || filter), + limitsPath, + }); + } }, - } -); + { + flags: { + boolean: [ + 'core', + 'watch', + 'oss', + 'examples', + 'dist', + 'cache', + 'profile', + 'inspect-workers', + 'validate-limits', + 'update-limits', + ], + string: ['workers', 'scan-dir', 'filter', 'limits'], + default: { + core: true, + examples: true, + cache: true, + 'inspect-workers': true, + filter: [], + focus: [], + }, + help: ` + --watch run the optimizer in watch mode + --workers max number of workers to use + --oss only build oss plugins + --profile profile the webpack builds and write stats.json files to build outputs + --no-core disable generating the core bundle + --no-cache disable the cache + --focus just like --filter, except dependencies are automatically included, --filter applies to result + --filter comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported + --no-examples don't build the example plugins + --dist create bundles that are suitable for inclusion in the Kibana distributable, enabled when running with --update-limits + --scan-dir add a directory to the list of directories scanned for plugins (specify as many times as necessary) + --no-inspect-workers when inspecting the parent process, don't inspect the workers + --limits path to a limits.yml file to read, defaults to $KBN_OPTIMIZER_LIMITS_PATH or source file + --validate-limits validate the limits.yml config to ensure that there are limits defined for every bundle + --update-limits run a build and rewrite the limits file to include the current bundle sizes +5kb + `, + }, + } + ); +} diff --git a/packages/kbn-optimizer/src/index.ts b/packages/kbn-optimizer/src/index.ts index 551d2ffacfcfb..8d6e89008bc68 100644 --- a/packages/kbn-optimizer/src/index.ts +++ b/packages/kbn-optimizer/src/index.ts @@ -11,3 +11,4 @@ export * from './run_optimizer'; export * from './log_optimizer_state'; export * from './node'; export * from './limits'; +export * from './cli'; diff --git a/packages/kbn-optimizer/src/limits.ts b/packages/kbn-optimizer/src/limits.ts index 292314a4608e4..077fe38ddc7f6 100644 --- a/packages/kbn-optimizer/src/limits.ts +++ b/packages/kbn-optimizer/src/limits.ts @@ -15,15 +15,14 @@ import { createFailError, ToolingLog, CiStatsMetrics } from '@kbn/dev-utils'; import { OptimizerConfig, Limits } from './optimizer'; -const LIMITS_PATH = require.resolve('../limits.yml'); const DEFAULT_BUDGET = 15000; const diff = (a: T[], b: T[]): T[] => a.filter((item) => !b.includes(item)); -export function readLimits(): Limits { +export function readLimits(path: string): Limits { let yaml; try { - yaml = Fs.readFileSync(LIMITS_PATH, 'utf8'); + yaml = Fs.readFileSync(path, 'utf8'); } catch (error) { if (error.code !== 'ENOENT') { throw error; @@ -33,8 +32,12 @@ export function readLimits(): Limits { return yaml ? Yaml.safeLoad(yaml) : {}; } -export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerConfig) { - const limitBundleIds = Object.keys(readLimits().pageLoadAssetSize || {}); +export function validateLimitsForAllBundles( + log: ToolingLog, + config: OptimizerConfig, + limitsPath: string +) { + const limitBundleIds = Object.keys(readLimits(limitsPath).pageLoadAssetSize || {}); const configBundleIds = config.bundles.map((b) => b.id); const missingBundleIds = diff(configBundleIds, limitBundleIds); @@ -73,10 +76,16 @@ interface UpdateBundleLimitsOptions { log: ToolingLog; config: OptimizerConfig; dropMissing: boolean; + limitsPath: string; } -export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLimitsOptions) { - const limits = readLimits(); +export function updateBundleLimits({ + log, + config, + dropMissing, + limitsPath, +}: UpdateBundleLimitsOptions) { + const limits = readLimits(limitsPath); const metrics: CiStatsMetrics = config.bundles .map((bundle) => JSON.parse(Fs.readFileSync(Path.resolve(bundle.outputDir, 'metrics.json'), 'utf-8')) @@ -102,6 +111,6 @@ export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLim pageLoadAssetSize, }; - Fs.writeFileSync(LIMITS_PATH, Yaml.safeDump(newLimits)); - log.success(`wrote updated limits to ${LIMITS_PATH}`); + Fs.writeFileSync(limitsPath, Yaml.safeDump(newLimits)); + log.success(`wrote updated limits to ${limitsPath}`); } diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index ed521d32a0a29..9110b6db27e92 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -115,6 +115,9 @@ interface Options { * - "k7light" */ themes?: ThemeTag | '*' | ThemeTag[]; + + /** path to a limits.yml file that should be used to inform ci-stats of metric limits */ + limitsPath?: string; } export interface ParsedOptions { @@ -211,7 +214,7 @@ export class OptimizerConfig { } static create(inputOptions: Options) { - const limits = readLimits(); + const limits = inputOptions.limitsPath ? readLimits(inputOptions.limitsPath) : {}; const options = OptimizerConfig.parseOptions(inputOptions); const plugins = findKibanaPlatformPlugins(options.pluginScanDirs, options.pluginPaths); const bundles = [ diff --git a/scripts/build_kibana_platform_plugins.js b/scripts/build_kibana_platform_plugins.js index b562183dd19f5..809e21bc652d0 100644 --- a/scripts/build_kibana_platform_plugins.js +++ b/scripts/build_kibana_platform_plugins.js @@ -6,4 +6,6 @@ * Side Public License, v 1. */ -require('@kbn/optimizer/target/cli'); +require('@kbn/optimizer').runKbnOptimizerCli({ + defaultLimitsPath: require.resolve('../packages/kbn-optimizer/limits.yml'), +}); diff --git a/src/dev/build/tasks/build_kibana_platform_plugins.ts b/src/dev/build/tasks/build_kibana_platform_plugins.ts index d2d2d3275270b..dfac3edf9847b 100644 --- a/src/dev/build/tasks/build_kibana_platform_plugins.ts +++ b/src/dev/build/tasks/build_kibana_platform_plugins.ts @@ -27,6 +27,7 @@ export const BuildKibanaPlatformPlugins: Task = { watch: false, dist: true, includeCoreBundle: true, + limitsPath: Path.resolve(REPO_ROOT, 'packages/kbn-optimizer/limits.yml'), }); await lastValueFrom(runOptimizer(config).pipe(logOptimizerState(log, config)));