diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 469ff0be66ae5..f256d3467caa5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -887,6 +887,7 @@ packages/home/sample_data_tab @elastic/kibana-global-experience packages/home/sample_data_types @elastic/kibana-global-experience packages/kbn-ace @elastic/platform-deployment-management packages/kbn-alerts @elastic/security-solution +packages/kbn-ambient-common-types @elastic/kibana-operations packages/kbn-ambient-storybook-types @elastic/kibana-operations packages/kbn-ambient-ui-types @elastic/kibana-operations packages/kbn-analytics @elastic/kibana-core @@ -958,6 +959,8 @@ packages/kbn-monaco @elastic/kibana-app-services packages/kbn-optimizer @elastic/kibana-operations packages/kbn-optimizer-webpack-helpers @elastic/kibana-operations packages/kbn-osquery-io-ts-types @elastic/security-asset-management +packages/kbn-peggy @elastic/kibana-operations +packages/kbn-peggy-loader @elastic/kibana-operations packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing packages/kbn-plugin-discovery @elastic/kibana-operations packages/kbn-plugin-generator @elastic/kibana-operations diff --git a/dev_docs/operations/operations_landing.mdx b/dev_docs/operations/operations_landing.mdx index 9004141255f58..b38570ea44d13 100644 --- a/dev_docs/operations/operations_landing.mdx +++ b/dev_docs/operations/operations_landing.mdx @@ -64,6 +64,7 @@ layout: landing { pageId: "kibDevDocsOpsExpect" }, { pageId: "kibDevDocsOpsAmbientStorybookTypes" }, { pageId: "kibDevDocsOpsAmbientUiTypes" }, + { pageId: "kibDevDocsOpsAmbientCommonTypes" }, { pageId: "kibDevDocsOpsTestSubjSelector" }, { pageId: "kibDevDocsOpsBazelRunner" }, { pageId: "kibDevDocsOpsCliDevMode" }, @@ -77,5 +78,7 @@ layout: landing { pageId: "kibDevDocsOpsManagedVscodeConfigCli" }, { pageId: "kibDevDocsOpsTest" }, { pageId: "kibDevDocsOpsEsArchiver" }, + { pageId: "kibDevDocsOpsPeggy" }, + { pageId: "kibDevDocsOpsPeggyLoader" }, ]} /> \ No newline at end of file diff --git a/package.json b/package.json index 67a78a92d9582..4e4123e8cdc98 100644 --- a/package.json +++ b/package.json @@ -718,7 +718,9 @@ "@istanbuljs/schema": "^0.1.2", "@jest/console": "^29.3.1", "@jest/reporters": "^29.3.1", + "@jest/transform": "^29.3.1", "@jest/types": "^29.3.1", + "@kbn/ambient-common-types": "link:bazel-bin/packages/kbn-ambient-common-types", "@kbn/ambient-storybook-types": "link:bazel-bin/packages/kbn-ambient-storybook-types", "@kbn/ambient-ui-types": "link:bazel-bin/packages/kbn-ambient-ui-types", "@kbn/apm-synthtrace": "link:bazel-bin/packages/kbn-apm-synthtrace", @@ -757,6 +759,8 @@ "@kbn/managed-vscode-config-cli": "link:bazel-bin/packages/kbn-managed-vscode-config-cli", "@kbn/optimizer": "link:bazel-bin/packages/kbn-optimizer", "@kbn/optimizer-webpack-helpers": "link:bazel-bin/packages/kbn-optimizer-webpack-helpers", + "@kbn/peggy": "link:bazel-bin/packages/kbn-peggy", + "@kbn/peggy-loader": "link:bazel-bin/packages/kbn-peggy-loader", "@kbn/performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor", "@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator", "@kbn/plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers", @@ -1060,7 +1064,7 @@ "jsondiffpatch": "0.4.1", "license-checker": "^25.0.1", "listr": "^0.14.1", - "lmdb-store": "^1.6.11", + "lmdb-store": "^1", "loader-utils": "^2.0.4", "marge": "^1.0.1", "micromatch": "^4.0.5", diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index f8c862ee2f690..48c0fbca968ed 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -194,6 +194,7 @@ filegroup( "//packages/home/sample_data_types:build", "//packages/kbn-ace:build", "//packages/kbn-alerts:build", + "//packages/kbn-ambient-common-types:build", "//packages/kbn-ambient-storybook-types:build", "//packages/kbn-ambient-ui-types:build", "//packages/kbn-analytics:build", @@ -265,6 +266,8 @@ filegroup( "//packages/kbn-optimizer:build", "//packages/kbn-optimizer-webpack-helpers:build", "//packages/kbn-osquery-io-ts-types:build", + "//packages/kbn-peggy:build", + "//packages/kbn-peggy-loader:build", "//packages/kbn-performance-testing-dataset-extractor:build", "//packages/kbn-plugin-discovery:build", "//packages/kbn-plugin-generator:build", @@ -617,6 +620,8 @@ filegroup( "//packages/kbn-optimizer:build_types", "//packages/kbn-optimizer-webpack-helpers:build_types", "//packages/kbn-osquery-io-ts-types:build_types", + "//packages/kbn-peggy:build_types", + "//packages/kbn-peggy-loader:build_types", "//packages/kbn-performance-testing-dataset-extractor:build_types", "//packages/kbn-plugin-discovery:build_types", "//packages/kbn-plugin-generator:build_types", diff --git a/packages/kbn-ambient-common-types/BUILD.bazel b/packages/kbn-ambient-common-types/BUILD.bazel new file mode 100644 index 0000000000000..3a8b17248da22 --- /dev/null +++ b/packages/kbn-ambient-common-types/BUILD.bazel @@ -0,0 +1,58 @@ +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "pkg_npm") + +PKG_DIRNAME = "kbn-ambient-common-types" +PKG_REQUIRE_NAME = "@kbn/ambient-common-types" + +SRCS = glob( + [ + "*.d.ts", + ] +) + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +js_library( + name = PKG_DIRNAME, + srcs = SRCS + NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS, + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +alias( + name = "npm_module_types", + actual = ":" + PKG_DIRNAME, + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-ambient-common-types/README.mdx b/packages/kbn-ambient-common-types/README.mdx new file mode 100644 index 0000000000000..f08885537225c --- /dev/null +++ b/packages/kbn-ambient-common-types/README.mdx @@ -0,0 +1,20 @@ +--- +id: kibDevDocsOpsAmbientCommonTypes +slug: /kibana-dev-docs/ops/ambient-common-types +title: "@kbn/ambient-common-types" +description: A package holding ambient type definitions for files that are expected to run on the server and the browser +date: 2022-05-18 +tags: ['kibana', 'dev', 'contributor', 'operations', 'ambient', 'ui', 'common', 'server', 'types'] +--- + +This package holds ambient typescript definitions which should be included in projects which are expected to run on the server and the browser. + +## Plugins +These types will automatically be included for plugins. + +## Packages + +To include these types in a package: + +- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file. +- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file. \ No newline at end of file diff --git a/packages/kbn-ambient-common-types/index.d.ts b/packages/kbn-ambient-common-types/index.d.ts new file mode 100644 index 0000000000000..4d139c64b8cca --- /dev/null +++ b/packages/kbn-ambient-common-types/index.d.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * These ambient types are used to define default types for anything which is + * supported in both server/browser environments. + */ + +/** + * peggy grammars are built automatically on import in both browser/server + */ +declare module '*.peggy' { + export interface ParserOptions { + [key: string]: any; + /** + * Object that will be attached to the each `LocationRange` object created by + * the parser. For example, this can be path to the parsed file or even the + * File object. + */ + grammarSource?: any; + startRule?: string; + tracer?: ParserTracer; + } + + /** + * parse `input` using the peggy grammer + * @param input code to parse + * @param options parse options + */ + export function parse(input: string, options?: ParserOptions): any; +} diff --git a/packages/kbn-ambient-common-types/jest.config.js b/packages/kbn-ambient-common-types/jest.config.js new file mode 100644 index 0000000000000..f94290b64cd90 --- /dev/null +++ b/packages/kbn-ambient-common-types/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-ambient-common-types'], +}; diff --git a/packages/kbn-ambient-common-types/kibana.jsonc b/packages/kbn-ambient-common-types/kibana.jsonc new file mode 100644 index 0000000000000..70537777cc825 --- /dev/null +++ b/packages/kbn-ambient-common-types/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/ambient-common-types", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-ambient-common-types/package.json b/packages/kbn-ambient-common-types/package.json new file mode 100644 index 0000000000000..1794b046ef16e --- /dev/null +++ b/packages/kbn-ambient-common-types/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/ambient-common-types", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-ambient-common-types/tsconfig.json b/packages/kbn-ambient-common-types/tsconfig.json new file mode 100644 index 0000000000000..292157c18591a --- /dev/null +++ b/packages/kbn-ambient-common-types/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-es-query/BUILD.bazel b/packages/kbn-es-query/BUILD.bazel index 772fecdbb9806..95e7dcdcbe3cc 100644 --- a/packages/kbn-es-query/BUILD.bazel +++ b/packages/kbn-es-query/BUILD.bazel @@ -1,5 +1,4 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config") -load("@npm//peggy:index.bzl", "peggy") load("@build_bazel_rules_nodejs//:index.bzl", "js_library") load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") @@ -9,6 +8,8 @@ PKG_REQUIRE_NAME = "@kbn/es-query" SOURCE_FILES = glob( [ "**/*.ts", + "**/grammar.peggy.config.json", + "**/grammar.peggy", ], exclude = [ "**/*.config.js", @@ -51,6 +52,7 @@ RUNTIME_DEPS = [ TYPES_DEPS = [ "//packages/kbn-utility-types:npm_module_types", "//packages/kbn-i18n:npm_module_types", + "//packages/kbn-ambient-common-types:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//tslib", "@npm//@types/jest", @@ -59,26 +61,14 @@ TYPES_DEPS = [ "@npm//@types/node", ] -peggy( - name = "grammar", - data = [ - ":grammar/grammar.peggy", - ":grammar/grammar.config.json" - ], - output_dir = True, - args = [ - "--extra-options-file", - "./%s/grammar/grammar.config.json" % package_name(), - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/grammar.peggy" % package_name() - ], -) - jsts_transpiler( name = "target_node", srcs = SRCS, build_pkg_name = package_name(), + additional_args = [ + "--copy-files", + "--quiet" + ], ) jsts_transpiler( @@ -86,6 +76,10 @@ jsts_transpiler( srcs = SRCS, build_pkg_name = package_name(), web = True, + additional_args = [ + "--copy-files", + "--quiet" + ], ) ts_config( @@ -110,7 +104,7 @@ ts_project( js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], @@ -118,7 +112,7 @@ js_library( js_library( name = "npm_module_types", - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], diff --git a/packages/kbn-es-query/grammar/grammar.peggy b/packages/kbn-es-query/src/kuery/grammar/grammar.peggy similarity index 100% rename from packages/kbn-es-query/grammar/grammar.peggy rename to packages/kbn-es-query/src/kuery/grammar/grammar.peggy diff --git a/packages/kbn-es-query/grammar/grammar.config.json b/packages/kbn-es-query/src/kuery/grammar/grammar.peggy.config.json similarity index 100% rename from packages/kbn-es-query/grammar/grammar.config.json rename to packages/kbn-es-query/src/kuery/grammar/grammar.peggy.config.json diff --git a/packages/kbn-es-query/src/kuery/grammar/index.ts b/packages/kbn-es-query/src/kuery/grammar/index.ts index b05e510486c27..fa1af47631a4a 100644 --- a/packages/kbn-es-query/src/kuery/grammar/index.ts +++ b/packages/kbn-es-query/src/kuery/grammar/index.ts @@ -6,4 +6,4 @@ * Side Public License, v 1. */ -export { parse } from '../../../../grammar/built_grammar.js'; +export { parse } from './grammar.peggy'; diff --git a/packages/kbn-es-query/tsconfig.json b/packages/kbn-es-query/tsconfig.json index 292157c18591a..8561f4bdc4a31 100644 --- a/packages/kbn-es-query/tsconfig.json +++ b/packages/kbn-es-query/tsconfig.json @@ -6,7 +6,8 @@ "outDir": "target_types", "types": [ "jest", - "node" + "node", + "@kbn/ambient-common-types" ] }, "include": [ diff --git a/packages/kbn-import-resolver/src/import_resolver.ts b/packages/kbn-import-resolver/src/import_resolver.ts index 74e2ed52ca453..bab9a9000dd6c 100644 --- a/packages/kbn-import-resolver/src/import_resolver.ts +++ b/packages/kbn-import-resolver/src/import_resolver.ts @@ -125,14 +125,10 @@ export class ImportResolver { return true; } - // ignore requests to grammar/built_grammar.js files or bazel target dirs, these files are only - // available in the build output and will never resolve in dev. We will validate that people don't - // import these files from outside the package in another rule - if ( - req.endsWith('grammar/built_grammar.js') || - req.includes('/target_workers/') || - req.includes('/target_node/') - ) { + // ignore requests to bazel target dirs, these files are only available in the build output + // and will never resolve in dev. We will validate that people don't import these files from + // outside the package in another rule + if (req.includes('/target_workers/') || req.includes('/target_node/')) { return true; } diff --git a/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts b/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts index 1857e6ef26453..a6a3a84602cd7 100644 --- a/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts +++ b/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts @@ -77,12 +77,6 @@ describe('#resolve()', () => { }); it('returns ignore results for known unresolvable but okay import statements', () => { - expect(resolver.resolve('../../grammar/built_grammar.js', FIXTURES_DIR)).toMatchInlineSnapshot(` - Object { - "type": "ignore", - } - `); - expect(resolver.resolve('kibana-buildkite-library', FIXTURES_DIR)).toMatchInlineSnapshot(` Object { "type": "ignore", diff --git a/packages/kbn-interpreter/BUILD.bazel b/packages/kbn-interpreter/BUILD.bazel index 9613a29188bd3..26a359a89cbe6 100644 --- a/packages/kbn-interpreter/BUILD.bazel +++ b/packages/kbn-interpreter/BUILD.bazel @@ -1,5 +1,4 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config") -load("@npm//peggy:index.bzl", "peggy") load("@build_bazel_rules_nodejs//:index.bzl", "js_library") load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") @@ -10,6 +9,8 @@ SOURCE_FILES = glob( [ "**/*.ts", "**/*.js", + "**/grammar.peggy.config.json", + "**/grammar.peggy", ], exclude = [ "**/*.config.js", @@ -48,12 +49,17 @@ TYPES_DEPS = [ "@npm//@types/jest", "@npm//@types/lodash", "@npm//@types/node", + "//packages/kbn-ambient-common-types:npm_module_types" ] jsts_transpiler( name = "target_node", srcs = SRCS, build_pkg_name = package_name(), + additional_args = [ + "--copy-files", + "--quiet" + ], ) jsts_transpiler( @@ -61,21 +67,9 @@ jsts_transpiler( srcs = SRCS, build_pkg_name = package_name(), web = True, -) - -peggy( - name = "grammar", - data = [ - ":grammar/grammar.config.json", - ":grammar/grammar.peggy" - ], - output_dir = True, - args = [ - "--extra-options-file", - "./%s/grammar/grammar.config.json" % package_name(), - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/grammar.peggy" % package_name() + additional_args = [ + "--copy-files", + "--quiet" ], ) @@ -102,7 +96,7 @@ ts_project( js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], @@ -110,7 +104,7 @@ js_library( js_library( name = "npm_module_types", - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], diff --git a/packages/kbn-interpreter/grammar/grammar.peggy b/packages/kbn-interpreter/src/common/lib/grammar.peggy similarity index 100% rename from packages/kbn-interpreter/grammar/grammar.peggy rename to packages/kbn-interpreter/src/common/lib/grammar.peggy diff --git a/packages/kbn-interpreter/grammar/grammar.config.json b/packages/kbn-interpreter/src/common/lib/grammar.peggy.config.json similarity index 100% rename from packages/kbn-interpreter/grammar/grammar.config.json rename to packages/kbn-interpreter/src/common/lib/grammar.peggy.config.json diff --git a/packages/kbn-interpreter/src/common/lib/parse.ts b/packages/kbn-interpreter/src/common/lib/parse.ts index a6d9cc8285689..12a43e06a83c4 100644 --- a/packages/kbn-interpreter/src/common/lib/parse.ts +++ b/packages/kbn-interpreter/src/common/lib/parse.ts @@ -7,7 +7,7 @@ */ import type { Ast, AstWithMeta } from './ast'; -import { parse } from '../../../../grammar/built_grammar.js'; +import { parse } from './grammar.peggy'; interface Options { startRule?: string; diff --git a/packages/kbn-interpreter/tsconfig.json b/packages/kbn-interpreter/tsconfig.json index 57eff16f422bb..e3b4140d05822 100644 --- a/packages/kbn-interpreter/tsconfig.json +++ b/packages/kbn-interpreter/tsconfig.json @@ -6,7 +6,8 @@ "outDir": "target_types", "types": [ "jest", - "node" + "node", + "@kbn/ambient-common-types" ] }, "include": [ diff --git a/packages/kbn-optimizer/BUILD.bazel b/packages/kbn-optimizer/BUILD.bazel index 4906af1ad6f6c..da4b69f02a724 100644 --- a/packages/kbn-optimizer/BUILD.bazel +++ b/packages/kbn-optimizer/BUILD.bazel @@ -47,6 +47,7 @@ RUNTIME_DEPS = [ "//packages/kbn-ui-shared-deps-src", "//packages/kbn-utils", "//packages/kbn-synthetic-package-map", + "//packages/kbn-peggy", "@npm//@babel/core", "@npm//chalk", "@npm//clean-webpack-plugin", @@ -81,6 +82,7 @@ TYPES_DEPS = [ "//packages/kbn-utils:npm_module_types", "//packages/kbn-tooling-log:npm_module_types", "//packages/kbn-synthetic-package-map:npm_module_types", + "//packages/kbn-peggy:npm_module_types", "@npm//chalk", "@npm//clean-webpack-plugin", "@npm//cpy", diff --git a/packages/kbn-optimizer/src/node/cache.ts b/packages/kbn-optimizer/src/node/cache.ts index 1320d275022a9..a521d8e0a0f7a 100644 --- a/packages/kbn-optimizer/src/node/cache.ts +++ b/packages/kbn-optimizer/src/node/cache.ts @@ -101,28 +101,27 @@ export class Cache { } } - async update(path: string, file: { mtime: string; code: string; map: any }) { - const key = this.getKey(path); + close() { + clearTimeout(this.timer); + } + async update(path: string, file: { mtime: string; code: string; map?: any }) { + const key = this.getKey(path); await Promise.all([ this.safePut(this.atimes, key, GLOBAL_ATIME), this.safePut(this.mtimes, key, file.mtime), this.safePut(this.codes, key, file.code), - this.safePut(this.sourceMaps, key, JSON.stringify(file.map)), + file.map != null ? this.safePut(this.sourceMaps, key, JSON.stringify(file.map)) : null, ]); } - close() { - clearTimeout(this.timer); - } - private getKey(path: string) { const normalizedPath = Path.sep !== '/' ? Path.relative(this.pathRoot, path).split(Path.sep).join('/') : Path.relative(this.pathRoot, path); - return `${this.prefix}${normalizedPath}`; + return `${this.prefix}:${normalizedPath}`; } private safeGet(db: LmdbStore.Database, key: string) { diff --git a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts index 8ce58aa74214f..393abd0dc24d0 100644 --- a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts +++ b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts @@ -53,7 +53,7 @@ it('returns undefined until values are set', async () => { const log = makeTestLog(); const cache = makeCache({ dir: DIR, - prefix: 'prefix:', + prefix: 'prefix', log, pathRoot: '/foo/', }); diff --git a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts index 6aa11a3f7020f..5de38ff74cf98 100644 --- a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts +++ b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts @@ -37,15 +37,17 @@ import Fs from 'fs'; import Path from 'path'; import Crypto from 'crypto'; -import * as babel from '@babel/core'; +import { version as babelVersion } from '@babel/core'; +import { VERSION as peggyVersion } from '@kbn/peggy'; import { addHook } from 'pirates'; import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/utils'; import sourceMapSupport from 'source-map-support'; import { readHashOfPackageMap } from '@kbn/synthetic-package-map'; -import { Cache } from './cache'; +import { TRANSFORMS } from './transforms'; +import { getBabelOptions } from './transforms/babel'; -const cwd = process.cwd(); +import { Cache } from './cache'; const IGNORE_PATTERNS = [ /[\/\\]kbn-pm[\/\\]dist[\/\\]/, @@ -63,70 +65,6 @@ const IGNORE_PATTERNS = [ /[\/\\]packages[\/\\](eslint-|kbn-)[^\/\\]+[\/\\](?!src[\/\\].*|(.+[\/\\])?(test|__tests__)[\/\\].+|.+\.test\.(js|ts|tsx)$)(.+$)/, ]; -function getBabelOptions(path: string) { - return babel.loadOptions({ - cwd, - sourceRoot: Path.dirname(path) + Path.sep, - filename: path, - babelrc: false, - presets: [require.resolve('@kbn/babel-preset/node_preset')], - sourceMaps: 'both', - ast: false, - })!; -} - -/** - * @babel/register uses a JSON encoded copy of the config + babel.version - * as the cache key for files, so we do something similar but we don't need - * a unique cache key for every file as our config isn't different for - * different files (by design). Instead we determine a unique prefix and - * automatically prepend all paths with the prefix to create cache keys - */ -function determineCachePrefix() { - const json = JSON.stringify({ - synthPkgMapHash: readHashOfPackageMap(), - babelVersion: babel.version, - // get a config for a fake js, ts, and tsx file to make sure we - // capture conditional config portions based on the file extension - js: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.js')), - ts: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.ts')), - tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')), - }); - - const checksum = Crypto.createHash('sha256').update(json).digest('hex').slice(0, 8); - return `${checksum}:`; -} - -function compile(cache: Cache, source: string, path: string) { - try { - const mtime = `${Fs.statSync(path).mtimeMs}`; - if (cache.getMtime(path) === mtime) { - const code = cache.getCode(path); - if (code) { - // code *should* always be defined, but if it isn't for some reason rebuild it - return code; - } - } - - const options = getBabelOptions(path); - const result = babel.transform(source, options); - - if (!result || !result.code || !result.map) { - throw new Error(`babel failed to transpile [${path}]`); - } - - cache.update(path, { - mtime, - map: result.map, - code: result.code, - }); - - return result.code; - } catch (error) { - throw error; - } -} - let installed = false; export function registerNodeAutoTranspilation() { @@ -135,16 +73,45 @@ export function registerNodeAutoTranspilation() { } installed = true; + const cacheLog = process.env.DEBUG_NODE_TRANSPILER_CACHE + ? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log')) + : undefined; + + const cacheDir = Path.resolve( + REPO_ROOT, + 'data/node_auto_transpilation_cache_v5', + UPSTREAM_BRANCH + ); + + /** + * @babel/register uses a JSON encoded copy of the config + babel.version + * as the cache key for files, so we do something similar but we don't need + * a unique cache key for every file as our config isn't different for + * different files (by design). Instead we determine a unique prefix and + * automatically prepend all paths with the prefix to create cache keys + */ + const cache = new Cache({ + dir: cacheDir, + log: cacheLog, pathRoot: REPO_ROOT, - dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache_v4', UPSTREAM_BRANCH), - prefix: determineCachePrefix(), - log: process.env.DEBUG_NODE_TRANSPILER_CACHE - ? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), { - flags: 'a', + prefix: Crypto.createHash('sha256') + .update( + JSON.stringify({ + synthPkgMapHash: readHashOfPackageMap(), + babelVersion, + peggyVersion, + // get a config for a fake js, ts, and tsx file to make sure we + // capture conditional config portions based on the file extension + js: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.js')), + ts: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.ts')), + tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')), }) - : undefined, + ) + .digest('hex') + .slice(0, 8), }); + cacheLog?.write(`cache initialized\n`); sourceMapSupport.install({ handleUncaughtExceptions: false, @@ -152,39 +119,36 @@ export function registerNodeAutoTranspilation() { // @ts-expect-error bad source-map-support types retrieveSourceMap(path: string) { const map = cache.getSourceMap(path); - - if (map) { - return { - url: null, - map, - }; - } else { - return null; - } + return map ? { map, url: null } : null; }, }); - let compiling = false; - + let transformInProgress = false; addHook( (code, path) => { - if (compiling) { + if (transformInProgress) { return code; } - if (IGNORE_PATTERNS.some((re) => re.test(path))) { + const ext = Path.extname(path); + + if (ext !== '.peggy' && IGNORE_PATTERNS.some((re) => re.test(path))) { return code; } try { - compiling = true; - return compile(cache, code, path); + transformInProgress = true; + const transform = Object.hasOwn(TRANSFORMS, ext) + ? TRANSFORMS[ext as keyof typeof TRANSFORMS] + : TRANSFORMS.default; + + return transform(path, code, cache); } finally { - compiling = false; + transformInProgress = false; } }, { - exts: ['.js', '.ts', '.tsx'], + exts: ['.js', '.ts', '.tsx', '.peggy'], ignoreNodeModules: false, } ); diff --git a/packages/kbn-optimizer/src/node/transforms/babel.ts b/packages/kbn-optimizer/src/node/transforms/babel.ts new file mode 100644 index 0000000000000..6bbe7ba67f76a --- /dev/null +++ b/packages/kbn-optimizer/src/node/transforms/babel.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; + +import * as babel from '@babel/core'; + +import { Transform } from './transform'; + +export function getBabelOptions(path: string) { + return babel.loadOptions({ + cwd: process.cwd(), + sourceRoot: Path.dirname(path) + Path.sep, + filename: path, + babelrc: false, + presets: [require.resolve('@kbn/babel-preset/node_preset')], + sourceMaps: 'both', + ast: false, + })!; +} + +export const babelTransform: Transform = (path, source, cache) => { + const mtime = `${Fs.statSync(path).mtimeMs}`; + + if (cache.getMtime(path) === mtime) { + const code = cache.getCode(path); + if (code) { + return code; + } + } + + const options = getBabelOptions(path); + const result = babel.transform(source, options); + + if (!result || !result.code || !result.map) { + throw new Error(`babel failed to transpile [${path}]`); + } + + cache.update(path, { + mtime, + code: result.code, + map: result.map, + }); + + return result.code; +}; diff --git a/packages/kbn-optimizer/src/node/transforms/index.ts b/packages/kbn-optimizer/src/node/transforms/index.ts new file mode 100644 index 0000000000000..bda2dcfa19826 --- /dev/null +++ b/packages/kbn-optimizer/src/node/transforms/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { peggyTransform } from './peggy'; +import { babelTransform } from './babel'; + +export const TRANSFORMS = { + '.peggy': peggyTransform, + default: babelTransform, +}; diff --git a/packages/kbn-optimizer/src/node/transforms/peggy.ts b/packages/kbn-optimizer/src/node/transforms/peggy.ts new file mode 100644 index 0000000000000..23edb608ef560 --- /dev/null +++ b/packages/kbn-optimizer/src/node/transforms/peggy.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Fs from 'fs'; +import Crypto from 'crypto'; + +import * as Peggy from '@kbn/peggy'; + +import { Transform } from './transform'; + +export const peggyTransform: Transform = (path, source, cache) => { + const config = Peggy.findConfigFile(path); + const mtime = `${Fs.statSync(path).mtimeMs}`; + const key = !config + ? path + : `${path}.config.${Crypto.createHash('sha256') + .update(config.source) + .digest('hex') + .slice(0, 8)}`; + + if (cache.getMtime(key) === mtime) { + const code = cache.getCode(key); + if (code) { + return code; + } + } + + const code = Peggy.getJsSourceSync({ + content: source, + path, + format: 'commonjs', + optimize: 'speed', + config, + skipConfigSearch: true, + }).source; + + cache.update(key, { + code, + mtime, + }); + + return code; +}; diff --git a/packages/kbn-interpreter/src/common/lib/grammar.d.ts b/packages/kbn-optimizer/src/node/transforms/transform.ts similarity index 75% rename from packages/kbn-interpreter/src/common/lib/grammar.d.ts rename to packages/kbn-optimizer/src/node/transforms/transform.ts index a9495dfac5b58..49c76a8c14bd9 100644 --- a/packages/kbn-interpreter/src/common/lib/grammar.d.ts +++ b/packages/kbn-optimizer/src/node/transforms/transform.ts @@ -6,6 +6,6 @@ * Side Public License, v 1. */ -declare module '*/grammar/built_grammar.js' { - export const parse: import('./parse').Parse; -} +import { Cache } from '../cache'; + +export type Transform = (path: string, source: string, cache: Cache) => string; diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 04074fb2b10b4..888089203be4b 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -239,6 +239,10 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: loader: 'raw-loader', }, }, + { + test: /\.peggy$/, + loader: '@kbn/peggy-loader', + }, ], }, diff --git a/packages/kbn-peggy-loader/BUILD.bazel b/packages/kbn-peggy-loader/BUILD.bazel new file mode 100644 index 0000000000000..2d8bed8dd59a4 --- /dev/null +++ b/packages/kbn-peggy-loader/BUILD.bazel @@ -0,0 +1,128 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-peggy-loader" +PKG_REQUIRE_NAME = "@kbn/peggy-loader" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ + "//packages/kbn-peggy", + "@npm//peggy", + "@npm//webpack", +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "//packages/kbn-peggy:npm_module_types", + "@npm//@types/node", + "@npm//@types/jest", + "@npm//@types/webpack", + "@npm//peggy", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +js_library( + name = "npm_module_types", + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node", ":tsc_types"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-peggy-loader/README.md b/packages/kbn-peggy-loader/README.md new file mode 100644 index 0000000000000..2d0d587eaded3 --- /dev/null +++ b/packages/kbn-peggy-loader/README.md @@ -0,0 +1,10 @@ +--- +id: kibDevDocsOpsPeggyLoader +slug: /kibana-dev-docs/ops/peggy-loader +title: "@kbn/peggy" +description: A package which wraps @kbn/peggy for use in Webpack +date: 2022-05-18 +tags: ['kibana', 'dev', 'contributor', 'operations', 'peggy', 'loader'] +--- + +This package wraps the package so that webpack can consume it via its loader interface. This loader, like the `@kbn/peggy` package, loads config files next to grammar files for configuring the peggy parser-generator. \ No newline at end of file diff --git a/packages/kbn-peggy-loader/index.ts b/packages/kbn-peggy-loader/index.ts new file mode 100644 index 0000000000000..4fb6e7cdd8290 --- /dev/null +++ b/packages/kbn-peggy-loader/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getJsSource } from '@kbn/peggy'; +import webpack from 'webpack'; + +// eslint-disable-next-line import/no-default-export +export default function (this: webpack.loader.LoaderContext) { + this.cacheable(true); + + const callback = this.async(); + if (!callback) { + throw new Error('loader requires async support'); + } + + getJsSource({ + path: this.resourcePath, + format: 'esm', + optimize: 'size', + }).then((result) => { + if (result.config) { + this.addDependency(result.config.path); + } + + callback(null, result.source); + }, callback); +} diff --git a/packages/kbn-peggy-loader/jest.config.js b/packages/kbn-peggy-loader/jest.config.js new file mode 100644 index 0000000000000..1a481aa8c088b --- /dev/null +++ b/packages/kbn-peggy-loader/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-peggy-loader'], +}; diff --git a/packages/kbn-peggy-loader/kibana.jsonc b/packages/kbn-peggy-loader/kibana.jsonc new file mode 100644 index 0000000000000..e651946dce5eb --- /dev/null +++ b/packages/kbn-peggy-loader/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/peggy-loader", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-peggy-loader/package.json b/packages/kbn-peggy-loader/package.json new file mode 100644 index 0000000000000..6c2807a006f4a --- /dev/null +++ b/packages/kbn-peggy-loader/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/peggy-loader", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-peggy-loader/tsconfig.json b/packages/kbn-peggy-loader/tsconfig.json new file mode 100644 index 0000000000000..292157c18591a --- /dev/null +++ b/packages/kbn-peggy-loader/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-peggy/BUILD.bazel b/packages/kbn-peggy/BUILD.bazel new file mode 100644 index 0000000000000..dcb225c7da403 --- /dev/null +++ b/packages/kbn-peggy/BUILD.bazel @@ -0,0 +1,124 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-peggy" +PKG_REQUIRE_NAME = "@kbn/peggy" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ + "@npm//peggy", +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "@npm//peggy", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +js_library( + name = "npm_module_types", + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node", ":tsc_types"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-peggy/README.mdx b/packages/kbn-peggy/README.mdx new file mode 100644 index 0000000000000..5d9779976b15a --- /dev/null +++ b/packages/kbn-peggy/README.mdx @@ -0,0 +1,23 @@ +--- +id: kibDevDocsOpsPeggy +slug: /kibana-dev-docs/ops/peggy +title: "@kbn/peggy" +description: A package which wraps the peggy library for use in Kibana +date: 2022-05-18 +tags: ['kibana', 'dev', 'contributor', 'operations', 'peggy'] +--- + +This package wraps the peggy package, exposing a synchronous and async version of the generator which includes two modifications: + + 1. When a `path` is provided a `${basename}.config.json` file will be loaded if it exists and is expected to include peggy config options as defined in [the peggy docs](https://peggyjs.org/documentation.html#generating-a-parser-javascript-api). This config will be used when compiling this file + +## Plugins +These types will automatically be included for plugins. + +## Packages + +To include these types in a package: + +- add `"//packages/kbn-ambient-ui-types"` to the `RUNTIME_DEPS` portion of the `BUILD.bazel` file. +- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file. +- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file. \ No newline at end of file diff --git a/packages/kbn-peggy/index.ts b/packages/kbn-peggy/index.ts new file mode 100644 index 0000000000000..b5b35f131d2ee --- /dev/null +++ b/packages/kbn-peggy/index.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; +import Fsp from 'fs/promises'; + +import Peggy from 'peggy'; + +export interface Options { + /** + * The path to the peggy content. If this is not defined then + * config files can not be found and `content` must be passed. + */ + path?: string; + /** + * Prevent loading the content from disk by specifying it here + */ + content?: string; + /** + * Prevent loading the config from disk by specifying it here + */ + config?: Config; + /** + * What type of module format should the generated code use. Defaults to + * commonjs for broadest compatibility + */ + format?: 'esm' | 'commonjs'; + /** + * Should the parser optimize for execution speed or size of the code + */ + optimize?: 'size' | 'speed'; + /** + * Disable checking for a config file a `{basename}.config.json` in + * the same directory as the grammar file. + */ + skipConfigSearch?: boolean; +} + +export interface Config { + /** the path of the discovered config file */ + path: string; + /** the content of the config file as a string (primarily for hashing) */ + source: string; + /** the parsed content of the config file */ + parsed: any; +} + +export interface Result { + /** + * The source code of the module which parses expressions in the format + * defined by the peggy grammar file + */ + config: Config | null; + + /** + * The loaded config if it was found + */ + source: string; +} + +export function findConfigFile(grammarPath: string): Config | undefined { + const path = Path.resolve(Path.dirname(grammarPath), `${Path.basename(grammarPath)}.config.json`); + + let source; + let parsed; + try { + source = Fs.readFileSync(path, 'utf8'); + parsed = JSON.parse(source); + } catch (error) { + if (error.code === 'ENOENT') { + return undefined; + } + + throw error; + } + + return { path, source, parsed }; +} + +export async function getJsSource(options: Options): Promise { + let source; + if (options.content) { + source = options.content; + } else if (options.path) { + source = await Fsp.readFile(options.path, 'utf8'); + } else { + throw new Error('you must either specify the path of the grammar file, or the content'); + } + + return getJsSourceSync({ + content: source, + ...options, + }); +} + +export function getJsSourceSync( + options: Options & { + /** The content of the grammar file to parse */ + content: string; + } +): Result { + const config = + options.config ?? + (options.path && options.skipConfigSearch !== true ? findConfigFile(options.path) : null); + + const result = Peggy.generate(options.content, { + ...config?.parsed, + format: options.format === 'esm' ? 'es' : 'commonjs', + optimize: options.optimize, + output: 'source', + }); + + return { + /** + * The source code of the module which parses expressions in the format + * defined by the peggy grammar file + */ + source: result as unknown as string, + + /** + * The loaded config if it was found + */ + config: config ?? null, + }; +} + +export const VERSION = Peggy.VERSION; diff --git a/packages/kbn-peggy/jest.config.js b/packages/kbn-peggy/jest.config.js new file mode 100644 index 0000000000000..be8ef99a01b3d --- /dev/null +++ b/packages/kbn-peggy/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-peggy'], +}; diff --git a/packages/kbn-peggy/kibana.jsonc b/packages/kbn-peggy/kibana.jsonc new file mode 100644 index 0000000000000..8b44dc5604d19 --- /dev/null +++ b/packages/kbn-peggy/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/peggy", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-peggy/package.json b/packages/kbn-peggy/package.json new file mode 100644 index 0000000000000..cd976c2e8d97b --- /dev/null +++ b/packages/kbn-peggy/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/peggy", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-peggy/tsconfig.json b/packages/kbn-peggy/tsconfig.json new file mode 100644 index 0000000000000..292157c18591a --- /dev/null +++ b/packages/kbn-peggy/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-storybook/src/webpack.config.ts b/packages/kbn-storybook/src/webpack.config.ts index 3d00fb88a089a..cca984fbe83b5 100644 --- a/packages/kbn-storybook/src/webpack.config.ts +++ b/packages/kbn-storybook/src/webpack.config.ts @@ -82,6 +82,12 @@ export default ({ config: storybookConfig }: { config: Configuration }) => { loader: 'raw-loader', }, }, + { + test: /\.peggy$/, + use: { + loader: '@kbn/peggy-loader', + }, + }, { test: /\.scss$/, exclude: /\.module.(s(a|c)ss)$/, diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index 56b3829fc77a5..425ef3864406f 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -100,6 +100,7 @@ TYPES_DEPS = [ "//packages/kbn-bazel-packages:npm_module_types", "//packages/kbn-get-repo-files:npm_module_types", "//packages/kbn-ftr-screenshot-filename:npm_module_types", + "//packages/kbn-peggy:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//@jest/console", "@npm//@jest/reporters", @@ -119,6 +120,7 @@ TYPES_DEPS = [ "@npm//rxjs", "@npm//playwright", "@npm//xmlbuilder", + "@npm//@jest/transform", "@npm//@types/archiver", "@npm//@types/chance", "@npm//@types/dedent", diff --git a/packages/kbn-test/jest-preset.js b/packages/kbn-test/jest-preset.js index 2c4b3a960e59c..7eb6c8377ebe2 100644 --- a/packages/kbn-test/jest-preset.js +++ b/packages/kbn-test/jest-preset.js @@ -120,9 +120,9 @@ module.exports = { // A map from regular expressions to paths to transformers transform: { - '^.+\\.(js|tsx?)$': '/node_modules/@kbn/test/target_node/src/jest/babel_transform.js', - '^.+\\.txt?$': '/node_modules/@kbn/test/target_node/src/jest/raw_transform.js', - '^.+\\.html?$': '/node_modules/@kbn/test/target_node/src/jest/raw_transform.js', + '^.+\\.(js|tsx?)$': '/node_modules/@kbn/test/target_node/src/jest/transforms/babel.js', + '^.+\\.(txt|html)?$': '/node_modules/@kbn/test/target_node/src/jest/transforms/raw.js', + '^.+\\.peggy?$': '/node_modules/@kbn/test/target_node/src/jest/transforms/peggy.js', }, // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation diff --git a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts index 976a2c417c747..ccdd9cc902c5b 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts @@ -38,6 +38,10 @@ export class EsVersion { this.parsed = parsed; } + toJSON() { + return this.toString(); + } + toString() { return this.parsed.version; } diff --git a/packages/kbn-test/src/jest/babel_transform.js b/packages/kbn-test/src/jest/transforms/babel.js similarity index 100% rename from packages/kbn-test/src/jest/babel_transform.js rename to packages/kbn-test/src/jest/transforms/babel.js diff --git a/packages/kbn-test/src/jest/transforms/peggy.js b/packages/kbn-test/src/jest/transforms/peggy.js new file mode 100644 index 0000000000000..ec32d03150b53 --- /dev/null +++ b/packages/kbn-test/src/jest/transforms/peggy.js @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +const Peggy = require('@kbn/peggy'); +const Crypto = require('crypto'); + +/** @type {import('@jest/transform').AsyncTransformer} */ +module.exports = { + canInstrument: false, + + getCacheKey(sourceText, sourcePath) { + const config = Peggy.findConfigFile(sourcePath); + return ( + Crypto.createHash('sha256').update(sourceText).digest('hex') + + (!config ? '' : `-${Crypto.createHash('sha256').update(config.source).digest('hex')}`) + ); + }, + + process(sourceText, sourcePath) { + return { + code: Peggy.getJsSourceSync({ + content: sourceText, + path: sourcePath, + format: 'commonjs', + optimize: 'speed', + }).source, + }; + }, +}; diff --git a/packages/kbn-test/src/jest/raw_transform.js b/packages/kbn-test/src/jest/transforms/raw.js similarity index 100% rename from packages/kbn-test/src/jest/raw_transform.js rename to packages/kbn-test/src/jest/transforms/raw.js diff --git a/packages/kbn-timelion-grammar/BUILD.bazel b/packages/kbn-timelion-grammar/BUILD.bazel index 3c7ea13cadf64..7898fef88f1fc 100644 --- a/packages/kbn-timelion-grammar/BUILD.bazel +++ b/packages/kbn-timelion-grammar/BUILD.bazel @@ -1,32 +1,18 @@ load("@build_bazel_rules_nodejs//:index.bzl", "js_library") -load("@npm//peggy:index.bzl", "peggy") load("//src/dev/bazel:index.bzl", "pkg_npm") PKG_DIRNAME = "kbn-timelion-grammar" PKG_REQUIRE_NAME = "@kbn/timelion-grammar" NPM_MODULE_EXTRA_FILES = [ + "index.js", + "chain.peggy", "package.json", ] -peggy( - name = "grammar", - data = [ - ":grammar/chain.peggy" - ], - output_dir = True, - args = [ - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/chain.peggy" % package_name() - ], -) - js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [ - ":grammar" - ], + srcs = NPM_MODULE_EXTRA_FILES, package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], ) diff --git a/packages/kbn-timelion-grammar/grammar/chain.peggy b/packages/kbn-timelion-grammar/chain.peggy similarity index 100% rename from packages/kbn-timelion-grammar/grammar/chain.peggy rename to packages/kbn-timelion-grammar/chain.peggy diff --git a/packages/kbn-es-query/src/kuery/grammar/grammar.d.ts b/packages/kbn-timelion-grammar/index.js similarity index 82% rename from packages/kbn-es-query/src/kuery/grammar/grammar.d.ts rename to packages/kbn-timelion-grammar/index.js index 245f6839d7d04..6dd2fdb6abea7 100644 --- a/packages/kbn-es-query/src/kuery/grammar/grammar.d.ts +++ b/packages/kbn-timelion-grammar/index.js @@ -6,6 +6,4 @@ * Side Public License, v 1. */ -declare module '*/grammar/built_grammar.js' { - export const parse: any; -} +module.exports = require('./chain.peggy'); diff --git a/packages/kbn-timelion-grammar/package.json b/packages/kbn-timelion-grammar/package.json index 4676d1e6b94d9..5b519870c423d 100644 --- a/packages/kbn-timelion-grammar/package.json +++ b/packages/kbn-timelion-grammar/package.json @@ -2,6 +2,5 @@ "name": "@kbn/timelion-grammar", "version": "1.0.0", "license": "SSPL-1.0 OR Elastic License 2.0", - "private": true, - "main": "grammar/built_grammar.js" + "private": true } \ No newline at end of file diff --git a/packages/kbn-tinymath/BUILD.bazel b/packages/kbn-tinymath/BUILD.bazel index b8ee25a4973b8..b9f1fb9daf849 100644 --- a/packages/kbn-tinymath/BUILD.bazel +++ b/packages/kbn-tinymath/BUILD.bazel @@ -1,5 +1,4 @@ load("@build_bazel_rules_nodejs//:index.bzl", "js_library") -load("@npm//peggy:index.bzl", "peggy") load("//src/dev/bazel:index.bzl", "pkg_npm") PKG_DIRNAME = "kbn-tinymath" @@ -31,25 +30,9 @@ RUNTIME_DEPS = [ "@npm//lodash", ] -peggy( - name = "grammar", - data = [ - ":grammar/grammar.peggy" - ], - output_dir = True, - args = [ - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/grammar.peggy" % package_name() - ], -) - js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [ - ":srcs", - ":grammar" - ], + srcs = NPM_MODULE_EXTRA_FILES + [":srcs"], deps = RUNTIME_DEPS, package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], diff --git a/packages/kbn-tinymath/grammar/grammar.peggy b/packages/kbn-tinymath/src/grammar.peggy similarity index 100% rename from packages/kbn-tinymath/grammar/grammar.peggy rename to packages/kbn-tinymath/src/grammar.peggy diff --git a/packages/kbn-tinymath/src/index.js b/packages/kbn-tinymath/src/index.js index dd23b0847c999..5bc7d80faff9e 100644 --- a/packages/kbn-tinymath/src/index.js +++ b/packages/kbn-tinymath/src/index.js @@ -9,7 +9,7 @@ const { get } = require('lodash'); const memoizeOne = require('memoize-one'); const { functions: includedFunctions } = require('./functions'); -const { parse: parseFn } = require('../grammar/built_grammar.js'); +const { parse: parseFn } = require('./grammar.peggy'); function parse(input, options) { if (input == null) { diff --git a/packages/kbn-ui-shared-deps-src/BUILD.bazel b/packages/kbn-ui-shared-deps-src/BUILD.bazel index 97006c36eb285..91d64b9159be9 100644 --- a/packages/kbn-ui-shared-deps-src/BUILD.bazel +++ b/packages/kbn-ui-shared-deps-src/BUILD.bazel @@ -50,6 +50,7 @@ RUNTIME_DEPS = [ "//packages/kbn-std", "//packages/kbn-ui-shared-deps-npm", "//packages/kbn-ui-theme", + "//packages/kbn-peggy-loader", ] TYPES_DEPS = [ diff --git a/packages/kbn-ui-shared-deps-src/webpack.config.js b/packages/kbn-ui-shared-deps-src/webpack.config.js index 85d14314c5c60..bebe6eb424f06 100644 --- a/packages/kbn-ui-shared-deps-src/webpack.config.js +++ b/packages/kbn-ui-shared-deps-src/webpack.config.js @@ -56,6 +56,10 @@ module.exports = { }, ], }, + { + test: /\.peggy$/, + use: ['@kbn/peggy-loader'], + }, { test: /\.css$/, use: [MiniCssExtractPlugin.loader, 'css-loader'], diff --git a/src/dev/build/lib/integration_tests/scan_copy.test.ts b/src/dev/build/lib/integration_tests/scan_copy.test.ts index dd69ad78789a7..464821707dbab 100644 --- a/src/dev/build/lib/integration_tests/scan_copy.test.ts +++ b/src/dev/build/lib/integration_tests/scan_copy.test.ts @@ -89,7 +89,7 @@ it('applies filter function specified', async () => { await scanCopy({ source: FIXTURES, destination, - filter: (record) => !record.name.includes('bar'), + filter: (record) => !record.source.name.includes('bar'), }); expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([ diff --git a/src/dev/build/lib/scan_copy.ts b/src/dev/build/lib/scan_copy.ts index d8285e1853173..1c5e29f420588 100644 --- a/src/dev/build/lib/scan_copy.ts +++ b/src/dev/build/lib/scan_copy.ts @@ -9,11 +9,16 @@ import Fs from 'fs'; import Fsp from 'fs/promises'; import Path from 'path'; - -import { asyncMap, asyncForEach } from '@kbn/std'; +import * as Rx from 'rxjs'; import { assertAbsolute, mkdirp } from './fs'; +const fsReadDir$ = Rx.bindNodeCallback( + (path: string, cb: (err: Error | null, ents: Fs.Dirent[]) => void) => { + Fs.readdir(path, { withFileTypes: true }, cb); + } +); + interface Options { /** * directory to copy from @@ -26,75 +31,158 @@ interface Options { /** * function that is called with each Record */ - filter?: (record: Record) => boolean; + filter?: (record: Readonly) => boolean; /** * define permissions for reach item copied */ - permissions?: (record: Record) => number | undefined; + permissions?: (record: Readonly) => number | undefined; /** * Date to use for atime/mtime */ time?: Date; + /** + * + */ + map?: (record: Readonly) => Promise; } -class Record { +export class SomePath { + static fromAbs(path: string) { + return new SomePath(Path.dirname(path), Path.basename(path)); + } + constructor( - public isDirectory: boolean, - public name: string, - public absolute: string, - public absoluteDest: string + /** The directory of the item at this path */ + public readonly dir: string, + /** The name of the item at this path */ + public readonly name: string ) {} + + private _abs: string | null = null; + /** The absolute path of the file */ + public get abs() { + if (this._abs === null) { + this._abs = Path.resolve(this.dir, this.name); + } + + return this._abs; + } + + private _ext: string | null = null; + /** The extension of the filename, starts with a . like the Path.extname API */ + public get ext() { + if (this._ext === null) { + this._ext = Path.extname(this.name); + } + + return this._ext; + } + + /** return a file path with the file name changed to `name` */ + withName(name: string) { + return new SomePath(this.dir, name); + } + + /** return a file path with the file extension changed to `extension` */ + withExt(extension: string) { + return new SomePath(this.dir, Path.basename(this.name, this.ext) + extension); + } + + child(childName: string) { + return new SomePath(this.abs, childName); + } +} + +interface DirRecord { + type: 'dir'; + source: SomePath; + dest: SomePath; +} + +interface FileRecord { + type: 'file'; + source: SomePath; + dest: SomePath; + content?: string; } +type Record = FileRecord | DirRecord; + /** * Copy all of the files from one directory to another, optionally filtered with a * function or modifying mtime/atime for each file. */ export async function scanCopy(options: Options) { - const { source, destination, filter, time, permissions } = options; + const { source, destination, filter, time, permissions, map } = options; assertAbsolute(source); assertAbsolute(destination); - // create or copy each child of a directory - const copyChildren = async (parent: Record) => { - const names = await Fsp.readdir(parent.absolute); + /** + * recursively fetch all the file records within a directory, starting with the + * files in the passed directory, then the files in all the child directories in + * no particular order + */ + const readDir$ = (dir: DirRecord): Rx.Observable => + fsReadDir$(dir.source.abs).pipe( + Rx.mergeAll(), + Rx.mergeMap((ent) => { + const rec: Record = { + type: ent.isDirectory() ? 'dir' : 'file', + source: dir.source.child(ent.name), + dest: dir.dest.child(ent.name), + }; - const records = await asyncMap(names, async (name) => { - const absolute = Path.join(parent.absolute, name); - const stat = await Fsp.stat(absolute); - return new Record(stat.isDirectory(), name, absolute, Path.join(parent.absoluteDest, name)); - }); + if (filter && !filter(rec)) { + return Rx.EMPTY; + } + + return Rx.of(rec); + }) + ); - await asyncForEach(records, async (rec) => { - if (filter && !filter(rec)) { - return; + const handleGenericRec = async (rec: Record) => { + if (permissions) { + const perm = permissions(rec); + if (perm !== undefined) { + await Fsp.chmod(rec.dest.abs, perm); } + } + + if (time) { + await Fsp.utimes(rec.dest.abs, time, time); + } + }; - if (rec.isDirectory) { - await Fsp.mkdir(rec.absoluteDest, { - mode: permissions ? permissions(rec) : undefined, + const handleDir$ = (rec: DirRecord): Rx.Observable => + Rx.defer(async () => { + await mkdirp(rec.dest.abs); + await handleGenericRec(rec); + }).pipe( + Rx.mergeMap(() => readDir$(rec)), + Rx.mergeMap((ent) => (ent.type === 'dir' ? handleDir$(ent) : handleFile$(ent))) + ); + + const handleFile$ = (srcRec: FileRecord): Rx.Observable => + Rx.defer(async () => { + const rec = (map && (await map(srcRec))) ?? srcRec; + + if (rec.content) { + await Fsp.writeFile(rec.dest.abs, rec.content, { + flag: 'wx', }); } else { - await Fsp.copyFile(rec.absolute, rec.absoluteDest, Fs.constants.COPYFILE_EXCL); - if (permissions) { - const perm = permissions(rec); - if (perm !== undefined) { - await Fsp.chmod(rec.absoluteDest, perm); - } - } + await Fsp.copyFile(rec.source.abs, rec.dest.abs, Fs.constants.COPYFILE_EXCL); } - if (time) { - await Fsp.utimes(rec.absoluteDest, time, time); - } - - if (rec.isDirectory) { - await copyChildren(rec); - } + await handleGenericRec(rec); }); - }; - await mkdirp(destination); - await copyChildren(new Record(true, Path.basename(source), source, destination)); + await Rx.lastValueFrom( + handleDir$({ + type: 'dir', + source: SomePath.fromAbs(source), + dest: SomePath.fromAbs(destination), + }) + ); } diff --git a/src/dev/build/tasks/build_packages_task.ts b/src/dev/build/tasks/build_packages_task.ts index fdb32731fdd8e..2cb0bb585b56a 100644 --- a/src/dev/build/tasks/build_packages_task.ts +++ b/src/dev/build/tasks/build_packages_task.ts @@ -11,8 +11,9 @@ import Path from 'path'; import { REPO_ROOT } from '@kbn/utils'; import { discoverBazelPackages } from '@kbn/bazel-packages'; import { runBazel } from '@kbn/bazel-runner'; +import * as Peggy from '@kbn/peggy'; -import { Task, scanCopy, write } from '../lib'; +import { Task, scanCopy, write, deleteAll } from '../lib'; export const BuildBazelPackages: Task = { description: 'Building distributable versions of Bazel packages', @@ -26,16 +27,54 @@ export const BuildBazelPackages: Task = { log.info(`Copying build of`, pkg.manifest.id, 'into build'); const pkgDirInBuild = build.resolvePath(pkg.normalizedRepoRelativeDir); + const peggyConfigOutputPaths = new Set(); + const pkgBuildDir = config.resolveFromRepo( + 'bazel-bin', + pkg.normalizedRepoRelativeDir, + 'npm_module' + ); // copy the built npm_module target dir into the build, package.json is updated to copy // the sources we actually end up using into the node_modules directory when we run // yarn install await scanCopy({ - source: config.resolveFromRepo('bazel-bin', pkg.normalizedRepoRelativeDir, 'npm_module'), + source: pkgBuildDir, destination: pkgDirInBuild, - permissions: (rec) => (rec.isDirectory ? 0o755 : 0o644), + permissions: (rec) => (rec.type === 'file' ? 0o644 : 0o755), + filter: (rec) => !(rec.type === 'dir' && rec.source.name === 'target_web'), + async map(rec) { + const extname = Path.extname(rec.source.name); + if (extname !== '.peggy') { + return undefined; + } + + const result = await Peggy.getJsSource({ + path: rec.source.abs, + format: 'commonjs', + optimize: 'speed', + }); + + if (result.config) { + // if there was a config file for this peggy grammar, capture its output path and + // delete it after the copy is complete + peggyConfigOutputPaths.add( + Path.resolve(pkgDirInBuild, Path.relative(pkgBuildDir, result.config.path)) + ); + } + + return { + ...rec, + dest: rec.dest.withName(rec.dest.name + '.js'), + content: result.source, + }; + }, }); + // cleanup any peggy config files + if (peggyConfigOutputPaths.size) { + await deleteAll(Array.from(peggyConfigOutputPaths), log); + } + await write( Path.resolve(pkgDirInBuild, 'kibana.jsonc'), JSON.stringify(pkg.manifest, null, 2) diff --git a/tsconfig.base.json b/tsconfig.base.json index 12bbf2ea85736..7b0a8e6781baf 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -376,6 +376,8 @@ "@kbn/ace/*": ["packages/kbn-ace/*"], "@kbn/alerts": ["packages/kbn-alerts"], "@kbn/alerts/*": ["packages/kbn-alerts/*"], + "@kbn/ambient-common-types": ["packages/kbn-ambient-common-types"], + "@kbn/ambient-common-types/*": ["packages/kbn-ambient-common-types/*"], "@kbn/ambient-storybook-types": ["packages/kbn-ambient-storybook-types"], "@kbn/ambient-storybook-types/*": ["packages/kbn-ambient-storybook-types/*"], "@kbn/ambient-ui-types": ["packages/kbn-ambient-ui-types"], @@ -518,6 +520,10 @@ "@kbn/optimizer-webpack-helpers/*": ["packages/kbn-optimizer-webpack-helpers/*"], "@kbn/osquery-io-ts-types": ["packages/kbn-osquery-io-ts-types"], "@kbn/osquery-io-ts-types/*": ["packages/kbn-osquery-io-ts-types/*"], + "@kbn/peggy": ["packages/kbn-peggy"], + "@kbn/peggy/*": ["packages/kbn-peggy/*"], + "@kbn/peggy-loader": ["packages/kbn-peggy-loader"], + "@kbn/peggy-loader/*": ["packages/kbn-peggy-loader/*"], "@kbn/performance-testing-dataset-extractor": ["packages/kbn-performance-testing-dataset-extractor"], "@kbn/performance-testing-dataset-extractor/*": ["packages/kbn-performance-testing-dataset-extractor/*"], "@kbn/plugin-discovery": ["packages/kbn-plugin-discovery"], @@ -1276,6 +1282,7 @@ "@testing-library/jest-dom", "@emotion/react/types/css-prop", "@kbn/ambient-ui-types", + "@kbn/ambient-common-types", "@kbn/ambient-storybook-types" ] }, diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js index 24ae9b3faa1f1..2e69f52bdf651 100644 --- a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js +++ b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js @@ -187,6 +187,10 @@ module.exports = { ], use: require.resolve('null-loader'), }, + { + test: /\.peggy$/, + use: require.resolve('@kbn/peggy-loader'), + }, ], }, node: { diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts b/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts index fb1e93ddbe956..a0f6c8fe0e9bb 100644 --- a/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts +++ b/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts @@ -11,11 +11,8 @@ import Path from 'path'; import webpack from 'webpack'; import { ToolingLog } from '@kbn/tooling-log'; -import { REPO_ROOT } from '@kbn/utils'; -import normalizePath from 'normalize-path'; import { CiStatsReporter } from '@kbn/ci-stats-reporter'; import { isNormalModule, isConcatenatedModule } from '@kbn/optimizer-webpack-helpers'; -import { RUNTIME_SIZE_LIMIT } from './runtime_size_limit'; const IGNORED_EXTNAME = ['.map', '.br', '.gz']; @@ -91,10 +88,6 @@ export class CiStatsPlugin { group: `canvas shareable runtime`, id: 'total size', value: entry.size, - limit: RUNTIME_SIZE_LIMIT, - limitConfigPath: normalizePath( - Path.relative(REPO_ROOT, require.resolve('./runtime_size_limit')) - ), }, { group: `canvas shareable runtime`, diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts b/x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts deleted file mode 100644 index 51b16b6ccbd52..0000000000000 --- a/x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const RUNTIME_SIZE_LIMIT = 8_200_000; diff --git a/yarn.lock b/yarn.lock index 9052863b64a9e..f81e0fcfc311c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2705,6 +2705,10 @@ version "0.0.0" uid "" +"@kbn/ambient-common-types@link:bazel-bin/packages/kbn-ambient-common-types": + version "0.0.0" + uid "" + "@kbn/ambient-storybook-types@link:bazel-bin/packages/kbn-ambient-storybook-types": version "0.0.0" uid "" @@ -3741,6 +3745,14 @@ version "0.0.0" uid "" +"@kbn/peggy-loader@link:bazel-bin/packages/kbn-peggy-loader": + version "0.0.0" + uid "" + +"@kbn/peggy@link:bazel-bin/packages/kbn-peggy": + version "0.0.0" + uid "" + "@kbn/performance-testing-dataset-extractor@link:bazel-bin/packages/kbn-performance-testing-dataset-extractor": version "0.0.0" uid "" @@ -4399,6 +4411,36 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.2.0.tgz#901c5937e1441572ea23e631fe6deca68482fe76" + integrity sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ== + +"@msgpackr-extract/msgpackr-extract-darwin-x64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.2.0.tgz#fb877fe6bae3c4d3cea29786737840e2ae689066" + integrity sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.2.0.tgz#986179c38b10ac41fbdaf7d036c825cbc72855d9" + integrity sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA== + +"@msgpackr-extract/msgpackr-extract-linux-arm@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.2.0.tgz#15f2c6fe9e0adc06c21af7e95f484ff4880d79ce" + integrity sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg== + +"@msgpackr-extract/msgpackr-extract-linux-x64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.2.0.tgz#30cae5c9a202f3e1fa1deb3191b18ffcb2f239a2" + integrity sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw== + +"@msgpackr-extract/msgpackr-extract-win32-x64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz#016d855b6bc459fd908095811f6826e45dd4ba64" + integrity sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA== + "@nicolo-ribaudo/chokidar-2@2.1.8-no-fsevents.3": version "2.1.8-no-fsevents.3" resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz#323d72dd25103d0c4fbdce89dadf574a787b1f9b" @@ -18425,7 +18467,7 @@ listr@^0.14.1: p-map "^2.0.0" rxjs "^6.3.3" -lmdb-store@^1.6.11: +lmdb-store@^1: version "1.6.11" resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d" integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q== @@ -19781,20 +19823,26 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -msgpackr-extract@^1.0.14: - version "1.0.14" - resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.0.14.tgz#87d3fe825d226e7f3d9fe136375091137f958561" - integrity sha512-t8neMf53jNZRF+f0H9VvEUVvtjGZ21odSBRmFfjZiyxr9lKYY0mpY3kSWZAIc7YWXtCZGOvDQVx2oqcgGiRBrw== +msgpackr-extract@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262" + integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog== dependencies: - nan "^2.14.2" - node-gyp-build "^4.2.3" + node-gyp-build-optional-packages "5.0.3" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-arm" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0" msgpackr@^1.4.7: - version "1.4.7" - resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.4.7.tgz#d802ade841e7d2e873000b491cdda6574a3d5748" - integrity sha512-bhC8Ed1au3L3oHaR/fe4lk4w7PLGFcWQ5XY/Tk9N6tzDRz8YndjCG68TD8zcvYZoxNtw767eF/7VpaTpU9kf9w== + version "1.8.0" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.8.0.tgz#6cf213e88f04c5a358c61085a42a4dbe5542de44" + integrity sha512-1Cos3r86XACdjLVY4CN8r72Cgs5lUzxSON6yb81sNZP9vC9nnBrEbu1/ldBhuR9BKejtoYV5C9UhmYUvZFJSNQ== optionalDependencies: - msgpackr-extract "^1.0.14" + msgpackr-extract "^2.2.0" multicast-dns@^7.2.5: version "7.2.5" @@ -20670,9 +20718,9 @@ ora@^5.4.1: wcwidth "^1.0.1" ordered-binary@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.0.0.tgz#4f7485186b12aa42b99011aeb7aa272991d6a487" - integrity sha512-0RMlzqix3YAOZKMoXv97OIvHlqJxnmIzihjShVkYNV3JuzHbqeBOOP7wpz6yo4af1ZFnOHGsh8RK77ZmaBY3Lg== + version "1.4.0" + resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389" + integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ== ordered-read-streams@^1.0.0: version "1.0.1" @@ -27418,9 +27466,9 @@ wcwidth@^1.0.1: defaults "^1.0.3" weak-lru-cache@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.0.0.tgz#f1394721169883488c554703704fbd91cda05ddf" - integrity sha512-135bPugHHIJLNx20guHgk4etZAbd7nou34NQfdKkJPgMuC3Oqn4cT6f7ORVvnud9oEyXJVJXPcTFsUvttGm5xg== + version "1.2.2" + resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19" + integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw== web-namespaces@^1.0.0: version "1.1.4"