From cfdb8553ba37cb81ec045a94da8ffb75684805c8 Mon Sep 17 00:00:00 2001 From: Spencer Date: Tue, 22 Nov 2022 11:25:50 -0700 Subject: [PATCH] [pkgs/peggy] automatically transform peggy files with babel-register and webpack (#145615) In order to get us closer to the developer experience we want for packages, we are trying to move package builds out of bazel and instead we want to build files on demand. In the case of .peggy files this means importing them directly and teaching babel/jest/webpack how to handle these imports by automatically transpiling and caching the results. This change does just that, adding a `@kbn/peggy` package which wraps peggy for types, and also adds support for defining peggy config adjacent to a peggy grammar file in a `${basename}.config.json` file. This file will be parsed and used to configure things like `allowedStartRules` as described in [the peggy docs](https://peggyjs.org/documentation.html#generating-a-parser-javascript-api). This PR also implements `@kbn/peggy-loader` which uses `@kbn/peggy` to transpile peggy files in webpack, and a peggy transform for both Jest and our custom babel register hook. Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 3 + dev_docs/operations/operations_landing.mdx | 3 + package.json | 6 +- packages/BUILD.bazel | 5 + packages/kbn-ambient-common-types/BUILD.bazel | 58 ++++++ packages/kbn-ambient-common-types/README.mdx | 20 +++ packages/kbn-ambient-common-types/index.d.ts | 36 ++++ .../kbn-ambient-common-types/jest.config.js | 13 ++ .../kbn-ambient-common-types/kibana.jsonc | 8 + .../kbn-ambient-common-types/package.json | 8 + .../kbn-ambient-common-types/tsconfig.json | 15 ++ packages/kbn-es-query/BUILD.bazel | 32 ++-- .../{ => src/kuery}/grammar/grammar.peggy | 0 .../kuery/grammar/grammar.peggy.config.json} | 0 .../kbn-es-query/src/kuery/grammar/index.ts | 2 +- packages/kbn-es-query/tsconfig.json | 3 +- .../src/import_resolver.ts | 12 +- .../integration_tests/import_resolver.test.ts | 6 - packages/kbn-interpreter/BUILD.bazel | 30 ++-- .../{grammar => src/common/lib}/grammar.peggy | 0 .../common/lib/grammar.peggy.config.json} | 0 .../kbn-interpreter/src/common/lib/parse.ts | 2 +- packages/kbn-interpreter/tsconfig.json | 3 +- packages/kbn-optimizer/BUILD.bazel | 2 + packages/kbn-optimizer/src/node/cache.ts | 15 +- .../src/node/integration_tests/cache.test.ts | 2 +- .../src/node/node_auto_tranpilation.ts | 144 ++++++--------- .../src/node/transforms/babel.ts | 52 ++++++ .../src/node/transforms/index.ts | 15 ++ .../src/node/transforms/peggy.ts | 48 +++++ .../src/node/transforms/transform.ts} | 6 +- .../src/worker/webpack.config.ts | 4 + packages/kbn-peggy-loader/BUILD.bazel | 128 +++++++++++++ packages/kbn-peggy-loader/README.md | 10 ++ packages/kbn-peggy-loader/index.ts | 32 ++++ packages/kbn-peggy-loader/jest.config.js | 13 ++ packages/kbn-peggy-loader/kibana.jsonc | 8 + packages/kbn-peggy-loader/package.json | 8 + packages/kbn-peggy-loader/tsconfig.json | 15 ++ packages/kbn-peggy/BUILD.bazel | 124 +++++++++++++ packages/kbn-peggy/README.mdx | 23 +++ packages/kbn-peggy/index.ts | 133 ++++++++++++++ packages/kbn-peggy/jest.config.js | 13 ++ packages/kbn-peggy/kibana.jsonc | 8 + packages/kbn-peggy/package.json | 8 + packages/kbn-peggy/tsconfig.json | 15 ++ packages/kbn-storybook/src/webpack.config.ts | 6 + packages/kbn-test/BUILD.bazel | 2 + packages/kbn-test/jest-preset.js | 6 +- .../functional_test_runner/lib/es_version.ts | 4 + .../babel.js} | 0 .../kbn-test/src/jest/transforms/peggy.js | 34 ++++ .../{raw_transform.js => transforms/raw.js} | 0 packages/kbn-timelion-grammar/BUILD.bazel | 20 +-- .../{grammar => }/chain.peggy | 0 .../index.js} | 4 +- packages/kbn-timelion-grammar/package.json | 3 +- packages/kbn-tinymath/BUILD.bazel | 19 +- .../{grammar => src}/grammar.peggy | 0 packages/kbn-tinymath/src/index.js | 2 +- packages/kbn-ui-shared-deps-src/BUILD.bazel | 1 + .../kbn-ui-shared-deps-src/webpack.config.js | 4 + .../lib/integration_tests/scan_copy.test.ts | 2 +- src/dev/build/lib/scan_copy.ts | 170 +++++++++++++----- src/dev/build/tasks/build_packages_task.ts | 45 ++++- tsconfig.base.json | 7 + .../shareable_runtime/webpack.config.js | 4 + .../webpack/ci_stats_plugin.ts | 7 - .../webpack/runtime_size_limit.ts | 8 - yarn.lock | 82 +++++++-- 70 files changed, 1242 insertions(+), 279 deletions(-) create mode 100644 packages/kbn-ambient-common-types/BUILD.bazel create mode 100644 packages/kbn-ambient-common-types/README.mdx create mode 100644 packages/kbn-ambient-common-types/index.d.ts create mode 100644 packages/kbn-ambient-common-types/jest.config.js create mode 100644 packages/kbn-ambient-common-types/kibana.jsonc create mode 100644 packages/kbn-ambient-common-types/package.json create mode 100644 packages/kbn-ambient-common-types/tsconfig.json rename packages/kbn-es-query/{ => src/kuery}/grammar/grammar.peggy (100%) rename packages/kbn-es-query/{grammar/grammar.config.json => src/kuery/grammar/grammar.peggy.config.json} (100%) rename packages/kbn-interpreter/{grammar => src/common/lib}/grammar.peggy (100%) rename packages/kbn-interpreter/{grammar/grammar.config.json => src/common/lib/grammar.peggy.config.json} (100%) create mode 100644 packages/kbn-optimizer/src/node/transforms/babel.ts create mode 100644 packages/kbn-optimizer/src/node/transforms/index.ts create mode 100644 packages/kbn-optimizer/src/node/transforms/peggy.ts rename packages/{kbn-interpreter/src/common/lib/grammar.d.ts => kbn-optimizer/src/node/transforms/transform.ts} (75%) create mode 100644 packages/kbn-peggy-loader/BUILD.bazel create mode 100644 packages/kbn-peggy-loader/README.md create mode 100644 packages/kbn-peggy-loader/index.ts create mode 100644 packages/kbn-peggy-loader/jest.config.js create mode 100644 packages/kbn-peggy-loader/kibana.jsonc create mode 100644 packages/kbn-peggy-loader/package.json create mode 100644 packages/kbn-peggy-loader/tsconfig.json create mode 100644 packages/kbn-peggy/BUILD.bazel create mode 100644 packages/kbn-peggy/README.mdx create mode 100644 packages/kbn-peggy/index.ts create mode 100644 packages/kbn-peggy/jest.config.js create mode 100644 packages/kbn-peggy/kibana.jsonc create mode 100644 packages/kbn-peggy/package.json create mode 100644 packages/kbn-peggy/tsconfig.json rename packages/kbn-test/src/jest/{babel_transform.js => transforms/babel.js} (100%) create mode 100644 packages/kbn-test/src/jest/transforms/peggy.js rename packages/kbn-test/src/jest/{raw_transform.js => transforms/raw.js} (100%) rename packages/kbn-timelion-grammar/{grammar => }/chain.peggy (100%) rename packages/{kbn-es-query/src/kuery/grammar/grammar.d.ts => kbn-timelion-grammar/index.js} (82%) rename packages/kbn-tinymath/{grammar => src}/grammar.peggy (100%) delete mode 100644 x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 469ff0be66ae5..f256d3467caa5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -887,6 +887,7 @@ packages/home/sample_data_tab @elastic/kibana-global-experience packages/home/sample_data_types @elastic/kibana-global-experience packages/kbn-ace @elastic/platform-deployment-management packages/kbn-alerts @elastic/security-solution +packages/kbn-ambient-common-types @elastic/kibana-operations packages/kbn-ambient-storybook-types @elastic/kibana-operations packages/kbn-ambient-ui-types @elastic/kibana-operations packages/kbn-analytics @elastic/kibana-core @@ -958,6 +959,8 @@ packages/kbn-monaco @elastic/kibana-app-services packages/kbn-optimizer @elastic/kibana-operations packages/kbn-optimizer-webpack-helpers @elastic/kibana-operations packages/kbn-osquery-io-ts-types @elastic/security-asset-management +packages/kbn-peggy @elastic/kibana-operations +packages/kbn-peggy-loader @elastic/kibana-operations packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing packages/kbn-plugin-discovery @elastic/kibana-operations packages/kbn-plugin-generator @elastic/kibana-operations diff --git a/dev_docs/operations/operations_landing.mdx b/dev_docs/operations/operations_landing.mdx index 9004141255f58..b38570ea44d13 100644 --- a/dev_docs/operations/operations_landing.mdx +++ b/dev_docs/operations/operations_landing.mdx @@ -64,6 +64,7 @@ layout: landing { pageId: "kibDevDocsOpsExpect" }, { pageId: "kibDevDocsOpsAmbientStorybookTypes" }, { pageId: "kibDevDocsOpsAmbientUiTypes" }, + { pageId: "kibDevDocsOpsAmbientCommonTypes" }, { pageId: "kibDevDocsOpsTestSubjSelector" }, { pageId: "kibDevDocsOpsBazelRunner" }, { pageId: "kibDevDocsOpsCliDevMode" }, @@ -77,5 +78,7 @@ layout: landing { pageId: "kibDevDocsOpsManagedVscodeConfigCli" }, { pageId: "kibDevDocsOpsTest" }, { pageId: "kibDevDocsOpsEsArchiver" }, + { pageId: "kibDevDocsOpsPeggy" }, + { pageId: "kibDevDocsOpsPeggyLoader" }, ]} /> \ No newline at end of file diff --git a/package.json b/package.json index 67a78a92d9582..4e4123e8cdc98 100644 --- a/package.json +++ b/package.json @@ -718,7 +718,9 @@ "@istanbuljs/schema": "^0.1.2", "@jest/console": "^29.3.1", "@jest/reporters": "^29.3.1", + "@jest/transform": "^29.3.1", "@jest/types": "^29.3.1", + "@kbn/ambient-common-types": "link:bazel-bin/packages/kbn-ambient-common-types", "@kbn/ambient-storybook-types": "link:bazel-bin/packages/kbn-ambient-storybook-types", "@kbn/ambient-ui-types": "link:bazel-bin/packages/kbn-ambient-ui-types", "@kbn/apm-synthtrace": "link:bazel-bin/packages/kbn-apm-synthtrace", @@ -757,6 +759,8 @@ "@kbn/managed-vscode-config-cli": "link:bazel-bin/packages/kbn-managed-vscode-config-cli", "@kbn/optimizer": "link:bazel-bin/packages/kbn-optimizer", "@kbn/optimizer-webpack-helpers": "link:bazel-bin/packages/kbn-optimizer-webpack-helpers", + "@kbn/peggy": "link:bazel-bin/packages/kbn-peggy", + "@kbn/peggy-loader": "link:bazel-bin/packages/kbn-peggy-loader", "@kbn/performance-testing-dataset-extractor": "link:bazel-bin/packages/kbn-performance-testing-dataset-extractor", "@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator", "@kbn/plugin-helpers": "link:bazel-bin/packages/kbn-plugin-helpers", @@ -1060,7 +1064,7 @@ "jsondiffpatch": "0.4.1", "license-checker": "^25.0.1", "listr": "^0.14.1", - "lmdb-store": "^1.6.11", + "lmdb-store": "^1", "loader-utils": "^2.0.4", "marge": "^1.0.1", "micromatch": "^4.0.5", diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index f8c862ee2f690..48c0fbca968ed 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -194,6 +194,7 @@ filegroup( "//packages/home/sample_data_types:build", "//packages/kbn-ace:build", "//packages/kbn-alerts:build", + "//packages/kbn-ambient-common-types:build", "//packages/kbn-ambient-storybook-types:build", "//packages/kbn-ambient-ui-types:build", "//packages/kbn-analytics:build", @@ -265,6 +266,8 @@ filegroup( "//packages/kbn-optimizer:build", "//packages/kbn-optimizer-webpack-helpers:build", "//packages/kbn-osquery-io-ts-types:build", + "//packages/kbn-peggy:build", + "//packages/kbn-peggy-loader:build", "//packages/kbn-performance-testing-dataset-extractor:build", "//packages/kbn-plugin-discovery:build", "//packages/kbn-plugin-generator:build", @@ -617,6 +620,8 @@ filegroup( "//packages/kbn-optimizer:build_types", "//packages/kbn-optimizer-webpack-helpers:build_types", "//packages/kbn-osquery-io-ts-types:build_types", + "//packages/kbn-peggy:build_types", + "//packages/kbn-peggy-loader:build_types", "//packages/kbn-performance-testing-dataset-extractor:build_types", "//packages/kbn-plugin-discovery:build_types", "//packages/kbn-plugin-generator:build_types", diff --git a/packages/kbn-ambient-common-types/BUILD.bazel b/packages/kbn-ambient-common-types/BUILD.bazel new file mode 100644 index 0000000000000..3a8b17248da22 --- /dev/null +++ b/packages/kbn-ambient-common-types/BUILD.bazel @@ -0,0 +1,58 @@ +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "pkg_npm") + +PKG_DIRNAME = "kbn-ambient-common-types" +PKG_REQUIRE_NAME = "@kbn/ambient-common-types" + +SRCS = glob( + [ + "*.d.ts", + ] +) + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +js_library( + name = PKG_DIRNAME, + srcs = SRCS + NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS, + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +alias( + name = "npm_module_types", + actual = ":" + PKG_DIRNAME, + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-ambient-common-types/README.mdx b/packages/kbn-ambient-common-types/README.mdx new file mode 100644 index 0000000000000..f08885537225c --- /dev/null +++ b/packages/kbn-ambient-common-types/README.mdx @@ -0,0 +1,20 @@ +--- +id: kibDevDocsOpsAmbientCommonTypes +slug: /kibana-dev-docs/ops/ambient-common-types +title: "@kbn/ambient-common-types" +description: A package holding ambient type definitions for files that are expected to run on the server and the browser +date: 2022-05-18 +tags: ['kibana', 'dev', 'contributor', 'operations', 'ambient', 'ui', 'common', 'server', 'types'] +--- + +This package holds ambient typescript definitions which should be included in projects which are expected to run on the server and the browser. + +## Plugins +These types will automatically be included for plugins. + +## Packages + +To include these types in a package: + +- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file. +- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file. \ No newline at end of file diff --git a/packages/kbn-ambient-common-types/index.d.ts b/packages/kbn-ambient-common-types/index.d.ts new file mode 100644 index 0000000000000..4d139c64b8cca --- /dev/null +++ b/packages/kbn-ambient-common-types/index.d.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * These ambient types are used to define default types for anything which is + * supported in both server/browser environments. + */ + +/** + * peggy grammars are built automatically on import in both browser/server + */ +declare module '*.peggy' { + export interface ParserOptions { + [key: string]: any; + /** + * Object that will be attached to the each `LocationRange` object created by + * the parser. For example, this can be path to the parsed file or even the + * File object. + */ + grammarSource?: any; + startRule?: string; + tracer?: ParserTracer; + } + + /** + * parse `input` using the peggy grammer + * @param input code to parse + * @param options parse options + */ + export function parse(input: string, options?: ParserOptions): any; +} diff --git a/packages/kbn-ambient-common-types/jest.config.js b/packages/kbn-ambient-common-types/jest.config.js new file mode 100644 index 0000000000000..f94290b64cd90 --- /dev/null +++ b/packages/kbn-ambient-common-types/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-ambient-common-types'], +}; diff --git a/packages/kbn-ambient-common-types/kibana.jsonc b/packages/kbn-ambient-common-types/kibana.jsonc new file mode 100644 index 0000000000000..70537777cc825 --- /dev/null +++ b/packages/kbn-ambient-common-types/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/ambient-common-types", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-ambient-common-types/package.json b/packages/kbn-ambient-common-types/package.json new file mode 100644 index 0000000000000..1794b046ef16e --- /dev/null +++ b/packages/kbn-ambient-common-types/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/ambient-common-types", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-ambient-common-types/tsconfig.json b/packages/kbn-ambient-common-types/tsconfig.json new file mode 100644 index 0000000000000..292157c18591a --- /dev/null +++ b/packages/kbn-ambient-common-types/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-es-query/BUILD.bazel b/packages/kbn-es-query/BUILD.bazel index 772fecdbb9806..95e7dcdcbe3cc 100644 --- a/packages/kbn-es-query/BUILD.bazel +++ b/packages/kbn-es-query/BUILD.bazel @@ -1,5 +1,4 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config") -load("@npm//peggy:index.bzl", "peggy") load("@build_bazel_rules_nodejs//:index.bzl", "js_library") load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") @@ -9,6 +8,8 @@ PKG_REQUIRE_NAME = "@kbn/es-query" SOURCE_FILES = glob( [ "**/*.ts", + "**/grammar.peggy.config.json", + "**/grammar.peggy", ], exclude = [ "**/*.config.js", @@ -51,6 +52,7 @@ RUNTIME_DEPS = [ TYPES_DEPS = [ "//packages/kbn-utility-types:npm_module_types", "//packages/kbn-i18n:npm_module_types", + "//packages/kbn-ambient-common-types:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//tslib", "@npm//@types/jest", @@ -59,26 +61,14 @@ TYPES_DEPS = [ "@npm//@types/node", ] -peggy( - name = "grammar", - data = [ - ":grammar/grammar.peggy", - ":grammar/grammar.config.json" - ], - output_dir = True, - args = [ - "--extra-options-file", - "./%s/grammar/grammar.config.json" % package_name(), - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/grammar.peggy" % package_name() - ], -) - jsts_transpiler( name = "target_node", srcs = SRCS, build_pkg_name = package_name(), + additional_args = [ + "--copy-files", + "--quiet" + ], ) jsts_transpiler( @@ -86,6 +76,10 @@ jsts_transpiler( srcs = SRCS, build_pkg_name = package_name(), web = True, + additional_args = [ + "--copy-files", + "--quiet" + ], ) ts_config( @@ -110,7 +104,7 @@ ts_project( js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], @@ -118,7 +112,7 @@ js_library( js_library( name = "npm_module_types", - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], diff --git a/packages/kbn-es-query/grammar/grammar.peggy b/packages/kbn-es-query/src/kuery/grammar/grammar.peggy similarity index 100% rename from packages/kbn-es-query/grammar/grammar.peggy rename to packages/kbn-es-query/src/kuery/grammar/grammar.peggy diff --git a/packages/kbn-es-query/grammar/grammar.config.json b/packages/kbn-es-query/src/kuery/grammar/grammar.peggy.config.json similarity index 100% rename from packages/kbn-es-query/grammar/grammar.config.json rename to packages/kbn-es-query/src/kuery/grammar/grammar.peggy.config.json diff --git a/packages/kbn-es-query/src/kuery/grammar/index.ts b/packages/kbn-es-query/src/kuery/grammar/index.ts index b05e510486c27..fa1af47631a4a 100644 --- a/packages/kbn-es-query/src/kuery/grammar/index.ts +++ b/packages/kbn-es-query/src/kuery/grammar/index.ts @@ -6,4 +6,4 @@ * Side Public License, v 1. */ -export { parse } from '../../../../grammar/built_grammar.js'; +export { parse } from './grammar.peggy'; diff --git a/packages/kbn-es-query/tsconfig.json b/packages/kbn-es-query/tsconfig.json index 292157c18591a..8561f4bdc4a31 100644 --- a/packages/kbn-es-query/tsconfig.json +++ b/packages/kbn-es-query/tsconfig.json @@ -6,7 +6,8 @@ "outDir": "target_types", "types": [ "jest", - "node" + "node", + "@kbn/ambient-common-types" ] }, "include": [ diff --git a/packages/kbn-import-resolver/src/import_resolver.ts b/packages/kbn-import-resolver/src/import_resolver.ts index 74e2ed52ca453..bab9a9000dd6c 100644 --- a/packages/kbn-import-resolver/src/import_resolver.ts +++ b/packages/kbn-import-resolver/src/import_resolver.ts @@ -125,14 +125,10 @@ export class ImportResolver { return true; } - // ignore requests to grammar/built_grammar.js files or bazel target dirs, these files are only - // available in the build output and will never resolve in dev. We will validate that people don't - // import these files from outside the package in another rule - if ( - req.endsWith('grammar/built_grammar.js') || - req.includes('/target_workers/') || - req.includes('/target_node/') - ) { + // ignore requests to bazel target dirs, these files are only available in the build output + // and will never resolve in dev. We will validate that people don't import these files from + // outside the package in another rule + if (req.includes('/target_workers/') || req.includes('/target_node/')) { return true; } diff --git a/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts b/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts index 1857e6ef26453..a6a3a84602cd7 100644 --- a/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts +++ b/packages/kbn-import-resolver/src/integration_tests/import_resolver.test.ts @@ -77,12 +77,6 @@ describe('#resolve()', () => { }); it('returns ignore results for known unresolvable but okay import statements', () => { - expect(resolver.resolve('../../grammar/built_grammar.js', FIXTURES_DIR)).toMatchInlineSnapshot(` - Object { - "type": "ignore", - } - `); - expect(resolver.resolve('kibana-buildkite-library', FIXTURES_DIR)).toMatchInlineSnapshot(` Object { "type": "ignore", diff --git a/packages/kbn-interpreter/BUILD.bazel b/packages/kbn-interpreter/BUILD.bazel index 9613a29188bd3..26a359a89cbe6 100644 --- a/packages/kbn-interpreter/BUILD.bazel +++ b/packages/kbn-interpreter/BUILD.bazel @@ -1,5 +1,4 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config") -load("@npm//peggy:index.bzl", "peggy") load("@build_bazel_rules_nodejs//:index.bzl", "js_library") load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") @@ -10,6 +9,8 @@ SOURCE_FILES = glob( [ "**/*.ts", "**/*.js", + "**/grammar.peggy.config.json", + "**/grammar.peggy", ], exclude = [ "**/*.config.js", @@ -48,12 +49,17 @@ TYPES_DEPS = [ "@npm//@types/jest", "@npm//@types/lodash", "@npm//@types/node", + "//packages/kbn-ambient-common-types:npm_module_types" ] jsts_transpiler( name = "target_node", srcs = SRCS, build_pkg_name = package_name(), + additional_args = [ + "--copy-files", + "--quiet" + ], ) jsts_transpiler( @@ -61,21 +67,9 @@ jsts_transpiler( srcs = SRCS, build_pkg_name = package_name(), web = True, -) - -peggy( - name = "grammar", - data = [ - ":grammar/grammar.config.json", - ":grammar/grammar.peggy" - ], - output_dir = True, - args = [ - "--extra-options-file", - "./%s/grammar/grammar.config.json" % package_name(), - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/grammar.peggy" % package_name() + additional_args = [ + "--copy-files", + "--quiet" ], ) @@ -102,7 +96,7 @@ ts_project( js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], @@ -110,7 +104,7 @@ js_library( js_library( name = "npm_module_types", - srcs = NPM_MODULE_EXTRA_FILES + [":grammar"], + srcs = NPM_MODULE_EXTRA_FILES, deps = RUNTIME_DEPS + [":target_node", ":target_web", ":tsc_types"], package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], diff --git a/packages/kbn-interpreter/grammar/grammar.peggy b/packages/kbn-interpreter/src/common/lib/grammar.peggy similarity index 100% rename from packages/kbn-interpreter/grammar/grammar.peggy rename to packages/kbn-interpreter/src/common/lib/grammar.peggy diff --git a/packages/kbn-interpreter/grammar/grammar.config.json b/packages/kbn-interpreter/src/common/lib/grammar.peggy.config.json similarity index 100% rename from packages/kbn-interpreter/grammar/grammar.config.json rename to packages/kbn-interpreter/src/common/lib/grammar.peggy.config.json diff --git a/packages/kbn-interpreter/src/common/lib/parse.ts b/packages/kbn-interpreter/src/common/lib/parse.ts index a6d9cc8285689..12a43e06a83c4 100644 --- a/packages/kbn-interpreter/src/common/lib/parse.ts +++ b/packages/kbn-interpreter/src/common/lib/parse.ts @@ -7,7 +7,7 @@ */ import type { Ast, AstWithMeta } from './ast'; -import { parse } from '../../../../grammar/built_grammar.js'; +import { parse } from './grammar.peggy'; interface Options { startRule?: string; diff --git a/packages/kbn-interpreter/tsconfig.json b/packages/kbn-interpreter/tsconfig.json index 57eff16f422bb..e3b4140d05822 100644 --- a/packages/kbn-interpreter/tsconfig.json +++ b/packages/kbn-interpreter/tsconfig.json @@ -6,7 +6,8 @@ "outDir": "target_types", "types": [ "jest", - "node" + "node", + "@kbn/ambient-common-types" ] }, "include": [ diff --git a/packages/kbn-optimizer/BUILD.bazel b/packages/kbn-optimizer/BUILD.bazel index 4906af1ad6f6c..da4b69f02a724 100644 --- a/packages/kbn-optimizer/BUILD.bazel +++ b/packages/kbn-optimizer/BUILD.bazel @@ -47,6 +47,7 @@ RUNTIME_DEPS = [ "//packages/kbn-ui-shared-deps-src", "//packages/kbn-utils", "//packages/kbn-synthetic-package-map", + "//packages/kbn-peggy", "@npm//@babel/core", "@npm//chalk", "@npm//clean-webpack-plugin", @@ -81,6 +82,7 @@ TYPES_DEPS = [ "//packages/kbn-utils:npm_module_types", "//packages/kbn-tooling-log:npm_module_types", "//packages/kbn-synthetic-package-map:npm_module_types", + "//packages/kbn-peggy:npm_module_types", "@npm//chalk", "@npm//clean-webpack-plugin", "@npm//cpy", diff --git a/packages/kbn-optimizer/src/node/cache.ts b/packages/kbn-optimizer/src/node/cache.ts index 1320d275022a9..a521d8e0a0f7a 100644 --- a/packages/kbn-optimizer/src/node/cache.ts +++ b/packages/kbn-optimizer/src/node/cache.ts @@ -101,28 +101,27 @@ export class Cache { } } - async update(path: string, file: { mtime: string; code: string; map: any }) { - const key = this.getKey(path); + close() { + clearTimeout(this.timer); + } + async update(path: string, file: { mtime: string; code: string; map?: any }) { + const key = this.getKey(path); await Promise.all([ this.safePut(this.atimes, key, GLOBAL_ATIME), this.safePut(this.mtimes, key, file.mtime), this.safePut(this.codes, key, file.code), - this.safePut(this.sourceMaps, key, JSON.stringify(file.map)), + file.map != null ? this.safePut(this.sourceMaps, key, JSON.stringify(file.map)) : null, ]); } - close() { - clearTimeout(this.timer); - } - private getKey(path: string) { const normalizedPath = Path.sep !== '/' ? Path.relative(this.pathRoot, path).split(Path.sep).join('/') : Path.relative(this.pathRoot, path); - return `${this.prefix}${normalizedPath}`; + return `${this.prefix}:${normalizedPath}`; } private safeGet(db: LmdbStore.Database, key: string) { diff --git a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts index 8ce58aa74214f..393abd0dc24d0 100644 --- a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts +++ b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts @@ -53,7 +53,7 @@ it('returns undefined until values are set', async () => { const log = makeTestLog(); const cache = makeCache({ dir: DIR, - prefix: 'prefix:', + prefix: 'prefix', log, pathRoot: '/foo/', }); diff --git a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts index 6aa11a3f7020f..5de38ff74cf98 100644 --- a/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts +++ b/packages/kbn-optimizer/src/node/node_auto_tranpilation.ts @@ -37,15 +37,17 @@ import Fs from 'fs'; import Path from 'path'; import Crypto from 'crypto'; -import * as babel from '@babel/core'; +import { version as babelVersion } from '@babel/core'; +import { VERSION as peggyVersion } from '@kbn/peggy'; import { addHook } from 'pirates'; import { REPO_ROOT, UPSTREAM_BRANCH } from '@kbn/utils'; import sourceMapSupport from 'source-map-support'; import { readHashOfPackageMap } from '@kbn/synthetic-package-map'; -import { Cache } from './cache'; +import { TRANSFORMS } from './transforms'; +import { getBabelOptions } from './transforms/babel'; -const cwd = process.cwd(); +import { Cache } from './cache'; const IGNORE_PATTERNS = [ /[\/\\]kbn-pm[\/\\]dist[\/\\]/, @@ -63,70 +65,6 @@ const IGNORE_PATTERNS = [ /[\/\\]packages[\/\\](eslint-|kbn-)[^\/\\]+[\/\\](?!src[\/\\].*|(.+[\/\\])?(test|__tests__)[\/\\].+|.+\.test\.(js|ts|tsx)$)(.+$)/, ]; -function getBabelOptions(path: string) { - return babel.loadOptions({ - cwd, - sourceRoot: Path.dirname(path) + Path.sep, - filename: path, - babelrc: false, - presets: [require.resolve('@kbn/babel-preset/node_preset')], - sourceMaps: 'both', - ast: false, - })!; -} - -/** - * @babel/register uses a JSON encoded copy of the config + babel.version - * as the cache key for files, so we do something similar but we don't need - * a unique cache key for every file as our config isn't different for - * different files (by design). Instead we determine a unique prefix and - * automatically prepend all paths with the prefix to create cache keys - */ -function determineCachePrefix() { - const json = JSON.stringify({ - synthPkgMapHash: readHashOfPackageMap(), - babelVersion: babel.version, - // get a config for a fake js, ts, and tsx file to make sure we - // capture conditional config portions based on the file extension - js: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.js')), - ts: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.ts')), - tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')), - }); - - const checksum = Crypto.createHash('sha256').update(json).digest('hex').slice(0, 8); - return `${checksum}:`; -} - -function compile(cache: Cache, source: string, path: string) { - try { - const mtime = `${Fs.statSync(path).mtimeMs}`; - if (cache.getMtime(path) === mtime) { - const code = cache.getCode(path); - if (code) { - // code *should* always be defined, but if it isn't for some reason rebuild it - return code; - } - } - - const options = getBabelOptions(path); - const result = babel.transform(source, options); - - if (!result || !result.code || !result.map) { - throw new Error(`babel failed to transpile [${path}]`); - } - - cache.update(path, { - mtime, - map: result.map, - code: result.code, - }); - - return result.code; - } catch (error) { - throw error; - } -} - let installed = false; export function registerNodeAutoTranspilation() { @@ -135,16 +73,45 @@ export function registerNodeAutoTranspilation() { } installed = true; + const cacheLog = process.env.DEBUG_NODE_TRANSPILER_CACHE + ? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log')) + : undefined; + + const cacheDir = Path.resolve( + REPO_ROOT, + 'data/node_auto_transpilation_cache_v5', + UPSTREAM_BRANCH + ); + + /** + * @babel/register uses a JSON encoded copy of the config + babel.version + * as the cache key for files, so we do something similar but we don't need + * a unique cache key for every file as our config isn't different for + * different files (by design). Instead we determine a unique prefix and + * automatically prepend all paths with the prefix to create cache keys + */ + const cache = new Cache({ + dir: cacheDir, + log: cacheLog, pathRoot: REPO_ROOT, - dir: Path.resolve(REPO_ROOT, 'data/node_auto_transpilation_cache_v4', UPSTREAM_BRANCH), - prefix: determineCachePrefix(), - log: process.env.DEBUG_NODE_TRANSPILER_CACHE - ? Fs.createWriteStream(Path.resolve(REPO_ROOT, 'node_auto_transpilation_cache.log'), { - flags: 'a', + prefix: Crypto.createHash('sha256') + .update( + JSON.stringify({ + synthPkgMapHash: readHashOfPackageMap(), + babelVersion, + peggyVersion, + // get a config for a fake js, ts, and tsx file to make sure we + // capture conditional config portions based on the file extension + js: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.js')), + ts: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.ts')), + tsx: getBabelOptions(Path.resolve(REPO_ROOT, 'foo.tsx')), }) - : undefined, + ) + .digest('hex') + .slice(0, 8), }); + cacheLog?.write(`cache initialized\n`); sourceMapSupport.install({ handleUncaughtExceptions: false, @@ -152,39 +119,36 @@ export function registerNodeAutoTranspilation() { // @ts-expect-error bad source-map-support types retrieveSourceMap(path: string) { const map = cache.getSourceMap(path); - - if (map) { - return { - url: null, - map, - }; - } else { - return null; - } + return map ? { map, url: null } : null; }, }); - let compiling = false; - + let transformInProgress = false; addHook( (code, path) => { - if (compiling) { + if (transformInProgress) { return code; } - if (IGNORE_PATTERNS.some((re) => re.test(path))) { + const ext = Path.extname(path); + + if (ext !== '.peggy' && IGNORE_PATTERNS.some((re) => re.test(path))) { return code; } try { - compiling = true; - return compile(cache, code, path); + transformInProgress = true; + const transform = Object.hasOwn(TRANSFORMS, ext) + ? TRANSFORMS[ext as keyof typeof TRANSFORMS] + : TRANSFORMS.default; + + return transform(path, code, cache); } finally { - compiling = false; + transformInProgress = false; } }, { - exts: ['.js', '.ts', '.tsx'], + exts: ['.js', '.ts', '.tsx', '.peggy'], ignoreNodeModules: false, } ); diff --git a/packages/kbn-optimizer/src/node/transforms/babel.ts b/packages/kbn-optimizer/src/node/transforms/babel.ts new file mode 100644 index 0000000000000..6bbe7ba67f76a --- /dev/null +++ b/packages/kbn-optimizer/src/node/transforms/babel.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; + +import * as babel from '@babel/core'; + +import { Transform } from './transform'; + +export function getBabelOptions(path: string) { + return babel.loadOptions({ + cwd: process.cwd(), + sourceRoot: Path.dirname(path) + Path.sep, + filename: path, + babelrc: false, + presets: [require.resolve('@kbn/babel-preset/node_preset')], + sourceMaps: 'both', + ast: false, + })!; +} + +export const babelTransform: Transform = (path, source, cache) => { + const mtime = `${Fs.statSync(path).mtimeMs}`; + + if (cache.getMtime(path) === mtime) { + const code = cache.getCode(path); + if (code) { + return code; + } + } + + const options = getBabelOptions(path); + const result = babel.transform(source, options); + + if (!result || !result.code || !result.map) { + throw new Error(`babel failed to transpile [${path}]`); + } + + cache.update(path, { + mtime, + code: result.code, + map: result.map, + }); + + return result.code; +}; diff --git a/packages/kbn-optimizer/src/node/transforms/index.ts b/packages/kbn-optimizer/src/node/transforms/index.ts new file mode 100644 index 0000000000000..bda2dcfa19826 --- /dev/null +++ b/packages/kbn-optimizer/src/node/transforms/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { peggyTransform } from './peggy'; +import { babelTransform } from './babel'; + +export const TRANSFORMS = { + '.peggy': peggyTransform, + default: babelTransform, +}; diff --git a/packages/kbn-optimizer/src/node/transforms/peggy.ts b/packages/kbn-optimizer/src/node/transforms/peggy.ts new file mode 100644 index 0000000000000..23edb608ef560 --- /dev/null +++ b/packages/kbn-optimizer/src/node/transforms/peggy.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Fs from 'fs'; +import Crypto from 'crypto'; + +import * as Peggy from '@kbn/peggy'; + +import { Transform } from './transform'; + +export const peggyTransform: Transform = (path, source, cache) => { + const config = Peggy.findConfigFile(path); + const mtime = `${Fs.statSync(path).mtimeMs}`; + const key = !config + ? path + : `${path}.config.${Crypto.createHash('sha256') + .update(config.source) + .digest('hex') + .slice(0, 8)}`; + + if (cache.getMtime(key) === mtime) { + const code = cache.getCode(key); + if (code) { + return code; + } + } + + const code = Peggy.getJsSourceSync({ + content: source, + path, + format: 'commonjs', + optimize: 'speed', + config, + skipConfigSearch: true, + }).source; + + cache.update(key, { + code, + mtime, + }); + + return code; +}; diff --git a/packages/kbn-interpreter/src/common/lib/grammar.d.ts b/packages/kbn-optimizer/src/node/transforms/transform.ts similarity index 75% rename from packages/kbn-interpreter/src/common/lib/grammar.d.ts rename to packages/kbn-optimizer/src/node/transforms/transform.ts index a9495dfac5b58..49c76a8c14bd9 100644 --- a/packages/kbn-interpreter/src/common/lib/grammar.d.ts +++ b/packages/kbn-optimizer/src/node/transforms/transform.ts @@ -6,6 +6,6 @@ * Side Public License, v 1. */ -declare module '*/grammar/built_grammar.js' { - export const parse: import('./parse').Parse; -} +import { Cache } from '../cache'; + +export type Transform = (path: string, source: string, cache: Cache) => string; diff --git a/packages/kbn-optimizer/src/worker/webpack.config.ts b/packages/kbn-optimizer/src/worker/webpack.config.ts index 04074fb2b10b4..888089203be4b 100644 --- a/packages/kbn-optimizer/src/worker/webpack.config.ts +++ b/packages/kbn-optimizer/src/worker/webpack.config.ts @@ -239,6 +239,10 @@ export function getWebpackConfig(bundle: Bundle, bundleRefs: BundleRefs, worker: loader: 'raw-loader', }, }, + { + test: /\.peggy$/, + loader: '@kbn/peggy-loader', + }, ], }, diff --git a/packages/kbn-peggy-loader/BUILD.bazel b/packages/kbn-peggy-loader/BUILD.bazel new file mode 100644 index 0000000000000..2d8bed8dd59a4 --- /dev/null +++ b/packages/kbn-peggy-loader/BUILD.bazel @@ -0,0 +1,128 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-peggy-loader" +PKG_REQUIRE_NAME = "@kbn/peggy-loader" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ + "//packages/kbn-peggy", + "@npm//peggy", + "@npm//webpack", +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "//packages/kbn-peggy:npm_module_types", + "@npm//@types/node", + "@npm//@types/jest", + "@npm//@types/webpack", + "@npm//peggy", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +js_library( + name = "npm_module_types", + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node", ":tsc_types"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-peggy-loader/README.md b/packages/kbn-peggy-loader/README.md new file mode 100644 index 0000000000000..2d0d587eaded3 --- /dev/null +++ b/packages/kbn-peggy-loader/README.md @@ -0,0 +1,10 @@ +--- +id: kibDevDocsOpsPeggyLoader +slug: /kibana-dev-docs/ops/peggy-loader +title: "@kbn/peggy" +description: A package which wraps @kbn/peggy for use in Webpack +date: 2022-05-18 +tags: ['kibana', 'dev', 'contributor', 'operations', 'peggy', 'loader'] +--- + +This package wraps the package so that webpack can consume it via its loader interface. This loader, like the `@kbn/peggy` package, loads config files next to grammar files for configuring the peggy parser-generator. \ No newline at end of file diff --git a/packages/kbn-peggy-loader/index.ts b/packages/kbn-peggy-loader/index.ts new file mode 100644 index 0000000000000..4fb6e7cdd8290 --- /dev/null +++ b/packages/kbn-peggy-loader/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getJsSource } from '@kbn/peggy'; +import webpack from 'webpack'; + +// eslint-disable-next-line import/no-default-export +export default function (this: webpack.loader.LoaderContext) { + this.cacheable(true); + + const callback = this.async(); + if (!callback) { + throw new Error('loader requires async support'); + } + + getJsSource({ + path: this.resourcePath, + format: 'esm', + optimize: 'size', + }).then((result) => { + if (result.config) { + this.addDependency(result.config.path); + } + + callback(null, result.source); + }, callback); +} diff --git a/packages/kbn-peggy-loader/jest.config.js b/packages/kbn-peggy-loader/jest.config.js new file mode 100644 index 0000000000000..1a481aa8c088b --- /dev/null +++ b/packages/kbn-peggy-loader/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-peggy-loader'], +}; diff --git a/packages/kbn-peggy-loader/kibana.jsonc b/packages/kbn-peggy-loader/kibana.jsonc new file mode 100644 index 0000000000000..e651946dce5eb --- /dev/null +++ b/packages/kbn-peggy-loader/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/peggy-loader", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-peggy-loader/package.json b/packages/kbn-peggy-loader/package.json new file mode 100644 index 0000000000000..6c2807a006f4a --- /dev/null +++ b/packages/kbn-peggy-loader/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/peggy-loader", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-peggy-loader/tsconfig.json b/packages/kbn-peggy-loader/tsconfig.json new file mode 100644 index 0000000000000..292157c18591a --- /dev/null +++ b/packages/kbn-peggy-loader/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-peggy/BUILD.bazel b/packages/kbn-peggy/BUILD.bazel new file mode 100644 index 0000000000000..dcb225c7da403 --- /dev/null +++ b/packages/kbn-peggy/BUILD.bazel @@ -0,0 +1,124 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-peggy" +PKG_REQUIRE_NAME = "@kbn/peggy" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ + "@npm//peggy", +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "@npm//peggy", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +js_library( + name = "npm_module_types", + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node", ":tsc_types"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-peggy/README.mdx b/packages/kbn-peggy/README.mdx new file mode 100644 index 0000000000000..5d9779976b15a --- /dev/null +++ b/packages/kbn-peggy/README.mdx @@ -0,0 +1,23 @@ +--- +id: kibDevDocsOpsPeggy +slug: /kibana-dev-docs/ops/peggy +title: "@kbn/peggy" +description: A package which wraps the peggy library for use in Kibana +date: 2022-05-18 +tags: ['kibana', 'dev', 'contributor', 'operations', 'peggy'] +--- + +This package wraps the peggy package, exposing a synchronous and async version of the generator which includes two modifications: + + 1. When a `path` is provided a `${basename}.config.json` file will be loaded if it exists and is expected to include peggy config options as defined in [the peggy docs](https://peggyjs.org/documentation.html#generating-a-parser-javascript-api). This config will be used when compiling this file + +## Plugins +These types will automatically be included for plugins. + +## Packages + +To include these types in a package: + +- add `"//packages/kbn-ambient-ui-types"` to the `RUNTIME_DEPS` portion of the `BUILD.bazel` file. +- add `"//packages/kbn-ambient-ui-types:npm_module_types"` to the `TYPES_DEPS` portion of the `BUILD.bazel` file. +- add `"@kbn/ambient-ui-types"` to the `types` portion of the `tsconfig.json` file. \ No newline at end of file diff --git a/packages/kbn-peggy/index.ts b/packages/kbn-peggy/index.ts new file mode 100644 index 0000000000000..b5b35f131d2ee --- /dev/null +++ b/packages/kbn-peggy/index.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; +import Fsp from 'fs/promises'; + +import Peggy from 'peggy'; + +export interface Options { + /** + * The path to the peggy content. If this is not defined then + * config files can not be found and `content` must be passed. + */ + path?: string; + /** + * Prevent loading the content from disk by specifying it here + */ + content?: string; + /** + * Prevent loading the config from disk by specifying it here + */ + config?: Config; + /** + * What type of module format should the generated code use. Defaults to + * commonjs for broadest compatibility + */ + format?: 'esm' | 'commonjs'; + /** + * Should the parser optimize for execution speed or size of the code + */ + optimize?: 'size' | 'speed'; + /** + * Disable checking for a config file a `{basename}.config.json` in + * the same directory as the grammar file. + */ + skipConfigSearch?: boolean; +} + +export interface Config { + /** the path of the discovered config file */ + path: string; + /** the content of the config file as a string (primarily for hashing) */ + source: string; + /** the parsed content of the config file */ + parsed: any; +} + +export interface Result { + /** + * The source code of the module which parses expressions in the format + * defined by the peggy grammar file + */ + config: Config | null; + + /** + * The loaded config if it was found + */ + source: string; +} + +export function findConfigFile(grammarPath: string): Config | undefined { + const path = Path.resolve(Path.dirname(grammarPath), `${Path.basename(grammarPath)}.config.json`); + + let source; + let parsed; + try { + source = Fs.readFileSync(path, 'utf8'); + parsed = JSON.parse(source); + } catch (error) { + if (error.code === 'ENOENT') { + return undefined; + } + + throw error; + } + + return { path, source, parsed }; +} + +export async function getJsSource(options: Options): Promise { + let source; + if (options.content) { + source = options.content; + } else if (options.path) { + source = await Fsp.readFile(options.path, 'utf8'); + } else { + throw new Error('you must either specify the path of the grammar file, or the content'); + } + + return getJsSourceSync({ + content: source, + ...options, + }); +} + +export function getJsSourceSync( + options: Options & { + /** The content of the grammar file to parse */ + content: string; + } +): Result { + const config = + options.config ?? + (options.path && options.skipConfigSearch !== true ? findConfigFile(options.path) : null); + + const result = Peggy.generate(options.content, { + ...config?.parsed, + format: options.format === 'esm' ? 'es' : 'commonjs', + optimize: options.optimize, + output: 'source', + }); + + return { + /** + * The source code of the module which parses expressions in the format + * defined by the peggy grammar file + */ + source: result as unknown as string, + + /** + * The loaded config if it was found + */ + config: config ?? null, + }; +} + +export const VERSION = Peggy.VERSION; diff --git a/packages/kbn-peggy/jest.config.js b/packages/kbn-peggy/jest.config.js new file mode 100644 index 0000000000000..be8ef99a01b3d --- /dev/null +++ b/packages/kbn-peggy/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-peggy'], +}; diff --git a/packages/kbn-peggy/kibana.jsonc b/packages/kbn-peggy/kibana.jsonc new file mode 100644 index 0000000000000..8b44dc5604d19 --- /dev/null +++ b/packages/kbn-peggy/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/peggy", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-peggy/package.json b/packages/kbn-peggy/package.json new file mode 100644 index 0000000000000..cd976c2e8d97b --- /dev/null +++ b/packages/kbn-peggy/package.json @@ -0,0 +1,8 @@ +{ + "name": "@kbn/peggy", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "types": "./target_types/index.d.ts", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-peggy/tsconfig.json b/packages/kbn-peggy/tsconfig.json new file mode 100644 index 0000000000000..292157c18591a --- /dev/null +++ b/packages/kbn-peggy/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-storybook/src/webpack.config.ts b/packages/kbn-storybook/src/webpack.config.ts index 3d00fb88a089a..cca984fbe83b5 100644 --- a/packages/kbn-storybook/src/webpack.config.ts +++ b/packages/kbn-storybook/src/webpack.config.ts @@ -82,6 +82,12 @@ export default ({ config: storybookConfig }: { config: Configuration }) => { loader: 'raw-loader', }, }, + { + test: /\.peggy$/, + use: { + loader: '@kbn/peggy-loader', + }, + }, { test: /\.scss$/, exclude: /\.module.(s(a|c)ss)$/, diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index 56b3829fc77a5..425ef3864406f 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -100,6 +100,7 @@ TYPES_DEPS = [ "//packages/kbn-bazel-packages:npm_module_types", "//packages/kbn-get-repo-files:npm_module_types", "//packages/kbn-ftr-screenshot-filename:npm_module_types", + "//packages/kbn-peggy:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//@jest/console", "@npm//@jest/reporters", @@ -119,6 +120,7 @@ TYPES_DEPS = [ "@npm//rxjs", "@npm//playwright", "@npm//xmlbuilder", + "@npm//@jest/transform", "@npm//@types/archiver", "@npm//@types/chance", "@npm//@types/dedent", diff --git a/packages/kbn-test/jest-preset.js b/packages/kbn-test/jest-preset.js index 2c4b3a960e59c..7eb6c8377ebe2 100644 --- a/packages/kbn-test/jest-preset.js +++ b/packages/kbn-test/jest-preset.js @@ -120,9 +120,9 @@ module.exports = { // A map from regular expressions to paths to transformers transform: { - '^.+\\.(js|tsx?)$': '/node_modules/@kbn/test/target_node/src/jest/babel_transform.js', - '^.+\\.txt?$': '/node_modules/@kbn/test/target_node/src/jest/raw_transform.js', - '^.+\\.html?$': '/node_modules/@kbn/test/target_node/src/jest/raw_transform.js', + '^.+\\.(js|tsx?)$': '/node_modules/@kbn/test/target_node/src/jest/transforms/babel.js', + '^.+\\.(txt|html)?$': '/node_modules/@kbn/test/target_node/src/jest/transforms/raw.js', + '^.+\\.peggy?$': '/node_modules/@kbn/test/target_node/src/jest/transforms/peggy.js', }, // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation diff --git a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts index 976a2c417c747..ccdd9cc902c5b 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts @@ -38,6 +38,10 @@ export class EsVersion { this.parsed = parsed; } + toJSON() { + return this.toString(); + } + toString() { return this.parsed.version; } diff --git a/packages/kbn-test/src/jest/babel_transform.js b/packages/kbn-test/src/jest/transforms/babel.js similarity index 100% rename from packages/kbn-test/src/jest/babel_transform.js rename to packages/kbn-test/src/jest/transforms/babel.js diff --git a/packages/kbn-test/src/jest/transforms/peggy.js b/packages/kbn-test/src/jest/transforms/peggy.js new file mode 100644 index 0000000000000..ec32d03150b53 --- /dev/null +++ b/packages/kbn-test/src/jest/transforms/peggy.js @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +const Peggy = require('@kbn/peggy'); +const Crypto = require('crypto'); + +/** @type {import('@jest/transform').AsyncTransformer} */ +module.exports = { + canInstrument: false, + + getCacheKey(sourceText, sourcePath) { + const config = Peggy.findConfigFile(sourcePath); + return ( + Crypto.createHash('sha256').update(sourceText).digest('hex') + + (!config ? '' : `-${Crypto.createHash('sha256').update(config.source).digest('hex')}`) + ); + }, + + process(sourceText, sourcePath) { + return { + code: Peggy.getJsSourceSync({ + content: sourceText, + path: sourcePath, + format: 'commonjs', + optimize: 'speed', + }).source, + }; + }, +}; diff --git a/packages/kbn-test/src/jest/raw_transform.js b/packages/kbn-test/src/jest/transforms/raw.js similarity index 100% rename from packages/kbn-test/src/jest/raw_transform.js rename to packages/kbn-test/src/jest/transforms/raw.js diff --git a/packages/kbn-timelion-grammar/BUILD.bazel b/packages/kbn-timelion-grammar/BUILD.bazel index 3c7ea13cadf64..7898fef88f1fc 100644 --- a/packages/kbn-timelion-grammar/BUILD.bazel +++ b/packages/kbn-timelion-grammar/BUILD.bazel @@ -1,32 +1,18 @@ load("@build_bazel_rules_nodejs//:index.bzl", "js_library") -load("@npm//peggy:index.bzl", "peggy") load("//src/dev/bazel:index.bzl", "pkg_npm") PKG_DIRNAME = "kbn-timelion-grammar" PKG_REQUIRE_NAME = "@kbn/timelion-grammar" NPM_MODULE_EXTRA_FILES = [ + "index.js", + "chain.peggy", "package.json", ] -peggy( - name = "grammar", - data = [ - ":grammar/chain.peggy" - ], - output_dir = True, - args = [ - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/chain.peggy" % package_name() - ], -) - js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [ - ":grammar" - ], + srcs = NPM_MODULE_EXTRA_FILES, package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], ) diff --git a/packages/kbn-timelion-grammar/grammar/chain.peggy b/packages/kbn-timelion-grammar/chain.peggy similarity index 100% rename from packages/kbn-timelion-grammar/grammar/chain.peggy rename to packages/kbn-timelion-grammar/chain.peggy diff --git a/packages/kbn-es-query/src/kuery/grammar/grammar.d.ts b/packages/kbn-timelion-grammar/index.js similarity index 82% rename from packages/kbn-es-query/src/kuery/grammar/grammar.d.ts rename to packages/kbn-timelion-grammar/index.js index 245f6839d7d04..6dd2fdb6abea7 100644 --- a/packages/kbn-es-query/src/kuery/grammar/grammar.d.ts +++ b/packages/kbn-timelion-grammar/index.js @@ -6,6 +6,4 @@ * Side Public License, v 1. */ -declare module '*/grammar/built_grammar.js' { - export const parse: any; -} +module.exports = require('./chain.peggy'); diff --git a/packages/kbn-timelion-grammar/package.json b/packages/kbn-timelion-grammar/package.json index 4676d1e6b94d9..5b519870c423d 100644 --- a/packages/kbn-timelion-grammar/package.json +++ b/packages/kbn-timelion-grammar/package.json @@ -2,6 +2,5 @@ "name": "@kbn/timelion-grammar", "version": "1.0.0", "license": "SSPL-1.0 OR Elastic License 2.0", - "private": true, - "main": "grammar/built_grammar.js" + "private": true } \ No newline at end of file diff --git a/packages/kbn-tinymath/BUILD.bazel b/packages/kbn-tinymath/BUILD.bazel index b8ee25a4973b8..b9f1fb9daf849 100644 --- a/packages/kbn-tinymath/BUILD.bazel +++ b/packages/kbn-tinymath/BUILD.bazel @@ -1,5 +1,4 @@ load("@build_bazel_rules_nodejs//:index.bzl", "js_library") -load("@npm//peggy:index.bzl", "peggy") load("//src/dev/bazel:index.bzl", "pkg_npm") PKG_DIRNAME = "kbn-tinymath" @@ -31,25 +30,9 @@ RUNTIME_DEPS = [ "@npm//lodash", ] -peggy( - name = "grammar", - data = [ - ":grammar/grammar.peggy" - ], - output_dir = True, - args = [ - "-o", - "$(@D)/built_grammar.js", - "./%s/grammar/grammar.peggy" % package_name() - ], -) - js_library( name = PKG_DIRNAME, - srcs = NPM_MODULE_EXTRA_FILES + [ - ":srcs", - ":grammar" - ], + srcs = NPM_MODULE_EXTRA_FILES + [":srcs"], deps = RUNTIME_DEPS, package_name = PKG_REQUIRE_NAME, visibility = ["//visibility:public"], diff --git a/packages/kbn-tinymath/grammar/grammar.peggy b/packages/kbn-tinymath/src/grammar.peggy similarity index 100% rename from packages/kbn-tinymath/grammar/grammar.peggy rename to packages/kbn-tinymath/src/grammar.peggy diff --git a/packages/kbn-tinymath/src/index.js b/packages/kbn-tinymath/src/index.js index dd23b0847c999..5bc7d80faff9e 100644 --- a/packages/kbn-tinymath/src/index.js +++ b/packages/kbn-tinymath/src/index.js @@ -9,7 +9,7 @@ const { get } = require('lodash'); const memoizeOne = require('memoize-one'); const { functions: includedFunctions } = require('./functions'); -const { parse: parseFn } = require('../grammar/built_grammar.js'); +const { parse: parseFn } = require('./grammar.peggy'); function parse(input, options) { if (input == null) { diff --git a/packages/kbn-ui-shared-deps-src/BUILD.bazel b/packages/kbn-ui-shared-deps-src/BUILD.bazel index 97006c36eb285..91d64b9159be9 100644 --- a/packages/kbn-ui-shared-deps-src/BUILD.bazel +++ b/packages/kbn-ui-shared-deps-src/BUILD.bazel @@ -50,6 +50,7 @@ RUNTIME_DEPS = [ "//packages/kbn-std", "//packages/kbn-ui-shared-deps-npm", "//packages/kbn-ui-theme", + "//packages/kbn-peggy-loader", ] TYPES_DEPS = [ diff --git a/packages/kbn-ui-shared-deps-src/webpack.config.js b/packages/kbn-ui-shared-deps-src/webpack.config.js index 85d14314c5c60..bebe6eb424f06 100644 --- a/packages/kbn-ui-shared-deps-src/webpack.config.js +++ b/packages/kbn-ui-shared-deps-src/webpack.config.js @@ -56,6 +56,10 @@ module.exports = { }, ], }, + { + test: /\.peggy$/, + use: ['@kbn/peggy-loader'], + }, { test: /\.css$/, use: [MiniCssExtractPlugin.loader, 'css-loader'], diff --git a/src/dev/build/lib/integration_tests/scan_copy.test.ts b/src/dev/build/lib/integration_tests/scan_copy.test.ts index dd69ad78789a7..464821707dbab 100644 --- a/src/dev/build/lib/integration_tests/scan_copy.test.ts +++ b/src/dev/build/lib/integration_tests/scan_copy.test.ts @@ -89,7 +89,7 @@ it('applies filter function specified', async () => { await scanCopy({ source: FIXTURES, destination, - filter: (record) => !record.name.includes('bar'), + filter: (record) => !record.source.name.includes('bar'), }); expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([ diff --git a/src/dev/build/lib/scan_copy.ts b/src/dev/build/lib/scan_copy.ts index d8285e1853173..1c5e29f420588 100644 --- a/src/dev/build/lib/scan_copy.ts +++ b/src/dev/build/lib/scan_copy.ts @@ -9,11 +9,16 @@ import Fs from 'fs'; import Fsp from 'fs/promises'; import Path from 'path'; - -import { asyncMap, asyncForEach } from '@kbn/std'; +import * as Rx from 'rxjs'; import { assertAbsolute, mkdirp } from './fs'; +const fsReadDir$ = Rx.bindNodeCallback( + (path: string, cb: (err: Error | null, ents: Fs.Dirent[]) => void) => { + Fs.readdir(path, { withFileTypes: true }, cb); + } +); + interface Options { /** * directory to copy from @@ -26,75 +31,158 @@ interface Options { /** * function that is called with each Record */ - filter?: (record: Record) => boolean; + filter?: (record: Readonly) => boolean; /** * define permissions for reach item copied */ - permissions?: (record: Record) => number | undefined; + permissions?: (record: Readonly) => number | undefined; /** * Date to use for atime/mtime */ time?: Date; + /** + * + */ + map?: (record: Readonly) => Promise; } -class Record { +export class SomePath { + static fromAbs(path: string) { + return new SomePath(Path.dirname(path), Path.basename(path)); + } + constructor( - public isDirectory: boolean, - public name: string, - public absolute: string, - public absoluteDest: string + /** The directory of the item at this path */ + public readonly dir: string, + /** The name of the item at this path */ + public readonly name: string ) {} + + private _abs: string | null = null; + /** The absolute path of the file */ + public get abs() { + if (this._abs === null) { + this._abs = Path.resolve(this.dir, this.name); + } + + return this._abs; + } + + private _ext: string | null = null; + /** The extension of the filename, starts with a . like the Path.extname API */ + public get ext() { + if (this._ext === null) { + this._ext = Path.extname(this.name); + } + + return this._ext; + } + + /** return a file path with the file name changed to `name` */ + withName(name: string) { + return new SomePath(this.dir, name); + } + + /** return a file path with the file extension changed to `extension` */ + withExt(extension: string) { + return new SomePath(this.dir, Path.basename(this.name, this.ext) + extension); + } + + child(childName: string) { + return new SomePath(this.abs, childName); + } +} + +interface DirRecord { + type: 'dir'; + source: SomePath; + dest: SomePath; +} + +interface FileRecord { + type: 'file'; + source: SomePath; + dest: SomePath; + content?: string; } +type Record = FileRecord | DirRecord; + /** * Copy all of the files from one directory to another, optionally filtered with a * function or modifying mtime/atime for each file. */ export async function scanCopy(options: Options) { - const { source, destination, filter, time, permissions } = options; + const { source, destination, filter, time, permissions, map } = options; assertAbsolute(source); assertAbsolute(destination); - // create or copy each child of a directory - const copyChildren = async (parent: Record) => { - const names = await Fsp.readdir(parent.absolute); + /** + * recursively fetch all the file records within a directory, starting with the + * files in the passed directory, then the files in all the child directories in + * no particular order + */ + const readDir$ = (dir: DirRecord): Rx.Observable => + fsReadDir$(dir.source.abs).pipe( + Rx.mergeAll(), + Rx.mergeMap((ent) => { + const rec: Record = { + type: ent.isDirectory() ? 'dir' : 'file', + source: dir.source.child(ent.name), + dest: dir.dest.child(ent.name), + }; - const records = await asyncMap(names, async (name) => { - const absolute = Path.join(parent.absolute, name); - const stat = await Fsp.stat(absolute); - return new Record(stat.isDirectory(), name, absolute, Path.join(parent.absoluteDest, name)); - }); + if (filter && !filter(rec)) { + return Rx.EMPTY; + } + + return Rx.of(rec); + }) + ); - await asyncForEach(records, async (rec) => { - if (filter && !filter(rec)) { - return; + const handleGenericRec = async (rec: Record) => { + if (permissions) { + const perm = permissions(rec); + if (perm !== undefined) { + await Fsp.chmod(rec.dest.abs, perm); } + } + + if (time) { + await Fsp.utimes(rec.dest.abs, time, time); + } + }; - if (rec.isDirectory) { - await Fsp.mkdir(rec.absoluteDest, { - mode: permissions ? permissions(rec) : undefined, + const handleDir$ = (rec: DirRecord): Rx.Observable => + Rx.defer(async () => { + await mkdirp(rec.dest.abs); + await handleGenericRec(rec); + }).pipe( + Rx.mergeMap(() => readDir$(rec)), + Rx.mergeMap((ent) => (ent.type === 'dir' ? handleDir$(ent) : handleFile$(ent))) + ); + + const handleFile$ = (srcRec: FileRecord): Rx.Observable => + Rx.defer(async () => { + const rec = (map && (await map(srcRec))) ?? srcRec; + + if (rec.content) { + await Fsp.writeFile(rec.dest.abs, rec.content, { + flag: 'wx', }); } else { - await Fsp.copyFile(rec.absolute, rec.absoluteDest, Fs.constants.COPYFILE_EXCL); - if (permissions) { - const perm = permissions(rec); - if (perm !== undefined) { - await Fsp.chmod(rec.absoluteDest, perm); - } - } + await Fsp.copyFile(rec.source.abs, rec.dest.abs, Fs.constants.COPYFILE_EXCL); } - if (time) { - await Fsp.utimes(rec.absoluteDest, time, time); - } - - if (rec.isDirectory) { - await copyChildren(rec); - } + await handleGenericRec(rec); }); - }; - await mkdirp(destination); - await copyChildren(new Record(true, Path.basename(source), source, destination)); + await Rx.lastValueFrom( + handleDir$({ + type: 'dir', + source: SomePath.fromAbs(source), + dest: SomePath.fromAbs(destination), + }) + ); } diff --git a/src/dev/build/tasks/build_packages_task.ts b/src/dev/build/tasks/build_packages_task.ts index fdb32731fdd8e..2cb0bb585b56a 100644 --- a/src/dev/build/tasks/build_packages_task.ts +++ b/src/dev/build/tasks/build_packages_task.ts @@ -11,8 +11,9 @@ import Path from 'path'; import { REPO_ROOT } from '@kbn/utils'; import { discoverBazelPackages } from '@kbn/bazel-packages'; import { runBazel } from '@kbn/bazel-runner'; +import * as Peggy from '@kbn/peggy'; -import { Task, scanCopy, write } from '../lib'; +import { Task, scanCopy, write, deleteAll } from '../lib'; export const BuildBazelPackages: Task = { description: 'Building distributable versions of Bazel packages', @@ -26,16 +27,54 @@ export const BuildBazelPackages: Task = { log.info(`Copying build of`, pkg.manifest.id, 'into build'); const pkgDirInBuild = build.resolvePath(pkg.normalizedRepoRelativeDir); + const peggyConfigOutputPaths = new Set(); + const pkgBuildDir = config.resolveFromRepo( + 'bazel-bin', + pkg.normalizedRepoRelativeDir, + 'npm_module' + ); // copy the built npm_module target dir into the build, package.json is updated to copy // the sources we actually end up using into the node_modules directory when we run // yarn install await scanCopy({ - source: config.resolveFromRepo('bazel-bin', pkg.normalizedRepoRelativeDir, 'npm_module'), + source: pkgBuildDir, destination: pkgDirInBuild, - permissions: (rec) => (rec.isDirectory ? 0o755 : 0o644), + permissions: (rec) => (rec.type === 'file' ? 0o644 : 0o755), + filter: (rec) => !(rec.type === 'dir' && rec.source.name === 'target_web'), + async map(rec) { + const extname = Path.extname(rec.source.name); + if (extname !== '.peggy') { + return undefined; + } + + const result = await Peggy.getJsSource({ + path: rec.source.abs, + format: 'commonjs', + optimize: 'speed', + }); + + if (result.config) { + // if there was a config file for this peggy grammar, capture its output path and + // delete it after the copy is complete + peggyConfigOutputPaths.add( + Path.resolve(pkgDirInBuild, Path.relative(pkgBuildDir, result.config.path)) + ); + } + + return { + ...rec, + dest: rec.dest.withName(rec.dest.name + '.js'), + content: result.source, + }; + }, }); + // cleanup any peggy config files + if (peggyConfigOutputPaths.size) { + await deleteAll(Array.from(peggyConfigOutputPaths), log); + } + await write( Path.resolve(pkgDirInBuild, 'kibana.jsonc'), JSON.stringify(pkg.manifest, null, 2) diff --git a/tsconfig.base.json b/tsconfig.base.json index 12bbf2ea85736..7b0a8e6781baf 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -376,6 +376,8 @@ "@kbn/ace/*": ["packages/kbn-ace/*"], "@kbn/alerts": ["packages/kbn-alerts"], "@kbn/alerts/*": ["packages/kbn-alerts/*"], + "@kbn/ambient-common-types": ["packages/kbn-ambient-common-types"], + "@kbn/ambient-common-types/*": ["packages/kbn-ambient-common-types/*"], "@kbn/ambient-storybook-types": ["packages/kbn-ambient-storybook-types"], "@kbn/ambient-storybook-types/*": ["packages/kbn-ambient-storybook-types/*"], "@kbn/ambient-ui-types": ["packages/kbn-ambient-ui-types"], @@ -518,6 +520,10 @@ "@kbn/optimizer-webpack-helpers/*": ["packages/kbn-optimizer-webpack-helpers/*"], "@kbn/osquery-io-ts-types": ["packages/kbn-osquery-io-ts-types"], "@kbn/osquery-io-ts-types/*": ["packages/kbn-osquery-io-ts-types/*"], + "@kbn/peggy": ["packages/kbn-peggy"], + "@kbn/peggy/*": ["packages/kbn-peggy/*"], + "@kbn/peggy-loader": ["packages/kbn-peggy-loader"], + "@kbn/peggy-loader/*": ["packages/kbn-peggy-loader/*"], "@kbn/performance-testing-dataset-extractor": ["packages/kbn-performance-testing-dataset-extractor"], "@kbn/performance-testing-dataset-extractor/*": ["packages/kbn-performance-testing-dataset-extractor/*"], "@kbn/plugin-discovery": ["packages/kbn-plugin-discovery"], @@ -1276,6 +1282,7 @@ "@testing-library/jest-dom", "@emotion/react/types/css-prop", "@kbn/ambient-ui-types", + "@kbn/ambient-common-types", "@kbn/ambient-storybook-types" ] }, diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js index 24ae9b3faa1f1..2e69f52bdf651 100644 --- a/x-pack/plugins/canvas/shareable_runtime/webpack.config.js +++ b/x-pack/plugins/canvas/shareable_runtime/webpack.config.js @@ -187,6 +187,10 @@ module.exports = { ], use: require.resolve('null-loader'), }, + { + test: /\.peggy$/, + use: require.resolve('@kbn/peggy-loader'), + }, ], }, node: { diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts b/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts index fb1e93ddbe956..a0f6c8fe0e9bb 100644 --- a/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts +++ b/x-pack/plugins/canvas/shareable_runtime/webpack/ci_stats_plugin.ts @@ -11,11 +11,8 @@ import Path from 'path'; import webpack from 'webpack'; import { ToolingLog } from '@kbn/tooling-log'; -import { REPO_ROOT } from '@kbn/utils'; -import normalizePath from 'normalize-path'; import { CiStatsReporter } from '@kbn/ci-stats-reporter'; import { isNormalModule, isConcatenatedModule } from '@kbn/optimizer-webpack-helpers'; -import { RUNTIME_SIZE_LIMIT } from './runtime_size_limit'; const IGNORED_EXTNAME = ['.map', '.br', '.gz']; @@ -91,10 +88,6 @@ export class CiStatsPlugin { group: `canvas shareable runtime`, id: 'total size', value: entry.size, - limit: RUNTIME_SIZE_LIMIT, - limitConfigPath: normalizePath( - Path.relative(REPO_ROOT, require.resolve('./runtime_size_limit')) - ), }, { group: `canvas shareable runtime`, diff --git a/x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts b/x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts deleted file mode 100644 index 51b16b6ccbd52..0000000000000 --- a/x-pack/plugins/canvas/shareable_runtime/webpack/runtime_size_limit.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const RUNTIME_SIZE_LIMIT = 8_200_000; diff --git a/yarn.lock b/yarn.lock index 9052863b64a9e..f81e0fcfc311c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2705,6 +2705,10 @@ version "0.0.0" uid "" +"@kbn/ambient-common-types@link:bazel-bin/packages/kbn-ambient-common-types": + version "0.0.0" + uid "" + "@kbn/ambient-storybook-types@link:bazel-bin/packages/kbn-ambient-storybook-types": version "0.0.0" uid "" @@ -3741,6 +3745,14 @@ version "0.0.0" uid "" +"@kbn/peggy-loader@link:bazel-bin/packages/kbn-peggy-loader": + version "0.0.0" + uid "" + +"@kbn/peggy@link:bazel-bin/packages/kbn-peggy": + version "0.0.0" + uid "" + "@kbn/performance-testing-dataset-extractor@link:bazel-bin/packages/kbn-performance-testing-dataset-extractor": version "0.0.0" uid "" @@ -4399,6 +4411,36 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@msgpackr-extract/msgpackr-extract-darwin-arm64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.2.0.tgz#901c5937e1441572ea23e631fe6deca68482fe76" + integrity sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ== + +"@msgpackr-extract/msgpackr-extract-darwin-x64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.2.0.tgz#fb877fe6bae3c4d3cea29786737840e2ae689066" + integrity sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw== + +"@msgpackr-extract/msgpackr-extract-linux-arm64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.2.0.tgz#986179c38b10ac41fbdaf7d036c825cbc72855d9" + integrity sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA== + +"@msgpackr-extract/msgpackr-extract-linux-arm@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.2.0.tgz#15f2c6fe9e0adc06c21af7e95f484ff4880d79ce" + integrity sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg== + +"@msgpackr-extract/msgpackr-extract-linux-x64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.2.0.tgz#30cae5c9a202f3e1fa1deb3191b18ffcb2f239a2" + integrity sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw== + +"@msgpackr-extract/msgpackr-extract-win32-x64@2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz#016d855b6bc459fd908095811f6826e45dd4ba64" + integrity sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA== + "@nicolo-ribaudo/chokidar-2@2.1.8-no-fsevents.3": version "2.1.8-no-fsevents.3" resolved "https://registry.yarnpkg.com/@nicolo-ribaudo/chokidar-2/-/chokidar-2-2.1.8-no-fsevents.3.tgz#323d72dd25103d0c4fbdce89dadf574a787b1f9b" @@ -18425,7 +18467,7 @@ listr@^0.14.1: p-map "^2.0.0" rxjs "^6.3.3" -lmdb-store@^1.6.11: +lmdb-store@^1: version "1.6.11" resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d" integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q== @@ -19781,20 +19823,26 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -msgpackr-extract@^1.0.14: - version "1.0.14" - resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.0.14.tgz#87d3fe825d226e7f3d9fe136375091137f958561" - integrity sha512-t8neMf53jNZRF+f0H9VvEUVvtjGZ21odSBRmFfjZiyxr9lKYY0mpY3kSWZAIc7YWXtCZGOvDQVx2oqcgGiRBrw== +msgpackr-extract@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262" + integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog== dependencies: - nan "^2.14.2" - node-gyp-build "^4.2.3" + node-gyp-build-optional-packages "5.0.3" + optionalDependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-arm" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0" msgpackr@^1.4.7: - version "1.4.7" - resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.4.7.tgz#d802ade841e7d2e873000b491cdda6574a3d5748" - integrity sha512-bhC8Ed1au3L3oHaR/fe4lk4w7PLGFcWQ5XY/Tk9N6tzDRz8YndjCG68TD8zcvYZoxNtw767eF/7VpaTpU9kf9w== + version "1.8.0" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.8.0.tgz#6cf213e88f04c5a358c61085a42a4dbe5542de44" + integrity sha512-1Cos3r86XACdjLVY4CN8r72Cgs5lUzxSON6yb81sNZP9vC9nnBrEbu1/ldBhuR9BKejtoYV5C9UhmYUvZFJSNQ== optionalDependencies: - msgpackr-extract "^1.0.14" + msgpackr-extract "^2.2.0" multicast-dns@^7.2.5: version "7.2.5" @@ -20670,9 +20718,9 @@ ora@^5.4.1: wcwidth "^1.0.1" ordered-binary@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.0.0.tgz#4f7485186b12aa42b99011aeb7aa272991d6a487" - integrity sha512-0RMlzqix3YAOZKMoXv97OIvHlqJxnmIzihjShVkYNV3JuzHbqeBOOP7wpz6yo4af1ZFnOHGsh8RK77ZmaBY3Lg== + version "1.4.0" + resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389" + integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ== ordered-read-streams@^1.0.0: version "1.0.1" @@ -27418,9 +27466,9 @@ wcwidth@^1.0.1: defaults "^1.0.3" weak-lru-cache@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.0.0.tgz#f1394721169883488c554703704fbd91cda05ddf" - integrity sha512-135bPugHHIJLNx20guHgk4etZAbd7nou34NQfdKkJPgMuC3Oqn4cT6f7ORVvnud9oEyXJVJXPcTFsUvttGm5xg== + version "1.2.2" + resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19" + integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw== web-namespaces@^1.0.0: version "1.1.4"