[kbn/pm] reduce responsibilities (#130592)

* [kbn/pm] reduce responsibilities

* [CI] Auto-commit changed files from 'node scripts/generate packages_build_manifest'

* [CI] Auto-commit changed files from 'yarn kbn run build -i @kbn/pm'

Co-authored-by: kibanamachine <[email protected]>

.github/CODEOWNERS

@@ -220,6 +220,7 @@
 /packages/kbn-ambient-ui-types/ @elastic/kibana-operations
 /packages/kbn-ambient-storybook-types/ @elastic/kibana-operations
 /packages/kbn-bazel-packages/ @elastic/kibana-operations
+/packages/kbn-bazel-runner/ @elastic/kibana-operations
 /packages/kbn-cli-dev-mode/ @elastic/kibana-operations
 /packages/kbn-dev-utils*/ @elastic/kibana-operations
 /packages/kbn-es-archiver/ @elastic/kibana-operations

package.json

@@ -481,6 +481,7 @@
     "@kbn/babel-plugin-synthetic-packages": "link:bazel-bin/packages/kbn-babel-plugin-synthetic-packages",
     "@kbn/babel-preset": "link:bazel-bin/packages/kbn-babel-preset",
     "@kbn/bazel-packages": "link:bazel-bin/packages/kbn-bazel-packages",
+    "@kbn/bazel-runner": "link:bazel-bin/packages/kbn-bazel-runner",
     "@kbn/ci-stats-client": "link:bazel-bin/packages/kbn-ci-stats-client",
     "@kbn/ci-stats-core": "link:bazel-bin/packages/kbn-ci-stats-core",
     "@kbn/ci-stats-reporter": "link:bazel-bin/packages/kbn-ci-stats-reporter",
@@ -607,6 +608,7 @@
     "@types/kbn__apm-utils": "link:bazel-bin/packages/kbn-apm-utils/npm_module_types",
     "@types/kbn__axe-config": "link:bazel-bin/packages/kbn-axe-config/npm_module_types",
     "@types/kbn__bazel-packages": "link:bazel-bin/packages/kbn-bazel-packages/npm_module_types",
+    "@types/kbn__bazel-runner": "link:bazel-bin/packages/kbn-bazel-runner/npm_module_types",
     "@types/kbn__ci-stats-client": "link:bazel-bin/packages/kbn-ci-stats-client/npm_module_types",
     "@types/kbn__ci-stats-core": "link:bazel-bin/packages/kbn-ci-stats-core/npm_module_types",
     "@types/kbn__ci-stats-reporter": "link:bazel-bin/packages/kbn-ci-stats-reporter/npm_module_types",

packages/BUILD.bazel

@@ -24,6 +24,7 @@ filegroup(
     "//packages/kbn-babel-plugin-synthetic-packages:build",
     "//packages/kbn-babel-preset:build",
     "//packages/kbn-bazel-packages:build",
+    "//packages/kbn-bazel-runner:build",
     "//packages/kbn-ci-stats-client:build",
     "//packages/kbn-ci-stats-core:build",
     "//packages/kbn-ci-stats-reporter:build",
@@ -121,6 +122,7 @@ filegroup(
     "//packages/kbn-apm-utils:build_types",
     "//packages/kbn-axe-config:build_types",
     "//packages/kbn-bazel-packages:build_types",
+    "//packages/kbn-bazel-runner:build_types",
     "//packages/kbn-ci-stats-client:build_types",
     "//packages/kbn-ci-stats-core:build_types",
     "//packages/kbn-ci-stats-reporter:build_types",

packages/kbn-bazel-packages/src/bazel_package.ts

@@ -70,6 +70,13 @@ export class BazelPackage {
     return !!(this.buildBazelContent && BUILD_TYPES_RULE_NAME.test(this.buildBazelContent));
   }
 
+  /**
+   * Returns true if the package is not intended to be in the build
+   */
+  isDevOnly() {
+    return !!this.pkg.kibana?.devOnly;
+  }
+
   /**
    * Custom inspect handler so that logging variables in scripts/generate doesn't
    * print all the BUILD.bazel files

packages/kbn-bazel-packages/src/parse_package_json.ts

@@ -13,17 +13,18 @@ import Fs from 'fs';
  * by `assertParsedPackageJson()` and extensible as needed in the future
  */
 export interface ParsedPackageJson {
-  /**
-   * The name of the package, usually `@kbn/`+something
-   */
+  /** The name of the package, usually `@kbn/`+something */
   name: string;
   /** "dependenices" property from package.json */
   dependencies?: Record<string, string>;
   /** "devDependenices" property from package.json */
   devDependencies?: Record<string, string>;
-  /**
-   * All other fields in the package.json are typed as unknown as we don't care what they are
-   */
+  /** Some kibana specific properties about this package */
+  kibana?: {
+    /** Is this package only intended for dev? */
+    devOnly?: boolean;
+  };
+  /** All other fields in the package.json are typed as unknown as we don't care what they are */
   [key: string]: unknown;
 }
 
@@ -46,6 +47,17 @@ export function assertParsedPackageJson(v: unknown): asserts v is ParsedPackageJ
   if (v.devDependencies && !isObj(v.devDependencies)) {
     throw new Error('Expected "dependencies" to be an object');
   }
+
+  const kibana = v.kibana;
+  if (kibana !== undefined) {
+    if (!isObj(kibana)) {
+      throw new Error('Expected "kibana" field in package.json to be an object');
+    }
+
+    if (kibana.devOnly !== undefined && typeof kibana.devOnly !== 'boolean') {
+      throw new Error('Expected "kibana.devOnly" field in package.json to be a boolean');
+    }
+  }
 }
 
 /**

packages/kbn-bazel-runner/BUILD.bazel

@@ -0,0 +1,122 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project")
+
+PKG_DIRNAME = "kbn-bazel-runner"
+PKG_REQUIRE_NAME = "@kbn/bazel-runner"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = [
+    "**/*.test.*",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+]
+
+# In this array place runtime dependencies, including other packages and NPM packages
+# which must be available for this code to run.
+#
+#  To reference other packages use:
+#    "//repo/relative/path/to/package"
+#    eg. "//packages/kbn-utils"
+#
+#  To reference a NPM package use:
+#    "@npm//name-of-package"
+#    eg. "@npm//lodash"
+RUNTIME_DEPS = [
+  "@npm//execa",
+  "@npm//chalk",
+  "@npm//rxjs",
+  "//packages/kbn-dev-utils",
+]
+
+# In this array place dependencies necessary to build the types, which will include the
+# :npm_module_types target of other packages and packages from NPM, including @types/*
+# packages.
+#
+#  To reference the types for another package use:
+#    "//repo/relative/path/to/package:npm_module_types"
+#    eg. "//packages/kbn-utils:npm_module_types"
+#
+#  References to NPM packages work the same as RUNTIME_DEPS
+TYPES_DEPS = [
+  "@npm//@types/node",
+  "@npm//@types/jest",
+  "@npm//execa",
+  "@npm//chalk",
+  "@npm//rxjs",
+  "//packages/kbn-dev-utils:npm_module_types",
+]
+
+jsts_transpiler(
+  name = "target_node",
+  srcs = SRCS,
+  build_pkg_name = package_name(),
+)
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+    "//:tsconfig.base.json",
+    "//:tsconfig.bazel.json",
+  ],
+)
+
+ts_project(
+  name = "tsc_types",
+  args = ['--pretty'],
+  srcs = SRCS,
+  deps = TYPES_DEPS,
+  declaration = True,
+  emit_declaration_only = True,
+  out_dir = "target_types",
+  root_dir = "src",
+  tsconfig = ":tsconfig",
+)
+
+js_library(
+  name = PKG_DIRNAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  deps = RUNTIME_DEPS + [":target_node"],
+  package_name = PKG_REQUIRE_NAME,
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [":" + PKG_DIRNAME],
+)
+
+filegroup(
+  name = "build",
+  srcs = [":npm_module"],
+  visibility = ["//visibility:public"],
+)
+
+pkg_npm_types(
+  name = "npm_module_types",
+  srcs = SRCS,
+  deps = [":tsc_types"],
+  package_name = PKG_REQUIRE_NAME,
+  tsconfig = ":tsconfig",
+  visibility = ["//visibility:public"],
+)
+
+filegroup(
+  name = "build_types",
+  srcs = [":npm_module_types"],
+  visibility = ["//visibility:public"],
+)

packages/kbn-bazel-runner/README.md

@@ -0,0 +1,3 @@
+# @kbn/bazel-runner
+
+Helpers for running bazel commands, used everywhere we programatically run bazel.

packages/kbn-bazel-runner/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-bazel-runner'],
+};

packages/kbn-bazel-runner/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@kbn/bazel-runner",
+  "private": true,
+  "version": "1.0.0",
+  "main": "./target_node/index.js",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "kibana": {
+    "devOnly": true
+  }
+}

packages/kbn-bazel-runner/src/bazel_runner.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import chalk from 'chalk';
+import execa from 'execa';
+import * as Rx from 'rxjs';
+import { tap } from 'rxjs/operators';
+import { ToolingLog } from '@kbn/tooling-log';
+import { observeLines } from '@kbn/stdio-dev-helpers';
+
+type BazelCommandRunner = 'bazel' | 'ibazel';
+
+interface BazelRunOptions {
+  log: ToolingLog;
+  bazelArgs: string[];
+  offline?: boolean;
+  execaOpts?: execa.Options;
+}
+
+async function runBazelCommandWithRunner(runner: BazelCommandRunner, options: BazelRunOptions) {
+  const bazelProc = execa(
+    runner,
+    options.offline ? [...options.bazelArgs, '--config=offline'] : options.bazelArgs,
+    {
+      ...options.execaOpts,
+      stdio: 'pipe',
+      preferLocal: true,
+    }
+  );
+
+  await Promise.all([
+    // Bazel outputs machine readable output into stdout and human readable output goes to stderr.
+    // Therefore we need to get both. In order to get errors we need to parse the actual text line
+    Rx.lastValueFrom(
+      Rx.merge(
+        observeLines(bazelProc.stdout!).pipe(
+          tap((line) => options.log.info(`${chalk.cyan(`[${runner}]`)} ${line}`))
+        ),
+        observeLines(bazelProc.stderr!).pipe(
+          tap((line) => options.log.info(`${chalk.cyan(`[${runner}]`)} ${line}`))
+        )
+      ).pipe(Rx.defaultIfEmpty(undefined))
+    ),
+
+    // Wait for process and logs to finish, unsubscribing in the end
+    bazelProc.catch(() => {
+      options.log.error(
+        'HINT: If experiencing problems with node_modules try `yarn kbn bootstrap --force-install` or as last resort `yarn kbn reset && yarn kbn bootstrap`'
+      );
+
+      throw new Error(`The bazel command that was running failed to complete.`);
+    }),
+  ]);
+}
+
+export async function runBazel(options: BazelRunOptions) {
+  await runBazelCommandWithRunner('bazel', options);
+}
+
+export async function runIBazel(options: BazelRunOptions) {
+  await runBazelCommandWithRunner('ibazel', {
+    ...options,
+    execaOpts: {
+      ...options.execaOpts,
+      env: {
+        ...options.execaOpts?.env,
+        IBAZEL_USE_LEGACY_WATCHER: '0',
+      },
+    },
+  });
+}

packages/kbn-pm/src/production/index.ts → packages/kbn-bazel-runner/src/index.ts

@@ -6,5 +6,4 @@
  * Side Public License, v 1.
  */
 
-export { buildBazelProductionProjects } from './build_bazel_production_projects';
-export { buildNonBazelProductionProjects } from './build_non_bazel_production_projects';
+export * from './bazel_runner';

packages/kbn-bazel-runner/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.bazel.json",
+  "compilerOptions": {
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "outDir": "target_types",
+    "rootDir": "src",
+    "stripInternal": false,
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ]
+}

packages/kbn-dev-utils/BUILD.bazel

@@ -36,7 +36,7 @@ filegroup(
 NPM_MODULE_EXTRA_FILES = [
   "package.json",
   "README.md",
-  ":certs"
+  ":certs",
 ]
 
 RUNTIME_DEPS = [