Merge branch 'master' into ml-annotations-delete-index

.buildkite/pipelines/performance/nightly.yml

@@ -0,0 +1,35 @@
+steps:
+  - block: ":gear: Performance Tests Configuration"
+    prompt: "Fill out the details for performance test"
+    fields:
+      - text: ":arrows_counterclockwise: Iterations"
+        key: "performance-test-iteration-count"
+        hint: "How many times you want to run tests? "
+        required: true
+    if: build.env('ITERATION_COUNT_ENV') == null
+
+  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+
+  - wait
+
+  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+
+  - label: ":muscle: Performance Tests"
+    command: .buildkite/scripts/steps/functional/performance.sh
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    concurrency: 50
+    concurrency_group: 'performance-test-group'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":male_superhero::skin-tone-2: Post-Build"
+    command: .buildkite/scripts/lifecycle/post_build.sh
+

.buildkite/scripts/build_kibana.sh

@@ -5,7 +5,11 @@ set -euo pipefail
 export KBN_NP_PLUGINS_BUILT=true
 
 echo "--- Build Kibana Distribution"
-node scripts/build --debug
+if [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-all-platforms"* ]]; then
+  node scripts/build --all-platforms --skip-os-packages
+else
+  node scripts/build
+fi
 
 echo "--- Archive Kibana Distribution"
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -55,21 +55,20 @@ const uploadPipeline = (pipelineContent) => {
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
 
     if (
-      await doAnyChangesMatch([
+      (await doAnyChangesMatch([
         /^x-pack\/plugins\/security_solution/,
         /^x-pack\/test\/security_solution_cypress/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
-      ]) || process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+      ])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
-    // Disabled for now, these are failing/disabled in Jenkins currently as well
     // if (
-    //   await doAnyChangesMatch([
-    //     /^x-pack\/plugins\/apm/,
-    //   ]) || process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    //   (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
+    //   process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     // ) {
     //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     // }

.buildkite/scripts/post_build_kibana.sh

@@ -12,7 +12,6 @@ fi
 echo "--- Upload Build Artifacts"
 # Moving to `target/` first will keep `buildkite-agent` from including directories in the artifact name
 cd "$KIBANA_DIR/target"
-mv kibana-*-linux-x86_64.tar.gz kibana-default.tar.gz
-buildkite-agent artifact upload kibana-default.tar.gz
-buildkite-agent artifact upload kibana-default-plugins.tar.gz
+cp kibana-*-linux-x86_64.tar.gz kibana-default.tar.gz
+buildkite-agent artifact upload "./*.tar.gz;./*.zip"
 cd -

.buildkite/scripts/steps/functional/performance.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -uo pipefail
+
+if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
+  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+else
+  ITERATION_COUNT=$ITERATION_COUNT_ENV
+fi
+
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - command: .buildkite/scripts/steps/functional/performance_sub.sh
+    parallelism: "$ITERATION_COUNT"
+EOF
+
+
+

.buildkite/scripts/steps/functional/performance_sub.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+cd "$XPACK_DIR"
+
+echo --- Run Performance Tests
+checks-reporter-with-killswitch "Run Performance Tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config test/performance/config.ts;

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=14.17.6
+ARG NODE_VERSION=16.11.1
 
 FROM node:${NODE_VERSION} AS base
 
@@ -10,7 +10,7 @@ RUN apt-get update && \
       libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
       libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
       libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
-      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-11-jre && \
     rm -rf /var/lib/apt/lists/*
 
 RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \

.eslintrc.js

@@ -156,6 +156,78 @@ const DEV_PATTERNS = [
   'x-pack/plugins/*/server/scripts/**/*',
 ];
 
+/** Restricted imports with suggested alternatives */
+const RESTRICTED_IMPORTS = [
+  {
+    name: 'lodash',
+    importNames: ['set', 'setWith'],
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash.set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash.setwith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/setWith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp',
+    importNames: ['set', 'setWith', 'assoc', 'assocPath'],
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/set',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/setWith',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/assoc',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash/fp/assocPath',
+    message: 'Please use @elastic/safer-lodash-set instead',
+  },
+  {
+    name: 'lodash',
+    importNames: ['template'],
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash.template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/fp',
+    importNames: ['template'],
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'lodash/fp/template',
+    message: 'lodash.template is unsafe, and not compatible with our content security policy.',
+  },
+  {
+    name: 'react-use',
+    message: 'Please use react-use/lib/{method} instead.',
+  },
+];
+
 module.exports = {
   root: true,
 
@@ -668,81 +740,7 @@ module.exports = {
         'no-restricted-imports': [
           2,
           {
-            paths: [
-              {
-                name: 'lodash',
-                importNames: ['set', 'setWith'],
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash.set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash.setwith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/setWith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp',
-                importNames: ['set', 'setWith', 'assoc', 'assocPath'],
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/set',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/setWith',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/assoc',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash/fp/assocPath',
-                message: 'Please use @elastic/safer-lodash-set instead',
-              },
-              {
-                name: 'lodash',
-                importNames: ['template'],
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash.template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/fp',
-                importNames: ['template'],
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'lodash/fp/template',
-                message:
-                  'lodash.template is unsafe, and not compatible with our content security policy.',
-              },
-              {
-                name: 'react-use',
-                message: 'Please use react-use/lib/{method} instead.',
-              },
-            ],
+            paths: RESTRICTED_IMPORTS,
           },
         ],
         'no-restricted-modules': [
@@ -835,6 +833,23 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ['**/common/**/*.{js,mjs,ts,tsx}', '**/public/**/*.{js,mjs,ts,tsx}'],
+      rules: {
+        'no-restricted-imports': [
+          2,
+          {
+            paths: [
+              ...RESTRICTED_IMPORTS,
+              {
+                name: 'semver',
+                message: 'Please use "semver/*/{function}" instead',
+              },
+            ],
+          },
+        ],
+      },
+    },
 
     /**
      * APM and Observability overrides
@@ -1582,8 +1597,8 @@ module.exports = {
       plugins: ['react', '@typescript-eslint'],
       files: ['x-pack/plugins/osquery/**/*.{js,mjs,ts,tsx}'],
       rules: {
-        // 'arrow-body-style': ['error', 'as-needed'],
-        // 'prefer-arrow-callback': 'error',
+        'arrow-body-style': ['error', 'as-needed'],
+        'prefer-arrow-callback': 'error',
         'no-unused-vars': 'off',
         'react/prop-types': 'off',
         '@typescript-eslint/explicit-module-boundary-types': 'off',

.node-version

@@ -1 +1 @@
-14.17.6
+16.11.1

.nvmrc

@@ -1 +1 @@
-14.17.6
+16.11.1

WORKSPACE.bazel

@@ -27,13 +27,13 @@ check_rules_nodejs_version(minimum_version_string = "3.8.0")
 # we can update that rule.
 node_repositories(
   node_repositories = {
-    "14.17.6-darwin_amd64": ("node-v14.17.6-darwin-x64.tar.gz", "node-v14.17.6-darwin-x64", "e3e4c02240d74fb1dc8a514daa62e5de04f7eaee0bcbca06a366ece73a52ad88"),
-    "14.17.6-linux_arm64": ("node-v14.17.6-linux-arm64.tar.xz", "node-v14.17.6-linux-arm64", "9c4f3a651e03cd9b5bddd33a80e8be6a6eb15e518513e410bb0852a658699156"),
-    "14.17.6-linux_s390x": ("node-v14.17.6-linux-s390x.tar.xz", "node-v14.17.6-linux-s390x", "3677f35b97608056013b5368f86eecdb044bdccc1b3976c1d4448736c37b6a0c"),
-    "14.17.6-linux_amd64": ("node-v14.17.6-linux-x64.tar.xz", "node-v14.17.6-linux-x64", "3bbe4faf356738d88b45be222bf5e858330541ff16bd0d4cfad36540c331461b"),
-    "14.17.6-windows_amd64": ("node-v14.17.6-win-x64.zip", "node-v14.17.6-win-x64", "b83e9ce542fda7fc519cec6eb24a2575a84862ea4227dedc171a8e0b5b614ac0"),
+    "16.11.1-darwin_amd64": ("node-v16.11.1-darwin-x64.tar.gz", "node-v16.11.1-darwin-x64", "ba54b8ed504bd934d03eb860fefe991419b4209824280d4274f6a911588b5e45"),
+    "16.11.1-linux_arm64": ("node-v16.11.1-linux-arm64.tar.xz", "node-v16.11.1-linux-arm64", "083fc51f0ea26de9041aaf9821874651a9fd3b20d1cf57071ce6b523a0436f17"),
+    "16.11.1-linux_s390x": ("node-v16.11.1-linux-s390x.tar.xz", "node-v16.11.1-linux-s390x", "855b5c83c2ccb05273d50bb04376335c68d47df57f3187cdebe1f22b972d2825"),
+    "16.11.1-linux_amd64": ("node-v16.11.1-linux-x64.tar.xz", "node-v16.11.1-linux-x64", "493bcc9b660eff983a6de65a0f032eb2717f57207edf74c745bcb86e360310b3"),
+    "16.11.1-windows_amd64": ("node-v16.11.1-win-x64.zip", "node-v16.11.1-win-x64", "4d3c179b82d42e66e321c3948a4e332ed78592917a69d38b86e3a242d7e62fb7"),
   },
-  node_version = "14.17.6",
+  node_version = "16.11.1",
   node_urls = [
     "https://nodejs.org/dist/v{version}/{filename}",
   ],

config/kibana.yml

@@ -94,6 +94,8 @@
 #logging.appenders.default:
 #  type: file
 #  fileName: /var/logs/kibana.log
+#  layout:
+#    type: json
 
 # Logs queries sent to Elasticsearch.
 #logging.loggers:

config/node.options

@@ -4,3 +4,6 @@
 
 ## max size of old space in megabytes
 #--max-old-space-size=4096
+
+## do not terminate process on unhandled promise rejection
+ --unhandled-rejections=warn

docs/dev-tools/console/console.asciidoc

@@ -129,13 +129,3 @@ image::dev-tools/console/images/console-settings.png["Console Settings", width=6
 
 For a list of available keyboard
 shortcuts, click *Help*.
-
-[float]
-[[console-settings]]
-=== Disable Console
-
-deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported."]
-If you don’t want to use *Console*, you can disable it by setting `console.enabled`
-to `false` in your `kibana.yml` configuration file. Changing this setting
-causes the server to regenerate assets on the next startup,
-which might cause a delay before pages start being served.