Merge branch 'main' into no-data-discover

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://[email protected]/elastic/kibana-buildkite-library#a0037514b7650296a23dbad99b165601d4eab1be"
+    "kibana-buildkite-library": "git+https://[email protected]/elastic/kibana-buildkite-library#ae4994aba5f2e72edcc5914e2aa208086e4b7ea3"
   }
 }

.buildkite/pipelines/artifacts.yml

@@ -41,7 +41,17 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/artifacts/docker_context.sh
+  - command: KIBANA_DOCKER_CONTEXT=default .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=cloud .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pipelines/on_merge.yml

@@ -107,6 +107,16 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
+    label: 'Populate local dev bazel cache (Linux)'
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 15
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - wait: ~
     continue_on_failure: true
 

.buildkite/scripts/common/setup_bazel.sh

@@ -32,6 +32,15 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
+if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
+  echo "[bazel] enabling caching with GCS buckets for local dev"
+
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+  build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+  build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+EOF
+fi
+
 if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then
   echo "[bazel] enabling caching with Buildbuddy"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
@@ -43,7 +52,7 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
-if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then
-  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]"
+if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|buildbuddy|none|) ]]; then
+  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs|buildbuddy,none]"
   exit 1
 fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -132,6 +132,10 @@ export SYNTHETICS_REMOTE_KIBANA_URL
 KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
 export KIBANA_BUILDBUDDY_CI_API_KEY
 
+BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
+export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+
 # By default, all steps should set up these things to get a full environment before running
 # It can be skipped for pipeline upload steps though, to make job start time a little faster
 if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then

.buildkite/scripts/steps/artifacts/build.sh

@@ -24,3 +24,8 @@ buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv"
 buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv.sha512.txt"
 buildkite-agent artifact upload 'i18n/*.json'
 cd -
+
+cd .beats
+buildkite-agent artifact upload 'metricbeat-*'
+buildkite-agent artifact upload 'filebeat-*'
+cd -

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -6,17 +6,29 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/artifacts/env.sh
 
+KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
+
 echo "--- Create contexts"
 mkdir -p target
 node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS")
 
-echo "--- Setup default context"
+echo "--- Setup context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)
 
-tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
+if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+fi
+
+tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
 buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+if [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
+  buildkite-agent artifact download "metricbeat-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "filebeat-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+fi
 
 echo "--- Build context"
 docker build .

.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
+export DISABLE_BOOTSTRAP_VALIDATION=true
+
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
+.buildkite/scripts/bootstrap.sh

.buildkite/scripts/steps/bazel_cache/bootstrap_mac.sh

@@ -2,13 +2,19 @@
 
 set -euo pipefail
 
-export BAZEL_CACHE_MODE=buildbuddy
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
 export DISABLE_BOOTSTRAP_VALIDATION=true
 
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
 # Since our Mac agents are currently static,
 # use a temporary HOME directory that gets cleaned out between builds
 TMP_HOME="$WORKSPACE/tmp_home"
 rm -rf "$TMP_HOME"
+mkdir -p "$TMP_HOME"
 export HOME="$TMP_HOME"
 
 .buildkite/scripts/bootstrap.sh

.github/CODEOWNERS

@@ -499,6 +499,7 @@
 /x-pack/plugins/security_solution/public/common/components/health_truncate_text @elastic/security-detections-response-rules
 /x-pack/plugins/security_solution/public/common/components/links_to_docs @elastic/security-detections-response-rules
 /x-pack/plugins/security_solution/public/common/components/callouts @elastic/security-detections-response-rules
+/x-pack/plugins/security_solution/public/common/components/ml_popover @elastic/security-detections-response-rules
 /x-pack/plugins/security_solution/public/detections/components/callouts @elastic/security-detections-response-rules
 /x-pack/plugins/security_solution/public/detections/mitre @elastic/security-detections-response-rules
 /x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules @elastic/security-detections-response-rules

docs/api/alerting/create_rule.asciidoc

@@ -6,15 +6,24 @@
 
 Create {kib} rules.
 
-WARNING: This API supports <<token-api-authentication>> only.
-
 [[create-rule-api-request]]
 ==== Request
 
 `POST <kibana host>:<port>/api/alerting/rule/<id>`
 
 `POST <kibana host>:<port>/s/<space_id>/api/alerting/rule/<id>`
 
+==== {api-description-title}
+
+[WARNING]
+====
+* This API supports only
+<<token-api-authentication,token-based authentication>>.
+* When you create a rule, it identifies which roles you have at that point in time.
+Thereafter, when the rule performs queries, it uses those security privileges.
+If a user with different privileges updates the rule, its behavior might change.
+====
+
 [[create-rule-api-path-params]]
 ==== Path parameters
 

docs/api/alerting/update_rule.asciidoc

@@ -6,15 +6,25 @@
 
 Update the attributes for an existing rule.
 
-WARNING: This API supports <<token-api-authentication>> only.
-
 [[update-rule-api-request]]
 ==== Request
 
 `PUT <kibana host>:<port>/api/alerting/rule/<id>`
 
 `PUT <kibana host>:<port>/s/<space_id>/api/alerting/rule/<id>`
 
+==== {api-description-title}
+
+[WARNING]
+====
+* This API supports only
+<<token-api-authentication,token-based authentication>>.
+* When you update a rule, it identifies which roles you have at that point in time.
+Thereafter, when the rule performs queries, it uses those security privileges.
+If you have different privileges than the user that created or most recently
+updated the rule, you might change its behavior.
+====
+
 [[update-rule-api-path-params]]
 ==== Path parameters
 

docs/api/logstash-configuration-management/list-pipeline.asciidoc

@@ -6,6 +6,8 @@
 
 experimental[] List all centrally-managed Logstash pipelines.
 
+IMPORTANT: Limit the number of pipelines to 10k or fewer. As the number of pipelines nears and surpasses 10k, you may see performance issues on {kib}.
+
 [[logstash-configuration-management-api-list-request]]
 ==== Request
 

docs/maps/trouble-shooting.asciidoc

@@ -12,10 +12,13 @@ Use the information in this section to inspect Elasticsearch requests and find s
 [float]
 === Inspect Elasticsearch requests
 
-Maps uses the {ref}/search-search.html[{es} search API] to get documents and aggregation results from {es}. To troubleshoot these requests, open the Inspector, which shows the most recent requests for each layer. You can switch between different requests using the *Request* dropdown.
+Maps uses the {ref}/search-vector-tile-api.html[{es} vector tile search API] and the {ref}/search-search.html[{es} search API] to get documents and aggregation results from {es}. Use *Vector tiles* inspector to view {es} vector tile search API requests. Use *Requests* inspector to view {es} search API requests.
 
 [role="screenshot"]
-image::maps/images/inspector.png[]
+image::maps/images/vector_tile_inspector.png[]
+
+[role="screenshot"]
+image::maps/images/requests_inspector.png[]
 
 [float]
 === Solutions to common problems

docs/setup/upgrade/resolving-migration-failures.asciidoc

@@ -171,7 +171,7 @@ Upgrade migrations fail because routing allocation is disabled or restricted (`c
 
 [source,sh]
 --------------------------------------------
-Unable to complete saved object migrations for the [.kibana] index: [unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}
+Unable to complete saved object migrations for the [.kibana] index: [incompatible_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}
 --------------------------------------------
 
 To get around the issue, remove the transient and persisted routing allocation settings:

package.json

@@ -73,7 +73,7 @@
     "yarn": "^1.21.1"
   },
   "resolutions": {
-    "**/@babel/runtime": "^7.17.8",
+    "**/@babel/runtime": "^7.17.9",
     "**/@types/node": "16.11.7",
     "**/chokidar": "^3.4.3",
     "**/deepmerge": "^4.2.2",
@@ -97,7 +97,7 @@
     "puppeteer/node-fetch": "^2.6.7"
   },
   "dependencies": {
-    "@babel/runtime": "^7.17.8",
+    "@babel/runtime": "^7.17.9",
     "@dnd-kit/core": "^3.1.1",
     "@dnd-kit/sortable": "^4.0.0",
     "@dnd-kit/utilities": "^2.0.0",
@@ -446,12 +446,12 @@
   "devDependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
     "@babel/cli": "^7.17.6",
-    "@babel/core": "^7.17.8",
+    "@babel/core": "^7.17.9",
     "@babel/eslint-parser": "^7.17.0",
     "@babel/eslint-plugin": "^7.17.7",
-    "@babel/generator": "^7.17.7",
+    "@babel/generator": "^7.17.9",
     "@babel/helper-plugin-utils": "^7.16.7",
-    "@babel/parser": "^7.17.8",
+    "@babel/parser": "^7.17.9",
     "@babel/plugin-proposal-class-properties": "^7.16.7",
     "@babel/plugin-proposal-export-namespace-from": "^7.16.7",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.16.7",
@@ -463,7 +463,7 @@
     "@babel/preset-react": "^7.16.7",
     "@babel/preset-typescript": "^7.16.7",
     "@babel/register": "^7.17.7",
-    "@babel/traverse": "^7.17.3",
+    "@babel/traverse": "^7.17.9",
     "@babel/types": "^7.17.0",
     "@bazel/ibazel": "^0.16.2",
     "@bazel/typescript": "4.6.2",
@@ -796,7 +796,7 @@
     "babel-plugin-add-module-exports": "^1.0.4",
     "babel-plugin-istanbul": "^6.1.1",
     "babel-plugin-require-context-hook": "^1.0.0",
-    "babel-plugin-styled-components": "^2.0.6",
+    "babel-plugin-styled-components": "^2.0.7",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
     "backport": "^7.3.1",
     "callsites": "^3.1.0",