Merge branch 'main' into apm-span-metrics

.buildkite/disabled_jest_configs.json

@@ -0,0 +1,3 @@
+[
+  "x-pack/plugins/watcher/jest.config.js"
+]

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -15,6 +15,8 @@ import { load as loadYaml } from 'js-yaml';
 import { BuildkiteClient, BuildkiteStep } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
+import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
+
 type RunGroup = TestGroupRunOrderResponse['types'][0];
 
 const getRequiredEnv = (name: string) => {
@@ -220,6 +222,7 @@ export async function pickTestGroupRunOrder() {
     ? globby.sync(['**/jest.config.js', '!**/__fixtures__/**'], {
         cwd: process.cwd(),
         absolute: false,
+        ignore: DISABLED_JEST_CONFIGS,
       })
     : [];
 

.buildkite/pipelines/performance/daily.yml

@@ -20,6 +20,12 @@ steps:
     depends_on: build
     key: tests
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
 
   - label: '🚢 Performance Tests dataset extraction for scalability benchmarking'
     command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

.buildkite/scripts/steps/artifacts/env.sh

@@ -15,14 +15,11 @@ fi
 
 if [[ "$RELEASE_BUILD" == "true" ]]; then
   FULL_VERSION="$QUALIFIER_VERSION"
-
-  # Beats artifacts will need to match a specific commit sha that matches other stack images
-  # for release builds.  For now we are skipping Cloud builds until there's a pointer.
-  BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
+  BUILD_ARGS="--release --version-qualifier=$VERSION_QUALIFIER"
   WORKFLOW="staging"
 else
   FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
-  BUILD_ARGS="--skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
+  BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
   WORKFLOW="snapshot"
 fi
 

.buildkite/scripts/steps/artifacts/publish.sh

@@ -18,6 +18,8 @@ cd target
 
 download "kibana-$FULL_VERSION-docker-image.tar.gz"
 download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
+download "kibana-cloud-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-cloud-$FULL_VERSION-docker-image-aarch64.tar.gz"
 download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz"
 
 download "kibana-$FULL_VERSION-arm64.deb"

.buildkite/scripts/steps/functional/performance_playwright.sh

@@ -11,26 +11,9 @@ is_test_execution_step
 rm -rf "$KIBANA_BUILD_LOCATION"
 .buildkite/scripts/download_build_artifacts.sh
 
-echo "--- 🦺 Starting Elasticsearch"
-
-node scripts/es snapshot&
-export esPid=$!
-trap 'kill ${esPid}' EXIT
-
-export TEST_ES_URL=http://elastic:changeme@localhost:9200
-export TEST_ES_DISABLE_STARTUP=true
-
-# Pings the es server every second for up to 2 minutes until it is green
-curl \
-  --fail \
-  --silent \
-  --retry 120 \
-  --retry-delay 1 \
-  --retry-connrefused \
-  -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow" \
-  > /dev/null
-
-echo "✅ ES is ready and will continue to run in the background"
+function is_running {
+  kill -0 "$1" &>/dev/null
+}
 
 # unset env vars defined in other parts of CI for automatic APM collection of
 # Kibana. We manage APM config in our FTR config and performance service, and
@@ -46,29 +29,100 @@ unset ELASTIC_APM_SERVER_URL
 unset ELASTIC_APM_SECRET_TOKEN
 unset ELASTIC_APM_GLOBAL_LABELS
 
-journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover" "data_stress_test_lens")
+# `kill $esPid` doesn't work, seems that kbn-es doesn't listen to signals correctly, this does work
+trap 'killall node -q' EXIT
+
+export TEST_ES_URL=http://elastic:changeme@localhost:9200
+export TEST_ES_DISABLE_STARTUP=true
+
+echo "--- determining which journeys to run"
+
+journeys=$(buildkite-agent meta-data get "failed-journeys" --default '')
+if [ "$journeys" != "" ]; then
+  echo "re-running failed journeys:${journeys}"
+else
+  paths=()
+  for path in x-pack/performance/journeys/*; do
+    paths+=("$path")
+  done
+  journeys=$(printf "%s\n" "${paths[@]}")
+  echo "running discovered journeys:${journeys}"
+fi
+
+# track failed journeys here which might get written to metadata
+failedJourneys=()
+
+while read -r journey; do
+  if [ "$journey" == "" ]; then
+    continue;
+  fi
+
+  echo "--- $journey - 🔎 Start es"
 
-for journey in "${journeys[@]}"; do
-  set +e
+  node scripts/es snapshot&
+  export esPid=$!
+
+  # Pings the es server every second for up to 2 minutes until it is green
+  curl \
+    --fail \
+    --silent \
+    --retry 120 \
+    --retry-delay 1 \
+    --retry-connrefused \
+    -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow" \
+    > /dev/null
+
+  echo "✅ ES is ready and will run in the background"
 
   phases=("WARMUP" "TEST")
+  status=0
   for phase in "${phases[@]}"; do
     echo "--- $journey - $phase"
 
     export TEST_PERFORMANCE_PHASE="$phase"
+
+    set +e
     node scripts/functional_tests \
-      --config "x-pack/performance/journeys/$journey.ts" \
+      --config "$journey" \
       --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
       --debug \
       --bail
-
     status=$?
+    set -e
+
     if [ $status -ne 0 ]; then
+      failedJourneys+=("$journey")
       echo "^^^ +++"
       echo "❌ FTR failed with status code: $status"
-      exit 1
+      break
+    fi
+  done
+
+  # remove trap, we're manually shutting down
+  trap - EXIT;
+
+  echo "--- $journey - 🔎 Shutdown ES"
+  killall node
+  echo "waiting for $esPid to exit gracefully";
+
+  timeout=30 #seconds
+  dur=0
+  while is_running $esPid; do
+    sleep 1;
+    ((dur=dur+1))
+    if [ $dur -ge $timeout ]; then
+      echo "es still running after $dur seconds, killing ES and node forcefully";
+      killall -SIGKILL java
+      killall -SIGKILL node
+      sleep 5;
     fi
   done
+done <<< "$journeys"
+
+echo "--- report/record failed journeys"
+if [ "${failedJourneys[*]}" != "" ]; then
+  buildkite-agent meta-data set "failed-journeys" "$(printf "%s\n" "${failedJourneys[@]}")"
 
-  set -e
-done
+  echo "failed journeys: ${failedJourneys[*]}"
+  exit 1
+fi

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -46,8 +46,13 @@ cd "${OUTPUT_DIR}/.."
 gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
 cd -
 
-echo "--- Promoting '${BUILD_ID}' dataset to LATEST"
-cd "${OUTPUT_DIR}/.."
-echo "${BUILD_ID}" > latest
-gsutil cp latest "${GCS_BUCKET}"
-cd -
+if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-single-user-performance" ]; then
+  echo "--- Promoting '${BUILD_ID}' dataset to LATEST"
+  cd "${OUTPUT_DIR}/.."
+  echo "${BUILD_ID}" > latest
+  gsutil cp latest "${GCS_BUCKET}"
+  cd -
+else
+  echo "--- Skipping promotion of dataset to LATEST"
+  echo "$BUILDKITE_PIPELINE_SLUG is not 'kibana-single-user-performance', so skipping"
+fi

.buildkite/scripts/steps/scalability/benchmarking.sh

@@ -116,7 +116,7 @@ for journey in scalability_traces/server/*; do
     export SCALABILITY_JOURNEY_PATH="$KIBANA_DIR/$journey"
     echo "--- Run scalability file: $SCALABILITY_JOURNEY_PATH"
     node scripts/functional_tests \
-      --config x-pack/test/performance/scalability/config.ts \
+      --config x-pack/test/scalability/config.ts \
       --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
       --logToFile \
       --debug

.ci/Dockerfile

@@ -1,7 +1,7 @@
 # NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
 # If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
 
-ARG NODE_VERSION=16.16.0
+ARG NODE_VERSION=16.17.1
 
 FROM node:${NODE_VERSION} AS base
 

.eslintrc.js

@@ -1314,15 +1314,15 @@ module.exports = {
     {
       // typescript for front and back end
       files: [
-        'x-pack/plugins/{alerting,stack_alerts,stack_connectors,actions,task_manager,event_log}/**/*.{ts,tsx}',
+        'x-pack/plugins/{alerting,stack_alerts,actions,task_manager,event_log}/**/*.{ts,tsx}',
       ],
       rules: {
         '@typescript-eslint/no-explicit-any': 'error',
       },
     },
     {
       // typescript only for back end
-      files: ['x-pack/plugins/triggers_actions_ui/server/**/*.ts'],
+      files: ['x-pack/plugins/{stack_connectors,triggers_actions_ui}/server/**/*.ts'],
       rules: {
         '@typescript-eslint/no-explicit-any': 'error',
       },

.github/CODEOWNERS

@@ -84,10 +84,12 @@
 /x-pack/examples/ui_actions_enhanced_examples/ @elastic/kibana-app-services
 /x-pack/plugins/embeddable_enhanced/ @elastic/kibana-app-services
 /x-pack/plugins/runtime_fields @elastic/kibana-app-services
-/x-pack/test/search_sessions_integration/ @elastic/kibana-app-services
 /src/plugins/dashboard/public/application/embeddable/viewport/print_media @elastic/kibana-app-services
 x-pack/plugins/files @elastic/kibana-app-services
 x-pack/examples/files_example @elastic/kibana-app-services
+/x-pack/test/search_sessions_integration/ @elastic/kibana-app-services
+/test/plugin_functional/test_suites/panel_actions @elastic/kibana-app-services
+/test/plugin_functional/test_suites/data_plugin @elastic/kibana-app-services
 
 ### Observability Plugins
 
@@ -130,6 +132,7 @@ x-pack/examples/files_example @elastic/kibana-app-services
 /x-pack/test/fleet_api_integration @elastic/fleet
 /x-pack/test/fleet_cypress @elastic/fleet
 /x-pack/test/fleet_functional @elastic/fleet
+/src/dev/build/tasks/bundle_fleet_packages.ts
 
 # APM
 /x-pack/plugins/apm/  @elastic/apm-ui
@@ -329,7 +332,9 @@ x-pack/examples/files_example @elastic/kibana-app-services
 /x-pack/plugins/event_log/ @elastic/response-ops
 /x-pack/plugins/task_manager/ @elastic/response-ops
 /x-pack/plugins/stack_connectors/ @elastic/response-ops
+/x-pack/plugins/stack_connectors/public/connector_types/stack/ @elastic/response-ops-execution
 /x-pack/plugins/stack_connectors/server/connector_types/stack/ @elastic/response-ops-execution
+/x-pack/plugins/stack_connectors/public/connector_types/cases/ @elastic/response-ops-cases
 /x-pack/plugins/stack_connectors/server/connector_types/cases/ @elastic/response-ops-cases
 /x-pack/test/alerting_api_integration/ @elastic/response-ops
 /x-pack/test/plugin_api_integration/test_suites/task_manager/ @elastic/response-ops
@@ -348,6 +353,7 @@ x-pack/examples/files_example @elastic/kibana-app-services
 # Enterprise Search
 /x-pack/plugins/enterprise_search @elastic/enterprise-search-frontend
 /x-pack/test/functional_enterprise_search/ @elastic/enterprise-search-frontend
+/x-pack/plugins/enterprise_search/public/applications/shared/doc_links @elastic/ent-search-docs-team
 
 # Management Experience - Deployment Management
 /src/plugins/dev_tools/ @elastic/platform-deployment-management
@@ -737,6 +743,8 @@ packages/core/http/core-http-browser-mocks @elastic/kibana-core
 packages/core/http/core-http-common @elastic/kibana-core
 packages/core/http/core-http-context-server-internal @elastic/kibana-core
 packages/core/http/core-http-context-server-mocks @elastic/kibana-core
+packages/core/http/core-http-request-handler-context-server @elastic/kibana-core
+packages/core/http/core-http-request-handler-context-server-internal @elastic/kibana-core
 packages/core/http/core-http-router-server-internal @elastic/kibana-core
 packages/core/http/core-http-router-server-mocks @elastic/kibana-core
 packages/core/http/core-http-server @elastic/kibana-core
@@ -784,6 +792,7 @@ packages/core/preboot/core-preboot-server-internal @elastic/kibana-core
 packages/core/preboot/core-preboot-server-mocks @elastic/kibana-core
 packages/core/rendering/core-rendering-browser-internal @elastic/kibana-core
 packages/core/rendering/core-rendering-browser-mocks @elastic/kibana-core
+packages/core/root/core-root-browser-internal @elastic/kibana-core
 packages/core/saved-objects/core-saved-objects-api-browser @elastic/kibana-core
 packages/core/saved-objects/core-saved-objects-api-server @elastic/kibana-core
 packages/core/saved-objects/core-saved-objects-api-server-internal @elastic/kibana-core
@@ -894,6 +903,7 @@ packages/kbn-mapbox-gl @elastic/kibana-gis
 packages/kbn-monaco @elastic/kibana-app-services
 packages/kbn-optimizer @elastic/kibana-operations
 packages/kbn-optimizer-webpack-helpers @elastic/kibana-operations
+packages/kbn-osquery-io-ts-types @elastic/security-asset-management
 packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing
 packages/kbn-plugin-discovery @elastic/kibana-operations
 packages/kbn-plugin-generator @elastic/kibana-operations

.node-version

@@ -1 +1 @@
-16.16.0
+16.17.1

.nvmrc

@@ -1 +1 @@
-16.16.0
+16.17.1

WORKSPACE.bazel

@@ -22,13 +22,13 @@ load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install
 # Setup the Node.js toolchain for the architectures we want to support
 node_repositories(
   node_repositories = {
-    "16.16.0-darwin_amd64": ("node-v16.16.0-darwin-x64.tar.gz", "node-v16.16.0-darwin-x64", "982edd0fad364ad6e2d72161671544ab9399bd0ca8c726bde3cd07775c4c709a"),
-    "16.16.0-darwin_arm64": ("node-v16.16.0-darwin-arm64.tar.gz", "node-v16.16.0-darwin-arm64", "167721c2d72402e54adc0f8c87ca840796216c4d98946509d73221b771ad3e4c"),
-    "16.16.0-linux_arm64": ("node-v16.16.0-linux-arm64.tar.xz", "node-v16.16.0-linux-arm64", "6cb8f1353480646c1cd8ab9911995e5591e1a97811f43ea4ab3e946a57e7c80e"),
-    "16.16.0-linux_amd64": ("node-v16.16.0-linux-x64.tar.xz", "node-v16.16.0-linux-x64", "edcb6e9bb049ae365611aa209fc03c4bfc7e0295dbcc5b2f1e710ac70384a8ec"),
-    "16.16.0-windows_amd64": ("node-v16.16.0-win-x64.zip", "node-v16.16.0-win-x64", "c657acc98af55018c8fd6113c7e08d67c8083af75ba0306f9561b0117abc39d4"),
+    "16.17.1-darwin_amd64": ("node-v16.17.1-darwin-x64.tar.gz", "node-v16.17.1-darwin-x64", "3db26761ad8493b894d42260d7e65094b7af9bc473588739e61bc1c32d6ff955"),
+    "16.17.1-darwin_arm64": ("node-v16.17.1-darwin-arm64.tar.gz", "node-v16.17.1-darwin-arm64", "f9f02f7872e2e8ee54320fce13deb9d56904f32bb0615b6e21aa3371d8899150"),
+    "16.17.1-linux_arm64": ("node-v16.17.1-linux-arm64.tar.xz", "node-v16.17.1-linux-arm64", "3dfb8fd8f6b97df69cdc56524abc906c50ef1d0bf091188616802e6c7c731389"),
+    "16.17.1-linux_amd64": ("node-v16.17.1-linux-x64.tar.xz", "node-v16.17.1-linux-x64", "06ba2eb34aa385967f5f58c87a44753f83212f6cccea892b33f80a2e7fda8384"),
+    "16.17.1-windows_amd64": ("node-v16.17.1-win-x64.zip", "node-v16.17.1-win-x64", "ed290151efb417262b9808a70738d4ab79e9d53653a6a9f4b8dd97912e279dce"),
   },
-  node_version = "16.16.0",
+  node_version = "16.17.1",
   node_urls = [
     "https://nodejs.org/dist/v{version}/{filename}",
   ],

api_docs/actions.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the actions plugin
-date: 2022-09-23
+date: 2022-09-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 ---
 import actionsObj from './actions.devdocs.json';

api_docs/advanced_settings.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the advancedSettings plugin
-date: 2022-09-23
+date: 2022-09-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 ---
 import advancedSettingsObj from './advanced_settings.devdocs.json';

api_docs/aiops.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the aiops plugin
-date: 2022-09-23
+date: 2022-09-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 ---
 import aiopsObj from './aiops.devdocs.json';

api_docs/alerting.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/alerting
 title: "alerting"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the alerting plugin
-date: 2022-09-23
+date: 2022-09-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
 ---
 import alertingObj from './alerting.devdocs.json';

api_docs/apm.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/apm
 title: "apm"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the apm plugin
-date: 2022-09-23
+date: 2022-09-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apm']
 ---
 import apmObj from './apm.devdocs.json';

api_docs/banners.mdx

@@ -8,7 +8,7 @@ slug: /kibana-dev-docs/api/banners
 title: "banners"
 image: https://source.unsplash.com/400x175/?github
 description: API docs for the banners plugin
-date: 2022-09-23
+date: 2022-09-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'banners']
 ---
 import bannersObj from './banners.devdocs.json';