Merge branch 'main' into obs-ai-assistant-ccs-by-default

.buildkite/ftr_oblt_serverless_configs.yml

@@ -12,6 +12,7 @@ enabled:
   - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -352,6 +352,7 @@ enabled:
   - x-pack/performance/journeys_e2e/promotion_tracking_dashboard.ts
   - x-pack/performance/journeys_e2e/web_logs_dashboard.ts
   - x-pack/performance/journeys_e2e/web_logs_dashboard_esql.ts
+  - x-pack/performance/journeys_e2e/web_logs_dashboard_dataview.ts
   - x-pack/performance/journeys_e2e/data_stress_test_lens.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_saved_search_only.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard_tsvb_gauge_only.ts

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -42,7 +42,6 @@ jq '
   .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_TEST_IMAGE'" |
   .resources.kibana[0].plan.kibana.version = "'$FULL_VERSION'" |
   .resources.elasticsearch[0].plan.elasticsearch.version = "'$FULL_VERSION'" |
-  .resources.enterprise_search[0].plan.enterprise_search.version = "'$FULL_VERSION'" |
   .resources.integrations_server[0].plan.integrations_server.version = "'$FULL_VERSION'"
   ' .buildkite/scripts/steps/cloud/deploy.json > "$DEPLOYMENT_SPEC"
 

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -126,15 +126,15 @@ steps:
       env:
         CONTAINER: "$KIBANA_IMAGE"
     soft_fail: true
-  - label: ":argo: Update kibana image tag for kibana-controller using gpctl"
+  - label: ":serverless::argo: Run synthetics tests and update kibana image tag to ${GIT_ABBREV_COMMIT} for kibana-controller"
     branches: main
-    trigger: gpctl-promote-with-e2e-tests
+    trigger: gpctl-promote-after-serverless-devenv-synthetics
     build:
       env:
         SERVICE_COMMIT_HASH: "$GIT_ABBREV_COMMIT"
         SERVICE: kibana
         REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/dev.yaml
-        DRY_RUN: "${DRY_RUN:-false}"
+        GPCTL_PROMOTE_DRY_RUN: ${DRY_RUN:-false}
 EOF
 
 else

.buildkite/scripts/steps/capture_oas_snapshot.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Capture OAS snapshot
-cmd="node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules --include-path /api/actions"
+cmd="node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules --include-path /api/actions --include-path /api/security/role --include-path /api/spaces"
 if is_pr && ! is_auto_commit_disabled; then
   cmd="$cmd --update"
 fi

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -68,7 +68,6 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
     .name = "'$CLOUD_DEPLOYMENT_NAME'" |
     .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
     .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" |
-    .resources.enterprise_search[0].plan.enterprise_search.version = "'$VERSION'" |
     .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'"
     ' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json
 

.buildkite/scripts/steps/esql_generate_function_metadata.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 VALIDATION_PACKAGE_DIR="packages/kbn-esql-validation-autocomplete"
-EDITOR_PACKAGE_DIR="packages/kbn-language-documentation-popover"
+EDITOR_PACKAGE_DIR="packages/kbn-language-documentation"
 GIT_SCOPE="$VALIDATION_PACKAGE_DIR/**/* $EDITOR_PACKAGE_DIR/**/*"
 
 report_main_step () {

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -7,3 +7,6 @@ source .buildkite/scripts/common/util.sh
 echo --- Merge Kibana OpenAPI specs
 
 (cd oas_docs && make api-docs && make api-docs-lint)
+(cd oas_docs && make api-docs-staging && make api-docs-lint)
+
+check_for_changed_files "make api-docs && make api-docs-staging" true

.buildkite/scripts/steps/storybooks/build_and_upload.ts

@@ -18,7 +18,6 @@ const STORYBOOKS = [
   'canvas',
   'cases',
   'cell_actions',
-  'cloud_chat',
   'coloring',
   'chart_icons',
   'content_management_examples',
@@ -54,7 +53,7 @@ const STORYBOOKS = [
   'language_documentation_popover',
   'unified_search',
   'random_sampling',
-  'text_based_editor',
+  'esql_editor',
 ];
 
 const GITHUB_CONTEXT = 'Build and Publish Storybooks';

.devcontainer/Dockerfile

@@ -4,7 +4,6 @@ ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
 ENV HOME=/home/vscode
 ENV NVM_DIR=${HOME}/nvm
 ENV NVM_VERSION=v0.39.1
-ENV KBN_DIR=/workspaces/kibana
 ENV OPENSSL_PATH=${HOME}/openssl
 # Only specific versions are FIPS certified.
 ENV OPENSSL_VERSION='3.0.8'
@@ -50,6 +49,7 @@ WORKDIR ${KBN_DIR}
 # Node and NVM setup
 COPY .node-version /tmp/
 
+# Mac will have permissions issues if Node and NVM are installed as root
 USER vscode
 
 RUN mkdir -p $NVM_DIR && \

.devcontainer/devcontainer.json

@@ -23,8 +23,11 @@
     9230,
     9231
   ],
-  "postStartCommand": "/workspaces/kibana/.devcontainer/scripts/post_start.sh",
+  "postStartCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/post_start.sh",
   "remoteUser": "vscode",
+  "containerEnv": {
+    "KBN_DIR": "${containerWorkspaceFolder}"
+  },
   "features": {
     "ghcr.io/devcontainers/features/docker-in-docker:2": {
       "version": "latest",

.github/CODEOWNERS

@@ -354,6 +354,7 @@ x-pack/plugins/data_quality @elastic/obs-ux-logs-team
 test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery
 packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery
 packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore
+x-pack/plugins/data_usage @elastic/obs-ai-assistant @elastic/security-solution
 src/plugins/data_view_editor @elastic/kibana-data-discovery
 examples/data_view_field_editor_example @elastic/kibana-data-discovery
 src/plugins/data_view_field_editor @elastic/kibana-data-discovery
@@ -428,6 +429,7 @@ src/plugins/esql @elastic/kibana-esql
 packages/kbn-esql-ast @elastic/kibana-esql
 examples/esql_ast_inspector @elastic/kibana-esql
 src/plugins/esql_datagrid @elastic/kibana-esql
+packages/kbn-esql-editor @elastic/kibana-esql
 packages/kbn-esql-utils @elastic/kibana-esql
 packages/kbn-esql-validation-autocomplete @elastic/kibana-esql
 examples/esql_validation_example @elastic/kibana-esql
@@ -474,7 +476,7 @@ src/plugins/files_management @elastic/appex-sharedux
 src/plugins/files @elastic/appex-sharedux
 packages/kbn-find-used-node-modules @elastic/kibana-operations
 x-pack/plugins/fleet @elastic/fleet
-packages/kbn-flot-charts @elastic/kibana-operations
+packages/kbn-flot-charts @elastic/kibana-presentation @elastic/stack-monitoring
 x-pack/test/ui_capabilities/common/plugins/foo_plugin @elastic/kibana-security
 packages/kbn-formatters @elastic/obs-ux-logs-team
 src/plugins/ftr_apis @elastic/kibana-core
@@ -552,7 +554,7 @@ src/plugins/kibana_usage_collection @elastic/kibana-core
 src/plugins/kibana_utils @elastic/appex-sharedux
 x-pack/plugins/kubernetes_security @elastic/kibana-cloud-security-posture
 x-pack/packages/kbn-langchain @elastic/security-generative-ai
-packages/kbn-language-documentation-popover @elastic/kibana-esql
+packages/kbn-language-documentation @elastic/kibana-esql
 x-pack/examples/lens_config_builder_example @elastic/kibana-visualizations
 packages/kbn-lens-embeddable-utils @elastic/obs-ux-infra_services-team @elastic/kibana-visualizations
 packages/kbn-lens-formula-docs @elastic/kibana-visualizations
@@ -608,13 +610,15 @@ x-pack/packages/ml/data_view_utils @elastic/ml-ui
 x-pack/packages/ml/date_picker @elastic/ml-ui
 x-pack/packages/ml/date_utils @elastic/ml-ui
 x-pack/packages/ml/error_utils @elastic/ml-ui
+x-pack/packages/ml/field_stats_flyout @elastic/ml-ui
 x-pack/packages/ml/in_memory_table @elastic/ml-ui
 x-pack/packages/ml/is_defined @elastic/ml-ui
 x-pack/packages/ml/is_populated_object @elastic/ml-ui
 x-pack/packages/ml/kibana_theme @elastic/ml-ui
 x-pack/packages/ml/local_storage @elastic/ml-ui
 x-pack/packages/ml/nested_property @elastic/ml-ui
 x-pack/packages/ml/number_utils @elastic/ml-ui
+x-pack/packages/ml/parse_interval @elastic/ml-ui
 x-pack/plugins/ml @elastic/ml-ui
 x-pack/packages/ml/query_utils @elastic/ml-ui
 x-pack/packages/ml/random_sampler_utils @elastic/ml-ui
@@ -626,6 +630,7 @@ x-pack/packages/ml/time_buckets @elastic/ml-ui
 x-pack/packages/ml/trained_models_utils @elastic/ml-ui
 x-pack/packages/ml/ui_actions @elastic/ml-ui
 x-pack/packages/ml/url_state @elastic/ml-ui
+x-pack/packages/ml/validators @elastic/ml-ui
 packages/kbn-mock-idp-plugin @elastic/kibana-security
 packages/kbn-mock-idp-utils @elastic/kibana-security
 packages/kbn-monaco @elastic/appex-sharedux
@@ -651,6 +656,7 @@ x-pack/plugins/observability_solution/observability_onboarding/e2e @elastic/obs-
 x-pack/plugins/observability_solution/observability_onboarding @elastic/obs-ux-logs-team @elastic/obs-ux-onboarding-team
 x-pack/plugins/observability_solution/observability @elastic/obs-ux-management-team
 x-pack/plugins/observability_solution/observability_shared @elastic/observability-ui
+x-pack/packages/observability/synthetics_test_data @elastic/obs-ux-management-team
 x-pack/packages/observability/observability_utils @elastic/observability-ui
 x-pack/test/security_api_integration/plugins/oidc_provider @elastic/kibana-security
 test/common/plugins/otel_metrics @elastic/obs-ux-infra_services-team
@@ -747,6 +753,8 @@ src/plugins/screenshot_mode @elastic/appex-sharedux
 x-pack/examples/screenshotting_example @elastic/appex-sharedux
 x-pack/plugins/screenshotting @elastic/kibana-reporting-services
 packages/kbn-screenshotting-server @elastic/appex-sharedux
+packages/kbn-search-api-keys-components @elastic/search-kibana
+packages/kbn-search-api-keys-server @elastic/search-kibana
 packages/kbn-search-api-panels @elastic/search-kibana
 x-pack/plugins/search_assistant @elastic/search-kibana
 packages/kbn-search-connectors @elastic/search-kibana
@@ -760,6 +768,7 @@ x-pack/plugins/search_inference_endpoints @elastic/search-kibana
 x-pack/plugins/search_notebooks @elastic/search-kibana
 x-pack/plugins/search_playground @elastic/search-kibana
 packages/kbn-search-response-warnings @elastic/kibana-data-discovery
+x-pack/packages/search/shared_ui @elastic/search-kibana
 packages/kbn-search-types @elastic/kibana-data-discovery
 x-pack/plugins/searchprofiler @elastic/kibana-management
 x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security
@@ -918,7 +927,6 @@ test
 x-pack/test 
 x-pack/performance @elastic/appex-qa
 x-pack/examples/testing_embedded_lens @elastic/kibana-visualizations
-packages/kbn-text-based-editor @elastic/kibana-esql
 x-pack/examples/third_party_lens_navigation_prompt @elastic/kibana-visualizations
 x-pack/examples/third_party_vis_lens_example @elastic/kibana-visualizations
 x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations
@@ -1105,6 +1113,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 # Observability AI Assistant
 x-pack/test/observability_ai_assistant_api_integration @elastic/obs-ai-assistant
 x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
+x-pack/test_serverless/**/test_suites/observability/ai_assistant @elastic/obs-ai-assistant
 
 # Infra Monitoring
 ## This plugin mostly contains the codebase for the infra services, but also includes some code for the Logs UI app.
@@ -1235,6 +1244,7 @@ x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
 /x-pack/test/functional/apps/canvas/ @elastic/kibana-presentation
 /x-pack/test_serverless/functional/test_suites/search/dashboards/ @elastic/kibana-presentation
 /test/plugin_functional/test_suites/panel_actions @elastic/kibana-presentation
+/x-pack/test/functional/es_archives/canvas/logstash_lens @elastic/kibana-presentation
 #CC# /src/plugins/kibana_react/public/code_editor/ @elastic/kibana-presentation
 
 # Machine Learning
@@ -1417,6 +1427,15 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib
 /x-pack/test_serverless/functional/test_suites/security/screenshot_creation/response_ops_docs @elastic/response-ops
 /x-pack/test_serverless/functional/test_suites/observability/screenshot_creation/response_ops_docs @elastic/response-ops
 /x-pack/test_serverless/api_integration/test_suites/common/alerting/ @elastic/response-ops
+/x-pack/test/functional/es_archives/action_task_params @elastic/response-ops
+/x-pack/test/functional/es_archives/actions @elastic/response-ops
+/x-pack/test/functional/es_archives/alerting @elastic/response-ops
+/x-pack/test/functional/es_archives/alerts @elastic/response-ops
+/x-pack/test/functional/es_archives/alerts_legacy @elastic/response-ops
+/x-pack/test/functional/es_archives/observability/alerts @elastic/response-ops
+/x-pack/test/functional/es_archives/actions @elastic/response-ops
+/x-pack/test/functional/es_archives/rules_scheduled_task_id  @elastic/response-ops
+/x-pack/test/functional/es_archives/alerting/8_2_0  @elastic/response-ops
 
 # Enterprise Search
 /x-pack/test/functional_enterprise_search/ @elastic/search-kibana
@@ -1448,6 +1467,9 @@ x-pack/test/api_integration/apis/management/index_management/inference_endpoints
 /x-pack/test/plugin_functional/test_suites/resolver/ @elastic/security-solution
 /x-pack/test/security_solution_api_integration @elastic/security-solution
 /x-pack/test/api_integration/apis/security_solution @elastic/security-solution
+/x-pack/test/functional/es_archives/auditbeat/default @elastic/security-solution
+/x-pack/test/functional/es_archives/auditbeat/hosts @elastic/security-solution
+
 #CC# /x-pack/plugins/security_solution/ @elastic/security-solution
 
 # Security Solution OpenAPI bundles
@@ -1518,6 +1540,7 @@ x-pack/test/security_solution_api_integration/test_suites/sources @elastic/secur
 x-pack/test/security_solution_cypress/cypress/screens/expandable_flyout  @elastic/security-threat-hunting-investigations
 x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/security-threat-hunting-investigations
 
+/x-pack/plugins/security_solution/common/timelines @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/alerts_viewer @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_action @elastic/security-threat-hunting-investigations
 /x-pack/plugins/security_solution/public/common/components/event_details @elastic/security-threat-hunting-investigations
@@ -1578,6 +1601,11 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network @elastic/security-threat-hunting-explore
 /x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users @elastic/security-threat-hunting-explore
 
+/x-pack/test/functional/es_archives/auditbeat/overview @elastic/security-threat-hunting-explore
+/x-pack/test/functional/es_archives/auditbeat/users @elastic/security-threat-hunting-explore
+
+/x-pack/test/functional/es_archives/auditbeat/uncommon_processes @elastic/security-threat-hunting-explore
+
 ## Generative AI owner connectors
 # OpenAI
 /x-pack/plugins/stack_connectors/public/connector_types/openai @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra
@@ -1677,6 +1705,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout  @elastic/
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/rule_gaps.ts @elastic/security-detection-engine
 /x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists @elastic/security-detection-engine
+/x-pack/test/functional/es_archives/asset_criticality @elastic/security-detection-engine
 
 ## Security Threat Intelligence - Under Security Platform
 /x-pack/plugins/security_solution/public/common/components/threat_match @elastic/security-detection-engine

.github/workflows/alert-failed-test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout kibana-operations
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           repository: 'elastic/kibana-operations'
           ref: main

.github/workflows/codeql-stats.yml

@@ -11,7 +11,7 @@ jobs:
     if: github.repository == 'elastic/kibana' # Hack: Do not run on forks
     steps:
     - name: Checkout kibana-operations
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         repository: 'elastic/kibana-operations'
         ref: main

.github/workflows/codeql.yml

@@ -18,11 +18,11 @@ jobs:
       fail-fast: false
       matrix:
         language: [ 'javascript' ]
-        branch: [ 'main', '7.17' ]
+        branch: [ 'main', '7.17', '8.x' ]
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         ref: ${{ matrix.branch }}
 
@@ -61,7 +61,7 @@ jobs:
     if: github.repository == 'elastic/kibana' # Hack: Do not run on forks
     steps:
     - name: Checkout kibana-operations
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       with:
         repository: 'elastic/kibana-operations'
         ref: main
@@ -73,6 +73,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{secrets.KIBANAMACHINE_TOKEN}}
         SLACK_TOKEN: ${{secrets.CODE_SCANNING_SLACK_TOKEN}}
+        CODEQL_BRANCHES: 7.17,8.x,main
       run: |
         npm ci --omit=dev
-        node codeql-alert 
+        node codeql-alert

.github/workflows/label-failed-test.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout kibana-operations
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           repository: 'elastic/kibana-operations'
           ref: main

.github/workflows/launchdarkly-code-references.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.repository.fork == false
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           fetch-depth: 11 # This value must be set if the lookback configuration option is not disabled for find-code-references. Read more: https://github.com/launchdarkly/ld-find-code-refs#searching-for-unused-flags-extinctions
       - name: LaunchDarkly Code References