Merge branch 'main' into 119109-es-integration-queries

.bazelrc.common

@@ -132,6 +132,9 @@ coverage --instrument_test_targets
 # Metadata settings
 build --workspace_status_command="node ./src/dev/bazel_workspace_status.js"
 
+# Load remote cache settings, if they exist
+try-import %workspace%/.bazelrc.cache
+
 # Load any settings specific to the current user.
 # .bazelrc.user should appear in .gitignore so that settings are not shared with team members
 # This needs to be last statement in this

.buildkite/scripts/common/env.sh

@@ -22,6 +22,9 @@ KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_BRANCH
 export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 
+KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
+export KIBANA_PKG_VERSION
+
 export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
 export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
 export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -92,6 +92,15 @@ export KIBANA_DOCKER_USERNAME
 KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
 export KIBANA_DOCKER_PASSWORD
 
+SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_USERNAME
+
+SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_PASSWORD
+
+SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+export SYNTHETICS_SERVICE_MANIFEST
+
 # Setup Failed Test Reporter Elasticsearch credentials
 {
   TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)

.buildkite/scripts/steps/package_testing/build.sh

@@ -4,9 +4,10 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-echo "--- Build Kibana Distribution"
 node scripts/build --all-platforms --debug --skip-docker-cloud --skip-docker-ubi --skip-docker-contexts
 
+DOCKER_FILE="kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz"
+
 cd target
-buildkite-agent artifact upload "./*-docker-image.tar.gz;./*.deb;./*.rpm"
+buildkite-agent artifact upload "./$DOCKER_FILE;./*.deb;./*.rpm"
 cd ..

.buildkite/scripts/steps/package_testing/test.sh

@@ -15,7 +15,7 @@ elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
   buildkite-agent artifact download 'kibana-*.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.50.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
-  buildkite-agent artifact download 'kibana-*-docker-image.tar.gz' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION-SNAPSHOT-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.50.7"
 fi
 cd ..

.gitignore

@@ -87,8 +87,9 @@ report.asciidoc
 /bazel
 /bazel-*
 .bazelrc.user
+.bazelrc.cache
 
 elastic-agent-*
 fleet-server-*
 elastic-agent.yml
-fleet-server.yml
+fleet-server.yml

docs/apm/troubleshooting.asciidoc

@@ -47,37 +47,14 @@ This section can help with any of the following:
 There are a number of factors that could be at play here.
 One important thing to double-check first is your index template.
 
-*Index template*
-An APM index template must exist for the APM app to work correctly.
-By default, this index template is created by APM Server on startup.
-However, this only happens if `setup.template.enabled` is `true` in `apm-server.yml`.
-You can create the index template manually by running `apm-server setup`.
+*Index templates*
+For the APM app to work correctly, an index template must exist for each APM data stream.
+By default, {fleet} sets up APM index templates when the APM integration is installed.
 Take note that index templates *cannot* be applied retroactively -- they are only applied at index creation time.
 More information is available in {apm-guide-ref}/apm-server-configuration.html[Set up and configure].
 
-You can check for the existence of an APM index template using the
-{ref}/indices-get-template.html[Get index template API].
-If you're using the default index naming pattern, that request would be:
-
-[source,js]
---------------------------------------------------
-GET /_template/apm-{version}
---------------------------------------------------
-// CONSOLE
-
-*Using Logstash, Kafka, etc.*
-If you're not outputting data directly from APM Server to Elasticsearch (perhaps you're using Logstash or Kafka),
-then the index template will not be set up automatically. Instead, you'll need to
-{apm-guide-ref}/apm-server-template.html[load the template manually].
-
-*Using a custom index names*
-This problem can also occur if you've customized the index name that you write APM data to.
-If you change the default, you must also configure the `setup.template.name` and `setup.template.pattern` options.
-See {apm-guide-ref}/configuration-template.html[Load the Elasticsearch index template].
-If the Elasticsearch index template has already been successfully loaded to the index,
-you can customize the indices that the APM app uses to display data.
-Navigate to *APM* > *Settings* > *Indices*, and change all `xpack.apm.indices.*` values to
-include the new index pattern. For example: `customIndexName-*`.
+You can check for the existence of APM index templates in Kibana.
+Go to **Stack Management** > **Index Management** > **Index Templates** and search for `apm`.
 
 [float]
 [[troubleshooting-too-many-transactions]]
@@ -164,39 +141,34 @@ You can also use the Agent's public API to manually set a name for the transacti
 === Fields are not searchable
 
 In Elasticsearch, index templates are used to define settings and mappings that determine how fields should be analyzed.
-The recommended index template file for APM Server is installed by the APM Server packages.
-This template, by default, enables and disables indexing on certain fields.
+The recommended index templates for APM are installed by {fleet} when the Elastic APM integration is installed.
+These templates, by default, enable and disable indexing on certain fields.
 
 As an example, some agents store cookie values in `http.request.cookies`.
 Since `http.request` has disabled dynamic indexing, and `http.request.cookies` is not declared in a custom mapping,
 the values in `http.request.cookies` are not indexed and thus not searchable.
 
-*Ensure an index pattern exists*
-As a first step, you should ensure the correct index pattern exists.
-Open the main menu, then click *Stack Management > Index Patterns*.
-In the pattern list, you should see an apm index pattern; The default is `apm-*`.
-If you don't, the index pattern doesn't exist. See <<no-apm-data-found>> for information on how to fix this problem.
+*Ensure an APM data view exists*
+As a first step, you should ensure the correct data view exists.
+In {kib}, go to *Stack Management* > *Data views*.
+You should see the APM data view--the default is
+`traces-apm*,apm-*,logs-apm*,apm-*,metrics-apm*,apm-*`.
+If you don't, the data view doesn't exist.
+To fix this, navigate to the APM app in {kib} and select *Add data*.
+In the APM tutorial, click *Load Kibana objects* to create the APM data view.
 
-Selecting the `apm-*` index pattern shows a listing of every field defined in the pattern.
+If creating an APM data view doesn't solve the problem,
+see <<no-apm-data-found>> for further troubleshooting.
 
 *Ensure a field is searchable*
 There are two things you can do to if you'd like to ensure a field is searchable:
 
 1. Index your additional data as {apm-guide-ref}/metadata.html[labels] instead.
 These are dynamic by default, which means they will be indexed and become searchable and aggregatable.
 
-2. Use the `append_fields` feature. As an example,
-adding the following to `apm-server.yml` will enable dynamic indexing for `http.request.cookies`:
-
-[source,yml]
-----
-setup.template.enabled: true
-setup.template.overwrite: true
-setup.template.append_fields:
-  - name: http.request.cookies
-    type: object
-    dynamic: true
-----
+2. Create a custom mapping for the field.
+// link will be added in a later PR.
+// docs will be added in https://github.com/elastic/apm-server/pull/6940
 
 [float]
 [[service-map-rum-connections]]

docs/developer/advanced/running-elasticsearch.asciidoc

@@ -71,13 +71,6 @@ elasticsearch.password: {{ password }}
 elasticsearch.ssl.verificationMode: none
 ----
 
-If many other users will be interacting with your remote cluster, you'll want to add the following to avoid causing conflicts:
-
-[source,bash]
-----
-kibana.index: '.{YourGitHubHandle}-kibana'
-----
-
 ==== Running remote clusters
 
 Setup remote clusters for cross cluster search (CCS) and cross cluster replication (CCR).

docs/settings/general-infra-logs-ui-settings.asciidoc

@@ -21,4 +21,7 @@ Field used to identify hosts. Defaults to `host.name`.
 Field used to identify Docker containers. Defaults to `container.id`.
 
 `xpack.infra.sources.default.fields.pod`::
-Field used to identify Kubernetes pods. Defaults to `kubernetes.pod.uid`.
+Field used to identify Kubernetes pods. Defaults to `kubernetes.pod.uid`.
+
+`xpack.infra.alerting.metric_threshold.group_by_page_size`::
+Controls the size of the composite aggregations used by the Metric Threshold group by feature. Defaults to `10000`.

docs/user/ml/index.asciidoc

@@ -47,10 +47,8 @@ pane:
 [role="screenshot"]
 image::user/ml/images/ml-job-management.png[Job Management]
 
-You can use the *Settings* pane to create and edit
-{ml-docs}/ml-ad-finding-anomalies.html#ml-ad-calendars[calendars] and the
-filters that are used in
-{ml-docs}/ml-ad-finding-anomalies.html#ml-ad-rules[custom rules]:
+You can use the *Settings* pane to create and edit calendars and the
+filters that are used in custom rules:
 
 [role="screenshot"]
 image::user/ml/images/ml-settings.png[Calendar Management]

package.json

@@ -227,7 +227,7 @@
     "deep-freeze-strict": "^1.1.1",
     "deepmerge": "^4.2.2",
     "del": "^5.1.0",
-    "elastic-apm-node": "^3.26.0",
+    "elastic-apm-node": "^3.27.0",
     "execa": "^4.0.2",
     "exit-hook": "^2.2.0",
     "expiry-js": "0.1.7",
@@ -411,7 +411,6 @@
     "venn.js": "0.2.20",
     "vinyl": "^2.2.0",
     "vt-pbf": "^3.1.1",
-    "wellknown": "^0.5.0",
     "whatwg-fetch": "^3.0.0",
     "xml2js": "^0.4.22",
     "yauzl": "^2.10.0"

packages/kbn-dev-utils/src/serializers/recursive_serializer.ts

@@ -6,11 +6,26 @@
  * Side Public License, v 1.
  */
 
-export function createRecursiveSerializer(test: (v: any) => boolean, print: (v: any) => string) {
+class RawPrint {
+  static fromString(s: string) {
+    return new RawPrint(s);
+  }
+  constructor(public readonly v: string) {}
+}
+
+export function createRecursiveSerializer(
+  test: (v: any) => boolean,
+  print: (v: any, printRaw: (v: string) => RawPrint) => string | RawPrint
+) {
   return {
     test: (v: any) => test(v),
     serialize: (v: any, ...rest: any[]) => {
-      const replacement = print(v);
+      const replacement = print(v, RawPrint.fromString);
+
+      if (replacement instanceof RawPrint) {
+        return replacement.v;
+      }
+
       const printer = rest.pop()!;
       return printer(replacement, ...rest);
     },

packages/kbn-es-archiver/src/cli.ts

@@ -18,7 +18,7 @@ import readline from 'readline';
 import Fs from 'fs';
 
 import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils';
-import { readConfigFile, KbnClient } from '@kbn/test';
+import { readConfigFile, KbnClient, EsVersion } from '@kbn/test';
 import { Client, HttpConnection } from '@elastic/elasticsearch';
 
 import { EsArchiver } from './es_archiver';
@@ -45,7 +45,7 @@ export function runCli() {
       if (typeof configPath !== 'string') {
         throw createFlagError('--config must be a string');
       }
-      const config = await readConfigFile(log, Path.resolve(configPath));
+      const config = await readConfigFile(log, EsVersion.getDefault(), Path.resolve(configPath));
       statsMeta.set('ftrConfigPath', configPath);
 
       let esUrl = flags['es-url'];