Merge branch 'master' into apm-removing-rate-aggs

.buildkite/scripts/common/env.sh

@@ -36,7 +36,12 @@ export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 
 if is_pr; then
-  export ELASTIC_APM_ACTIVE=false
+  if [[ "${GITHUB_PR_LABELS:-}" == *"ci:collect-apm"* ]]; then
+    export ELASTIC_APM_ACTIVE=true
+  else
+    export ELASTIC_APM_ACTIVE=false
+  fi
+
   export CHECKS_REPORTER_ACTIVE=true
 
   # These can be removed once we're not supporting Jenkins and Buildkite at the same time

.buildkite/scripts/common/persist_bazel_cache.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common/util.sh
+
+KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
+export KIBANA_BUILDBUDDY_CI_API_KEY
+
+# overwrites the file checkout .bazelrc file with the one intended for CI env
+cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$KIBANA_DIR/.bazelrc"
+
+###
+### append auth token to buildbuddy into "$KIBANA_DIR/.bazelrc";
+###
+echo "# Appended by .buildkite/scripts/persist_bazel_cache.sh" >> "$KIBANA_DIR/.bazelrc"
+echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$KIBANA_DIR/.bazelrc"

.buildkite/scripts/lifecycle/pre_command.sh

@@ -4,16 +4,17 @@ set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
 
-node .buildkite/scripts/lifecycle/print_agent_links.js || true
-
-echo '--- Job Environment Setup'
+BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+export BUILDKITE_TOKEN
 
+echo '--- Install buildkite dependencies'
 cd '.buildkite'
 retry 5 15 yarn install
 cd -
 
-BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
-export BUILDKITE_TOKEN
+node .buildkite/scripts/lifecycle/print_agent_links.js || true
+
+echo '--- Job Environment Setup'
 
 # Set up a custom ES Snapshot Manifest if one has been specified for this build
 {

.buildkite/scripts/steps/on_merge_build_and_metrics.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 # Write Bazel cache for Linux
-.buildkite/scripts/common/setup_bazel.sh
+.buildkite/scripts/common/persist_bazel_cache.sh
 
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/build_kibana.sh

.github/CODEOWNERS

@@ -63,7 +63,7 @@
 /packages/kbn-interpreter/ @elastic/kibana-app-services
 /src/plugins/bfetch/ @elastic/kibana-app-services
 /src/plugins/data/ @elastic/kibana-app-services
-/src/plugins/data-views/ @elastic/kibana-app-services
+/src/plugins/data_views/ @elastic/kibana-app-services
 /src/plugins/embeddable/ @elastic/kibana-app-services
 /src/plugins/expressions/ @elastic/kibana-app-services
 /src/plugins/field_formats/ @elastic/kibana-app-services
@@ -254,7 +254,6 @@
 /src/plugins/kibana_overview/ @elastic/kibana-core
 /x-pack/plugins/global_search_bar/ @elastic/kibana-core
 #CC# /src/core/server/csp/ @elastic/kibana-core
-#CC# /src/plugins/xpack_legacy/ @elastic/kibana-core
 #CC# /src/plugins/saved_objects/ @elastic/kibana-core
 #CC# /x-pack/plugins/cloud/ @elastic/kibana-core
 #CC# /x-pack/plugins/features/ @elastic/kibana-core

api_docs/charts.json

@@ -229,7 +229,7 @@
           ", xAccessor: string | number | ",
           "AccessorFn",
           ") => ({ x: selectedRange }: ",
-          "XYBrushArea",
+          "XYBrushEvent",
           ") => ",
           {
             "pluginId": "charts",
@@ -4266,4 +4266,4 @@
       }
     ]
   }
-}
+}

api_docs/plugin_directory.mdx

@@ -153,7 +153,6 @@ warning: This document is auto-generated and is meant to be viewed inside our ex
 | <DocLink id="kibVisualizationsPluginApi" text="visualizations"/> | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | Contains the shared architecture among all the legacy visualizations, e.g. the visualization type registry or the visualization embeddable. | 304 | 13 | 286 | 16 |
 | <DocLink id="kibVisualizePluginApi" text="visualize"/> | [Vis Editors](https://github.com/orgs/elastic/teams/kibana-vis-editors) | Contains the visualize application which includes the listing page and the app frame, which will load the visualization's editor. | 24 | 0 | 23 | 1 |
 | watcher | [Stack Management](https://github.com/orgs/elastic/teams/kibana-stack-management) | - | 0 | 0 | 0 | 0 |
-| xpackLegacy | [Kibana Core](https://github.com/orgs/elastic/teams/kibana-core) | - | 0 | 0 | 0 | 0 |
 
 ## Package Directory
 

dev_docs/key_concepts/kibana_platform_plugin_intro.mdx

@@ -33,13 +33,28 @@ At a super high-level, Kibana is composed of **plugins**, **core**, and **Kibana
 
 <DocAccordion buttonContent="(Internal only) FAQ: Should I put my code in a plugin or a package?" color="warning">
 <DocCallOut color="warning">
-If it's stateful, it has to go in a plugin, but packages are often a good choices for stateless utilities. Stateless code exported publicly from a plugin will increase the page load bundle size of _every single page_, even if none of those plugin's services are actually needed. With packages, however, only code that is needed for the current page is downloaded.
+When the [Bazel migration](https://github.com/elastic/kibana/blob/master/legacy_rfcs/text/0015_bazel.md) is complete, all code, including plugins, will be a package. With that, packages won't be required to be in the `packages/` directory and can be located somewhere that makes more sense structurally.
 
-The downside however is that the packages folder is far away from the plugins folder so having a part of your code in a plugin and the rest in a package may make it hard to find, leading to duplication.
+In the meantime, the following can be used to determine whether it makes sense to add code to a package inside the `packages` folder, or a plugin inside `src/plugins` or `x-pack/plugins`. 
 
-The Operations team hopes to resolve this conundrum by supporting co-located packages and plugins and automatically putting all stateless code inside a package. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886).
 
-Until then, consider whether it makes sense to logically separate the code, and consider the size of the exports, when determining whether you should put stateless public exports in a package or a plugin.
+**If the code is stateful, it has to be exposed from a plugin's <DocLink id="kibPlatformIntro" section="lifecycle-methods" text="lifecycle methods"/>. Do not statically export stateful code.**
+
+Benefits to packages:
+
+1. <b>_Potentially_ reduced page load time</b>. All code that is statically exported from plugins will be downloaded on _every single page load_, even if that code isn't needed. With packages, only code that is imported is downloaded, which can be minimized by using async imports.
+2. <b>Puts the consumer is in charge of how and when to async import</b>. If a consumer async imports code exported from a plugin, it makes no difference, because of the above point. It's already been downloaded. However, simply moving code into a package is _not_ a guaranteed performance improvement. It does give the consumer the power to make smart performance choices, however. If they require code from multiple packages, the consumer can async import from multiple packages at the same time. Read more in our <DocLink id="kibDevPerformance" text="performance docs"/>.
+
+Downsides to packages:
+
+1. <b>It's not <DocLink id="kibDevPrinciples" text="organized by domain"/></b>. The packages folder is far away from the plugins folder. Having your stateless code in a plugin and the rest in a package may make it hard to find, leading to duplication. The Operations team hopes to fix this by supporting packages and plugins existing in the same folder. You can track this work by following [this issue](https://github.com/elastic/kibana/issues/112886).
+
+2. <b>Development overhead</b>. Developers have to run `yarn kbn watch` to have changes rebuilt automatically. [Phase II](https://github.com/elastic/kibana/blob/master/legacy_rfcs/text/0015_bazel.md#phase-ii---docs-developer-experience) of the Bazel migration work will bring the development experience on par with plugin development. This work can be tracked [here](https://github.com/elastic/kibana/issues/104519). 
+
+3. <b>Development performance</b>. Rebuild time is typically longer than it would be for the same code in a plugin. The reasons are captured in [this issue](https://github.com/elastic/kibana/issues/107648). The ops team is actively working to reduce this performance increase.
+
+
+As you can see, the answer to "Should I put my code in a plugin or a package" is 'It Depends'. If you are still having a hard time determining what the best path location is, reach out to the Kibana Operations Team (#kibana-operations) for help.
 
 </DocCallOut>
 </DocAccordion>

docs/api/logstash-configuration-management/create-logstash.asciidoc

@@ -27,7 +27,16 @@ experimental[] Create a centrally-managed Logstash pipeline, or update an existi
   (Required, string) The pipeline definition.
 
 `settings`::
-  (Optional, object) The pipeline settings. Supported settings, represented as object keys, are `pipeline.workers`, `pipeline.batch.size`, `pipeline.batch.delay`, `queue.type`, `queue.max_bytes`, and `queue.checkpoint.writes`.
+  (Optional, object) The pipeline settings. Supported settings, represented as object keys, include the following:
+
+  * `pipeline.workers`
+  * `pipeline.batch.size`
+  * `pipeline.batch.delay`
+  * `pipeline.ecs_compatibility`
+  * `pipeline.ordered`
+  * `queue.type`
+  * `queue.max_bytes`
+  * `queue.checkpoint.writes`
 
 [[logstash-configuration-management-api-create-codes]]
 ==== Response code

docs/api/short-urls.asciidoc

@@ -0,0 +1,9 @@
+[[short-urls-api]]
+== Short URLs APIs
+
+Manage {kib} short URLs.
+
+include::short-urls/create-short-url.asciidoc[]
+include::short-urls/get-short-url.asciidoc[]
+include::short-urls/delete-short-url.asciidoc[]
+include::short-urls/resolve-short-url.asciidoc[]

docs/api/short-urls/create-short-url.asciidoc

@@ -0,0 +1,86 @@
+[[short-urls-api-create]]
+=== Create short URL API
+++++
+<titleabbrev>Create short URL</titleabbrev>
+++++
+
+experimental[] Create a {kib} short URL. {kib} URLs may be long and cumbersome, short URLs are much
+easier to remember and share.
+
+Short URLs are created by specifying the locator ID and locator parameters. When a short URL is
+resolved, the locator ID and locator parameters are used to redirect user to the right {kib} page.
+
+
+[[short-urls-api-create-request]]
+==== Request
+
+`POST <kibana host>:<port>/api/short_url`
+
+
+[[short-urls-api-create-request-body]]
+==== Request body
+
+`locatorId`::
+  (Required, string) ID of the locator.
+
+`params`::
+  (Required, object) Object which contains all necessary parameters for the given locator to resolve
+  to a {kib} location.
++
+WARNING: When you create a short URL, locator params are not validated, which allows you to pass
+arbitrary and ill-formed data into the API that can break {kib}. Make sure
+any data that you send to the API is properly formed.
+
+`slug`::
+  (Optional, string) A custom short URL slug. Slug is the part of the short URL that identifies it.
+  You can provide a custom slug which consists of latin alphabet letters, numbers and `-._`
+  characters. The slug must be at least 3 characters long, but no longer than 255 characters.
+
+`humanReadableSlug`::
+  (Optional, boolean) When the `slug` parameter is omitted, the API will generate a random
+  human-readable slug, if `humanReadableSlug` is set to `true`.
+
+
+[[short-urls-api-create-response-codes]]
+==== Response code
+
+`200`::
+    Indicates a successful call.
+
+
+[[short-urls-api-create-example]]
+==== Example
+
+[source,sh]
+--------------------------------------------------
+$ curl -X POST api/short_url  -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '
+{
+  "locatorId": "LOCATOR_ID",
+  "params": {},
+  "humanReadableSlug": true
+}'
+--------------------------------------------------
+// KIBANA
+
+The API returns the following:
+
+[source,sh]
+--------------------------------------------------
+{
+  "id": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", <1>
+  "slug": "adjective-adjective-noun", <2>
+  "locator": {
+    "id": "LOCATOR_ID",
+    "version": "x.x.x", <3>
+    "state": {} <4>
+  },
+  "accessCount": 0,
+  "accessDate": 1632680100000,
+  "createDate": 1632680100000
+}
+--------------------------------------------------
+
+<1> A random ID is automatically generated.
+<2> A random human-readable slug is automatically generated if the `humanReadableSlug` parameter is set to `true`. If set to `false` a random short string is generated.
+<3> The version of {kib} when short URL was created is stored.
+<4> Locator params provided as `params` property are stored.

docs/api/short-urls/delete-short-url.asciidoc

@@ -0,0 +1,39 @@
+[[short-urls-api-delete]]
+=== Delete short URL API
+++++
+<titleabbrev>Delete short URL</titleabbrev>
+++++
+
+experimental[] Delete a {kib} short URL.
+
+
+[[short-urls-api-delete-request]]
+==== Request
+
+`DELETE <kibana host>:<port>/api/short_url/<id>`
+
+
+[[short-urls-api-delete-path-params]]
+==== Path parameters
+
+`id`::
+  (Required, string) The short URL ID that you want to remove.
+
+
+[[short-urls-api-delete-response-codes]]
+==== Response code
+
+`200`::
+    Indicates a successful call.
+
+
+[[short-urls-api-delete-example]]
+==== Example
+
+Delete a short URL `12345` ID:
+
+[source,sh]
+--------------------------------------------------
+$ curl -X DELETE api/short_url/12345
+--------------------------------------------------
+// KIBANA

docs/api/short-urls/get-short-url.asciidoc

@@ -0,0 +1,56 @@
+[[short-urls-api-get]]
+=== Get short URL API
+++++
+<titleabbrev>Get short URL</titleabbrev>
+++++
+
+experimental[] Retrieve a single {kib} short URL.
+
+[[short-urls-api-get-request]]
+==== Request
+
+`GET <kibana host>:<port>/api/short_url/<id>`
+
+
+[[short-urls-api-get-params]]
+==== Path parameters
+
+`id`::
+  (Required, string) The ID of the short URL.
+
+
+[[short-urls-api-get-codes]]
+==== Response code
+
+`200`::
+    Indicates a successful call.
+
+
+[[short-urls-api-get-example]]
+==== Example
+
+Retrieve the short URL with the `12345` ID:
+
+[source,sh]
+--------------------------------------------------
+$ curl -X GET api/short_url/12345
+--------------------------------------------------
+// KIBANA
+
+The API returns the following:
+
+[source,sh]
+--------------------------------------------------
+{
+  "id": "12345",
+  "slug": "adjective-adjective-noun",
+  "locator": {
+    "id": "LOCATOR_ID",
+    "version": "x.x.x",
+    "state": {}
+  },
+  "accessCount": 0,
+  "accessDate": 1632680100000,
+  "createDate": 1632680100000
+}
+--------------------------------------------------