Merge branch 'master' into upgrade-versions

.github/CODEOWNERS

@@ -95,15 +95,15 @@
 # Observability Shared
 /x-pack/plugins/observability/ @elastic/observability-ui
 
-# Logs and Metrics (Infra)
-/x-pack/plugins/infra/ @elastic/logs-metrics-ui
-/x-pack/plugins/infra/server/routes/log* @elastic/logs-metrics-ui @elastic/logs-ui
-/x-pack/plugins/infra/public/pages/logs/ @elastic/logs-metrics-ui @elastic/logs-ui
-/x-pack/plugins/infra/server/routes/metrics* @elastic/logs-metrics-ui @elastic/metrics-ui
-/x-pack/plugins/infra/public/pages/metrics/ @elastic/logs-metrics-ui @elastic/metrics-ui
-
-# Stack Monitoring
-/x-pack/plugins/monitoring/ @elastic/stack-monitoring-ui @elastic/logs-metrics-ui @elastic/metrics-ui
+# Infra Monitoring
+/x-pack/plugins/infra/ @elastic/infra-monitoring-ui
+/x-pack/test/functional/apps/infra @elastic/infra-monitoring-ui
+/x-pack/test/api_integration/apis/infra @elastic/infra-monitoring-ui
+
+# Elastic Stack Monitoring
+/x-pack/plugins/monitoring/ @elastic/infra-monitoring-ui
+/x-pack/test/functional/apps/monitoring @elastic/infra-monitoring-ui
+/x-pack/test/api_integration/apis/monitoring @elastic/infra-monitoring-ui
 
 # Fleet
 /x-pack/plugins/fleet/ @elastic/fleet

config/kibana.yml

@@ -31,10 +31,6 @@
 # The URLs of the Elasticsearch instances to use for all your queries.
 #elasticsearch.hosts: ["http://localhost:9200"]
 
-# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
-# dashboards. Kibana creates a new index if the index doesn't already exist.
-#kibana.index: ".kibana"
-
 # If your Elasticsearch is protected with basic authentication, these settings provide
 # the username and password that the Kibana server uses to perform maintenance on the Kibana
 # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which

docs/developer/advanced/running-elasticsearch.asciidoc

@@ -76,7 +76,6 @@ If many other users will be interacting with your remote cluster, you'll want to
 [source,bash]
 ----
 kibana.index: '.{YourGitHubHandle}-kibana'
-xpack.task_manager.index: '.{YourGitHubHandle}-task-manager-kibana'
 ----
 
 ==== Running remote clusters

docs/development/core/server/kibana-plugin-core-server.elasticsearchclient.md

@@ -9,9 +9,9 @@ Client used to query the elasticsearch cluster.
 <b>Signature:</b>
 
 ```typescript
-export declare type ElasticsearchClient = Omit<KibanaClient, 'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close'> & {
+export declare type ElasticsearchClient = Omit<KibanaClient, 'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close' | 'diagnostic'> & {
     transport: {
-        request(params: TransportRequestParams, options?: TransportRequestOptions): TransportRequestPromise<ApiResponse>;
+        request<TResponse = unknown>(params: TransportRequestParams, options?: TransportRequestOptions): Promise<TransportResult<TResponse>>;
     };
 };
 ```

docs/development/core/server/kibana-plugin-core-server.elasticsearcherrordetails.error.md

@@ -0,0 +1,14 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [ElasticsearchErrorDetails](./kibana-plugin-core-server.elasticsearcherrordetails.md) &gt; [error](./kibana-plugin-core-server.elasticsearcherrordetails.error.md)
+
+## ElasticsearchErrorDetails.error property
+
+<b>Signature:</b>
+
+```typescript
+error?: {
+        type: string;
+        reason?: string;
+    };
+```

docs/development/core/server/kibana-plugin-core-server.elasticsearcherrordetails.md

@@ -0,0 +1,19 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [ElasticsearchErrorDetails](./kibana-plugin-core-server.elasticsearcherrordetails.md)
+
+## ElasticsearchErrorDetails interface
+
+
+<b>Signature:</b>
+
+```typescript
+export interface ElasticsearchErrorDetails 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [error](./kibana-plugin-core-server.elasticsearcherrordetails.error.md) | <code>{</code><br/><code>        type: string;</code><br/><code>        reason?: string;</code><br/><code>    }</code> |  |
+

docs/development/core/server/kibana-plugin-core-server.md

@@ -71,6 +71,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [DeprecationsServiceSetup](./kibana-plugin-core-server.deprecationsservicesetup.md) | The deprecations service provides a way for the Kibana platform to communicate deprecated features and configs with its users. These deprecations are only communicated if the deployment is using these features. Allowing for a user tailored experience for upgrading the stack version.<!-- -->The Deprecation service is consumed by the upgrade assistant to assist with the upgrade experience.<!-- -->If a deprecated feature can be resolved without manual user intervention. Using correctiveActions.api allows the Upgrade Assistant to use this api to correct the deprecation upon a user trigger. |
 |  [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. |
 |  [ElasticsearchConfigPreboot](./kibana-plugin-core-server.elasticsearchconfigpreboot.md) | A limited set of Elasticsearch configuration entries exposed to the <code>preboot</code> plugins at <code>setup</code>. |
+|  [ElasticsearchErrorDetails](./kibana-plugin-core-server.elasticsearcherrordetails.md) |  |
 |  [ElasticsearchServicePreboot](./kibana-plugin-core-server.elasticsearchservicepreboot.md) |  |
 |  [ElasticsearchServiceSetup](./kibana-plugin-core-server.elasticsearchservicesetup.md) |  |
 |  [ElasticsearchServiceStart](./kibana-plugin-core-server.elasticsearchservicestart.md) |  |

docs/development/core/server/kibana-plugin-core-server.savedobjectserrorhelpers.md

@@ -18,7 +18,6 @@ export declare class SavedObjectsErrorHelpers
 |  [createBadRequestError(reason)](./kibana-plugin-core-server.savedobjectserrorhelpers.createbadrequesterror.md) | <code>static</code> |  |
 |  [createConflictError(type, id, reason)](./kibana-plugin-core-server.savedobjectserrorhelpers.createconflicterror.md) | <code>static</code> |  |
 |  [createGenericNotFoundError(type, id)](./kibana-plugin-core-server.savedobjectserrorhelpers.creategenericnotfounderror.md) | <code>static</code> |  |
-|  [createGenericNotFoundEsUnavailableError(type, id)](./kibana-plugin-core-server.savedobjectserrorhelpers.creategenericnotfoundesunavailableerror.md) | <code>static</code> |  |
 |  [createIndexAliasNotFoundError(alias)](./kibana-plugin-core-server.savedobjectserrorhelpers.createindexaliasnotfounderror.md) | <code>static</code> |  |
 |  [createInvalidVersionError(versionInput)](./kibana-plugin-core-server.savedobjectserrorhelpers.createinvalidversionerror.md) | <code>static</code> |  |
 |  [createTooManyRequestsError(type, id)](./kibana-plugin-core-server.savedobjectserrorhelpers.createtoomanyrequestserror.md) | <code>static</code> |  |

docs/development/core/server/kibana-plugin-core-server.savedobjectsservicesetup.getkibanaindex.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [SavedObjectsServiceSetup](./kibana-plugin-core-server.savedobjectsservicesetup.md) &gt; [getKibanaIndex](./kibana-plugin-core-server.savedobjectsservicesetup.getkibanaindex.md)
+
+## SavedObjectsServiceSetup.getKibanaIndex property
+
+Returns the default index used for saved objects.
+
+<b>Signature:</b>
+
+```typescript
+getKibanaIndex: () => string;
+```

docs/development/core/server/kibana-plugin-core-server.savedobjectsservicesetup.md

@@ -52,6 +52,7 @@ export class Plugin() {
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [addClientWrapper](./kibana-plugin-core-server.savedobjectsservicesetup.addclientwrapper.md) | <code>(priority: number, id: string, factory: SavedObjectsClientWrapperFactory) =&gt; void</code> | Add a [client wrapper factory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) with the given priority. |
+|  [getKibanaIndex](./kibana-plugin-core-server.savedobjectsservicesetup.getkibanaindex.md) | <code>() =&gt; string</code> | Returns the default index used for saved objects. |
 |  [registerType](./kibana-plugin-core-server.savedobjectsservicesetup.registertype.md) | <code>&lt;Attributes = any&gt;(type: SavedObjectsType&lt;Attributes&gt;) =&gt; void</code> | Register a [savedObjects type](./kibana-plugin-core-server.savedobjectstype.md) definition.<!-- -->See the [mappings format](./kibana-plugin-core-server.savedobjectstypemappingdefinition.md) and [migration format](./kibana-plugin-core-server.savedobjectmigrationmap.md) for more details about these. |
 |  [setClientFactoryProvider](./kibana-plugin-core-server.savedobjectsservicesetup.setclientfactoryprovider.md) | <code>(clientFactoryProvider: SavedObjectsClientFactoryProvider) =&gt; void</code> | Set the default [factory provider](./kibana-plugin-core-server.savedobjectsclientfactoryprovider.md) for creating Saved Objects clients. Only one provider can be set, subsequent calls to this method will fail. |
 

docs/development/core/server/kibana-plugin-core-server.sharedglobalconfig.md

@@ -9,7 +9,6 @@
 
 ```typescript
 export declare type SharedGlobalConfig = RecursiveReadonly<{
-    kibana: Pick<KibanaConfigType, typeof SharedGlobalConfigKeys.kibana[number]>;
     elasticsearch: Pick<ElasticsearchConfigType, typeof SharedGlobalConfigKeys.elasticsearch[number]>;
     path: Pick<PathConfigType, typeof SharedGlobalConfigKeys.path[number]>;
     savedObjects: Pick<SavedObjectsConfigType, typeof SharedGlobalConfigKeys.savedObjects[number]>;

docs/settings/task-manager-settings.asciidoc

@@ -22,9 +22,6 @@ Task Manager runs background tasks by polling for work on an interval.  You can
 | `xpack.task_manager.request_capacity`
   | How many requests can Task Manager buffer before it rejects new requests.  Defaults to 1000.
 
-| `xpack.task_manager.index`
-  | The name of the index used to store task information.  Defaults to `.kibana_task_manager`.
-
   | `xpack.task_manager.max_workers`
   | The maximum number of tasks that this Kibana instance will run simultaneously.  Defaults to 10.
     Starting in 8.0, it will not be possible to set the value greater than 100.

docs/setup/settings.asciidoc

@@ -285,18 +285,6 @@ is an alternative to `elasticsearch.username` and `elasticsearch.password`.
 | `interpreter.enableInVisualize`
   | Enables use of interpreter in Visualize. *Default: `true`*
 
-|[[kibana-index]] `kibana.index:`
- | deprecated:[7.11.0,This setting will be removed in 8.0.] Multitenancy by
- changing `kibana.index` will not be supported starting in 8.0. See
- https://ela.st/kbn-remove-legacy-multitenancy[8.0 Breaking Changes] for more
- details.
- +
- {kib} uses an index in {es} to store saved searches, visualizations, and
- dashboards. {kib} creates a new index if the index doesn’t already exist. If
- you configure a custom index, the name must be lowercase, and conform to the
- {es} {ref}/indices-create-index.html[index name limitations].
- *Default: `".kibana"`*
-
 | `data.autocomplete.valueSuggestions.timeout:` {ess-icon}
  | Time in milliseconds to wait for autocomplete suggestions from {es}.
 This value must be a whole number greater than zero. *Default: `"1000"`*

docs/setup/upgrade/upgrade-migrations.asciidoc

@@ -16,8 +16,8 @@ WARNING: The following instructions assumes {kib} is using the default index nam
 
 Saved objects are stored in two indices: 
 
-* `.kibana_{kibana_version}_001`, or if the `kibana.index` configuration setting is set `.{kibana.index}_{kibana_version}_001`. E.g. for Kibana v7.12.0 `.kibana_7.12.0_001`.
-* `.kibana_task_manager_{kibana_version}_001`, or if the `xpack.tasks.index` configuration setting is set `.{xpack.tasks.index}_{kibana_version}_001` E.g. for Kibana v7.12.0 `.kibana_task_manager_7.12.0_001`.
+* `.kibana_{kibana_version}_001`, e.g. for Kibana v7.12.0 `.kibana_7.12.0_001`.
+* `.kibana_task_manager_{kibana_version}_001`, e.g. for Kibana v7.12.0 `.kibana_task_manager_7.12.0_001`.
 
 The index aliases `.kibana` and `.kibana_task_manager` will always point to
 the most up-to-date saved object indices.

docs/user/production-considerations/task-manager-production-considerations.asciidoc

@@ -12,7 +12,7 @@ This has three major benefits:
 
 [IMPORTANT]
 ==============================================
-Task definitions for alerts and actions are stored in the index specified by <<task-manager-settings, `xpack.task_manager.index`>>. The default is `.kibana_task_manager`.
+Task definitions for alerts and actions are stored in the index called `.kibana_task_manager`.
  
 You must have at least one replica of this index for production deployments.
 

package.json

@@ -100,7 +100,7 @@
     "@elastic/apm-rum-react": "^1.3.1",
     "@elastic/charts": "38.0.1",
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
-    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.21",
+    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.35",
     "@elastic/ems-client": "7.16.0",
     "@elastic/eui": "40.0.0",
     "@elastic/filesaver": "1.1.2",

packages/elastic-apm-synthtrace/README.md

@@ -93,23 +93,24 @@ const esEvents = toElasticsearchOutput([
 
 Via the CLI, you can upload scenarios, either using a fixed time range or continuously generating data. Some examples are available in in `src/scripts/examples`. Here's an example for live data:
 
-`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
+`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --live`
 
 For a fixed time window:
-`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-synthtrace/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
+`$ node packages/elastic-apm-synthtrace/src/scripts/run packages/elastic-apm-generator/src/examples/01_simple_trace.ts --target=http://admin:changeme@localhost:9200 --from=now-24h --to=now`
 
-The script will try to automatically find bootstrapped APM indices. **If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.**
+The script will try to automatically find bootstrapped APM indices. __If these indices do not exist, the script will exit with an error. It will not bootstrap the indices itself.__
 
 The following options are supported:
-| Option | Description | Default |
-| -------------- | ------------------------------------------------------- | ------------ |
-| `--from` | The start of the time window. | `now - 15m` |
-| `--to` | The end of the time window. | `now` |
-| `--live` | Continously ingest data | `false` |
-| `--bucketSize` | Size of bucket for which to generate data. | `15m` |
-| `--clean` | Clean APM indices before indexing new data. | `false` |
-| `--interval` | The interval at which to index data. | `10s` |
-| `--logLevel` | Log level. | `info` |
-| `--lookback` | The lookback window for which data should be generated. | `15m` |
-| `--target` | Elasticsearch target, including username/password. | **Required** |
-| `--workers` | Amount of simultaneously connected ES clients. | `1` |
+| Option            | Description                                             | Default      |
+| ------------------| ------------------------------------------------------- | ------------ |
+| `--target`        | Elasticsearch target, including username/password.      | **Required** |
+| `--from`          | The start of the time window.                           | `now - 15m`  |
+| `--to`            | The end of the time window.                             | `now`        |
+| `--live`          | Continously ingest data                                 | `false`      |
+| `--clean`         | Clean APM indices before indexing new data.             | `false`      |
+| `--workers`       | Amount of Node.js worker threads                        | `5`          |
+| `--bucketSize`    | Size of bucket for which to generate data.              | `15m`        |
+| `--interval`      | The interval at which to index data.                    | `10s`        |
+| `--clientWorkers` | Number of simultaneously connected ES clients           | `5`          |
+| `--batchSize`     | Number of documents per bulk index request              | `1000`       |
+| `--logLevel`      | Log level.                                              | `info`       |

packages/elastic-apm-synthtrace/src/lib/output/to_elasticsearch_output.ts

@@ -22,6 +22,18 @@ export interface ElasticsearchOutputWriteTargets {
   metric: string;
 }
 
+const observerDefaults = getObserverDefaults();
+
+const esDocumentDefaults = {
+  ecs: {
+    version: '1.4',
+  },
+};
+
+// eslint-disable-next-line guard-for-in
+for (const key in observerDefaults) {
+  set(esDocumentDefaults, key, observerDefaults[key as keyof typeof observerDefaults]);
+}
 export function toElasticsearchOutput({
   events,
   writeTargets,
@@ -30,22 +42,25 @@ export function toElasticsearchOutput({
   writeTargets: ElasticsearchOutputWriteTargets;
 }): ElasticsearchOutput[] {
   return events.map((event) => {
-    const values = {
-      ...event,
-      ...getObserverDefaults(),
+    const values = {};
+
+    Object.assign(values, event, {
       '@timestamp': new Date(event['@timestamp']!).toISOString(),
       'timestamp.us': event['@timestamp']! * 1000,
-      'ecs.version': '1.4',
       'service.node.name':
         event['service.node.name'] || event['container.id'] || event['host.name'],
-    };
+    });
 
     const document = {};
+
+    Object.assign(document, esDocumentDefaults);
+
     // eslint-disable-next-line guard-for-in
     for (const key in values) {
       const val = values[key as keyof typeof values];
       set(document, key, val);
     }
+
     return {
       _index: writeTargets[event['processor.event'] as keyof ElasticsearchOutputWriteTargets],
       _source: document,

packages/elastic-apm-synthtrace/src/lib/utils/generate_id.ts

@@ -6,20 +6,17 @@
  * Side Public License, v 1.
  */
 
-import uuidv5 from 'uuid/v5';
-
 let seq = 0;
 
-const namespace = 'f38d5b83-8eee-4f5b-9aa6-2107e15a71e3';
-
-function generateId(seed?: string) {
-  return uuidv5(seed ?? String(seq++), namespace).replace(/-/g, '');
+function generateId(seed?: string, length: number = 32) {
+  const str = seed ?? String(seq++);
+  return str.padStart(length, '0');
 }
 
 export function generateShortId(seed?: string) {
-  return generateId(seed).substr(0, 16);
+  return generateId(seed, 16);
 }
 
 export function generateLongId(seed?: string) {
-  return generateId(seed).substr(0, 32);
+  return generateId(seed, 32);
 }