Merge remote-tracking branch 'upstream/master' into rules_migrate_on_…

…save

.buildkite/pipelines/performance/nightly.yml

@@ -0,0 +1,35 @@
+steps:
+  - block: ":gear: Performance Tests Configuration"
+    prompt: "Fill out the details for performance test"
+    fields:
+      - text: ":arrows_counterclockwise: Iterations"
+        key: "performance-test-iteration-count"
+        hint: "How many times you want to run tests? "
+        required: true
+    if: build.env('ITERATION_COUNT_ENV') == null
+
+  - label: ":male-mechanic::skin-tone-2: Pre-Build"
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+
+  - wait
+
+  - label: ":factory_worker: Build Kibana Distribution and Plugins"
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      queue: c2-16
+    key: build
+
+  - label: ":muscle: Performance Tests"
+    command: .buildkite/scripts/steps/functional/performance.sh
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    concurrency: 50
+    concurrency_group: 'performance-test-group'
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ":male_superhero::skin-tone-2: Post-Build"
+    command: .buildkite/scripts/lifecycle/post_build.sh
+

.buildkite/scripts/steps/functional/performance.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -uo pipefail
+
+if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
+  ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
+else
+  ITERATION_COUNT=$ITERATION_COUNT_ENV
+fi
+
+tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
+
+cat << EOF | buildkite-agent pipeline upload
+steps:
+  - command: .buildkite/scripts/steps/functional/performance_sub.sh
+    parallelism: "$ITERATION_COUNT"
+EOF
+
+
+

.buildkite/scripts/steps/functional/performance_sub.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/download_build_artifacts.sh
+
+cd "$XPACK_DIR"
+
+echo --- Run Performance Tests
+checks-reporter-with-killswitch "Run Performance Tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+    --config test/performance/config.ts;

docs/management/connectors/action-types/pagerduty.asciidoc

@@ -68,7 +68,7 @@ PagerDuty actions have the following properties.
 
 Severity::      The perceived severity of on the affected system. This can be one of `Critical`, `Error`, `Warning` or `Info`(default).
 Event action::  One of `Trigger` (default), `Resolve`, or `Acknowledge`. See https://v2.developer.pagerduty.com/docs/events-api-v2#event-action[event action] for more details.
-Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<alert ID>:<alert instance ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
+Dedup Key::     All actions sharing this key will be associated with the same PagerDuty alert. This value is used to correlate trigger and resolution. This value is *optional*, and if not set, defaults to `<rule ID>:<alert ID>`. The maximum length is *255* characters. See https://v2.developer.pagerduty.com/docs/events-api-v2#alert-de-duplication[alert deduplication] for details. 
 Timestamp::     An *optional* https://v2.developer.pagerduty.com/v2/docs/types#datetime[ISO-8601 format date-time], indicating the time the event was detected or generated.
 Component::     An *optional* value indicating the component of the source machine that is responsible for the event, for example `mysql` or `eth0`.
 Group::         An *optional* value indicating the logical grouping of components of a service, for example `app-stack`.

docs/settings/spaces-settings.asciidoc

@@ -7,11 +7,6 @@
 
 By default, spaces is enabled in {kib}. To secure spaces, <<security-settings-kb,enable security>>.
 
-`xpack.spaces.enabled`::
-deprecated:[7.16.0,"In 8.0 and later, this setting will no longer be supported and it will not be possible to disable this plugin."]
-To enable spaces, set to `true`. 
-The default is `true`.
-
 `xpack.spaces.maxSpaces`::
 The maximum number of spaces that you can use with the {kib} instance. Some {kib} operations
 return all spaces using a single `_search` from {es}, so you must

docs/spaces/index.asciidoc

@@ -109,11 +109,6 @@ image::images/spaces-configure-landing-page.png["Configure space-level landing p
 
 [float]
 [[spaces-delete-started]]
-=== Disable and version updates
-
-Spaces are automatically enabled in {kib}. If you don't want use this feature,
-you can disable it. For more information, refer to <<spaces-settings-kb,Spaces settings in {kib}>>.
-
-When you upgrade {kib}, the default space contains all of your existing saved objects.
-
+=== Disabling spaces
 
+Starting in {kib} 8.0, the Spaces feature cannot be disabled.

packages/kbn-securitysolution-list-constants/src/index.ts

@@ -73,6 +73,6 @@ export const ENDPOINT_EVENT_FILTERS_LIST_DESCRIPTION = 'Endpoint Security Event
 
 export const ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_ID = 'endpoint_host_isolation_exceptions';
 export const ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_NAME =
-  'Endpoint Security Host Isolation Exceptions List';
+  'Endpoint Security Host isolation exceptions List';
 export const ENDPOINT_HOST_ISOLATION_EXCEPTIONS_LIST_DESCRIPTION =
-  'Endpoint Security Host Isolation Exceptions List';
+  'Endpoint Security Host isolation exceptions List';

src/core/server/elasticsearch/elasticsearch_config.test.ts

@@ -322,7 +322,7 @@ describe('deprecations', () => {
     const { messages } = applyElasticsearchDeprecations({ username: 'elastic' });
     expect(messages).toMatchInlineSnapshot(`
       Array [
-        "Setting [${CONFIG_PATH}.username] to \\"elastic\\" is deprecated. You should use the \\"kibana_system\\" user instead.",
+        "Kibana is configured to authenticate to Elasticsearch with the \\"elastic\\" user. Use a service account token instead.",
       ]
     `);
   });
@@ -331,7 +331,7 @@ describe('deprecations', () => {
     const { messages } = applyElasticsearchDeprecations({ username: 'kibana' });
     expect(messages).toMatchInlineSnapshot(`
       Array [
-        "Setting [${CONFIG_PATH}.username] to \\"kibana\\" is deprecated. You should use the \\"kibana_system\\" user instead.",
+        "Kibana is configured to authenticate to Elasticsearch with the \\"kibana\\" user. Use a service account token instead.",
       ]
     `);
   });
@@ -350,7 +350,7 @@ describe('deprecations', () => {
     const { messages } = applyElasticsearchDeprecations({ ssl: { key: '' } });
     expect(messages).toMatchInlineSnapshot(`
       Array [
-        "Setting [${CONFIG_PATH}.ssl.key] without [${CONFIG_PATH}.ssl.certificate] is deprecated. This has no effect, you should use both settings to enable TLS client authentication to Elasticsearch.",
+        "Use both \\"elasticsearch.ssl.key\\" and \\"elasticsearch.ssl.certificate\\" to enable Kibana to use Mutual TLS authentication with Elasticsearch.",
       ]
     `);
   });
@@ -359,7 +359,7 @@ describe('deprecations', () => {
     const { messages } = applyElasticsearchDeprecations({ ssl: { certificate: '' } });
     expect(messages).toMatchInlineSnapshot(`
       Array [
-        "Setting [${CONFIG_PATH}.ssl.certificate] without [${CONFIG_PATH}.ssl.key] is deprecated. This has no effect, you should use both settings to enable TLS client authentication to Elasticsearch.",
+        "Use both \\"elasticsearch.ssl.certificate\\" and \\"elasticsearch.ssl.key\\" to enable Kibana to use Mutual TLS authentication with Elasticsearch.",
       ]
     `);
   });

src/core/server/elasticsearch/elasticsearch_config.ts

@@ -8,6 +8,7 @@
 
 import { schema, TypeOf } from '@kbn/config-schema';
 import { readPkcs12Keystore, readPkcs12Truststore } from '@kbn/crypto';
+import { i18n } from '@kbn/i18n';
 import { Duration } from 'moment';
 import { readFileSync } from 'fs';
 import { ConfigDeprecationProvider } from 'src/core/server';
@@ -171,49 +172,82 @@ export const configSchema = schema.object({
 });
 
 const deprecations: ConfigDeprecationProvider = () => [
-  (settings, fromPath, addDeprecation) => {
+  (settings, fromPath, addDeprecation, { branch }) => {
     const es = settings[fromPath];
     if (!es) {
       return;
     }
-    if (es.username === 'elastic') {
-      addDeprecation({
-        configPath: `${fromPath}.username`,
-        message: `Setting [${fromPath}.username] to "elastic" is deprecated. You should use the "kibana_system" user instead.`,
-        correctiveActions: {
-          manualSteps: [`Replace [${fromPath}.username] from "elastic" to "kibana_system".`],
-        },
-      });
-    } else if (es.username === 'kibana') {
+
+    if (es.username === 'elastic' || es.username === 'kibana') {
+      const username = es.username;
       addDeprecation({
         configPath: `${fromPath}.username`,
-        message: `Setting [${fromPath}.username] to "kibana" is deprecated. You should use the "kibana_system" user instead.`,
-        correctiveActions: {
-          manualSteps: [`Replace [${fromPath}.username] from "kibana" to "kibana_system".`],
-        },
-      });
-    }
-    if (es.ssl?.key !== undefined && es.ssl?.certificate === undefined) {
-      addDeprecation({
-        configPath: `${fromPath}.ssl.key`,
-        message: `Setting [${fromPath}.ssl.key] without [${fromPath}.ssl.certificate] is deprecated. This has no effect, you should use both settings to enable TLS client authentication to Elasticsearch.`,
+        title: i18n.translate('core.deprecations.elasticsearchUsername.title', {
+          defaultMessage: 'Using "elasticsearch.username: {username}" is deprecated',
+          values: { username },
+        }),
+        message: i18n.translate('core.deprecations.elasticsearchUsername.message', {
+          defaultMessage:
+            'Kibana is configured to authenticate to Elasticsearch with the "{username}" user. Use a service account token instead.',
+          values: { username },
+        }),
+        level: 'warning',
+        documentationUrl: `https://www.elastic.co/guide/en/elasticsearch/reference/${branch}/service-accounts.html`,
         correctiveActions: {
           manualSteps: [
-            `Set [${fromPath}.ssl.certificate] in your kibana configs to enable TLS client authentication to Elasticsearch.`,
+            i18n.translate('core.deprecations.elasticsearchUsername.manualSteps1', {
+              defaultMessage:
+                'Use the elasticsearch-service-tokens CLI tool to create a new service account token for the "elastic/kibana" service account.',
+            }),
+            i18n.translate('core.deprecations.elasticsearchUsername.manualSteps2', {
+              defaultMessage: 'Add the "elasticsearch.serviceAccountToken" setting to kibana.yml.',
+            }),
+            i18n.translate('core.deprecations.elasticsearchUsername.manualSteps3', {
+              defaultMessage:
+                'Remove "elasticsearch.username" and "elasticsearch.password" from kibana.yml.',
+            }),
           ],
         },
       });
-    } else if (es.ssl?.certificate !== undefined && es.ssl?.key === undefined) {
+    }
+
+    const addSslDeprecation = (existingSetting: string, missingSetting: string) => {
       addDeprecation({
-        configPath: `${fromPath}.ssl.certificate`,
-        message: `Setting [${fromPath}.ssl.certificate] without [${fromPath}.ssl.key] is deprecated. This has no effect, you should use both settings to enable TLS client authentication to Elasticsearch.`,
+        configPath: existingSetting,
+        title: i18n.translate('core.deprecations.elasticsearchSSL.title', {
+          defaultMessage: 'Using "{existingSetting}" without "{missingSetting}" has no effect',
+          values: { existingSetting, missingSetting },
+        }),
+        message: i18n.translate('core.deprecations.elasticsearchSSL.message', {
+          defaultMessage:
+            'Use both "{existingSetting}" and "{missingSetting}" to enable Kibana to use Mutual TLS authentication with Elasticsearch.',
+          values: { existingSetting, missingSetting },
+        }),
+        level: 'warning',
+        documentationUrl: `https://www.elastic.co/guide/en/kibana/${branch}/elasticsearch-mutual-tls.html`,
         correctiveActions: {
           manualSteps: [
-            `Set [${fromPath}.ssl.key] in your kibana configs to enable TLS client authentication to Elasticsearch.`,
+            i18n.translate('core.deprecations.elasticsearchSSL.manualSteps1', {
+              defaultMessage: 'Add the "{missingSetting}" setting to kibana.yml.',
+              values: { missingSetting },
+            }),
+            i18n.translate('core.deprecations.elasticsearchSSL.manualSteps2', {
+              defaultMessage:
+                'Alternatively, if you don\'t want to use Mutual TLS authentication, remove "{existingSetting}" from kibana.yml.',
+              values: { existingSetting },
+            }),
           ],
         },
       });
-    } else if (es.logQueries === true) {
+    };
+
+    if (es.ssl?.key !== undefined && es.ssl?.certificate === undefined) {
+      addSslDeprecation(`${fromPath}.ssl.key`, `${fromPath}.ssl.certificate`);
+    } else if (es.ssl?.certificate !== undefined && es.ssl?.key === undefined) {
+      addSslDeprecation(`${fromPath}.ssl.certificate`, `${fromPath}.ssl.key`);
+    }
+
+    if (es.logQueries === true) {
       addDeprecation({
         configPath: `${fromPath}.logQueries`,
         message: `Setting [${fromPath}.logQueries] is deprecated and no longer used. You should set the log level to "debug" for the "elasticsearch.queries" context in "logging.loggers".`,

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -366,7 +366,6 @@ kibana_vars=(
     xpack.securitySolution.packagerTaskInterval
     xpack.securitySolution.prebuiltRulesFromFileSystem
     xpack.securitySolution.prebuiltRulesFromSavedObjects
-    xpack.spaces.enabled
     xpack.spaces.maxSpaces
     xpack.task_manager.index
     xpack.task_manager.max_attempts

src/plugins/custom_integrations/server/language_clients/index.ts

@@ -23,27 +23,14 @@ interface LanguageIntegration {
 const ELASTIC_WEBSITE_URL = 'https://www.elastic.co';
 const ELASTICSEARCH_CLIENT_URL = `${ELASTIC_WEBSITE_URL}/guide/en/elasticsearch/client`;
 export const integrations: LanguageIntegration[] = [
-  {
-    id: 'all',
-    title: i18n.translate('customIntegrations.languageclients.AllTitle', {
-      defaultMessage: 'Elasticsearch Clients',
-    }),
-    euiIconName: 'logoElasticsearch',
-    description: i18n.translate('customIntegrations.languageclients.AllDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official language clients.',
-    }),
-    docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/index.html`,
-  },
   {
     id: 'javascript',
     title: i18n.translate('customIntegrations.languageclients.JavascriptTitle', {
       defaultMessage: 'Elasticsearch JavaScript Client',
     }),
     icon: 'nodejs.svg',
     description: i18n.translate('customIntegrations.languageclients.JavascriptDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Node.js client.',
+      defaultMessage: 'Index data to Elasticsearch with the JavaScript client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/javascript-api/{branch}/introduction.html`,
   },
@@ -54,8 +41,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'ruby.svg',
     description: i18n.translate('customIntegrations.languageclients.RubyDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Ruby client.',
+      defaultMessage: 'Index data to Elasticsearch with the Ruby client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/ruby-api/{branch}/ruby_client.html`,
   },
@@ -66,8 +52,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'go.svg',
     description: i18n.translate('customIntegrations.languageclients.GoDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Go client.',
+      defaultMessage: 'Index data to Elasticsearch with the Go client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/go-api/{branch}/overview.html`,
   },
@@ -78,8 +63,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'dotnet.svg',
     description: i18n.translate('customIntegrations.languageclients.DotNetDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official .NET client.',
+      defaultMessage: 'Index data to Elasticsearch with the .NET client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/net-api/{branch}/index.html`,
   },
@@ -90,8 +74,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'php.svg',
     description: i18n.translate('customIntegrations.languageclients.PhpDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official .PHP client.',
+      defaultMessage: 'Index data to Elasticsearch with the PHP client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/php-api/{branch}/index.html`,
   },
@@ -102,8 +85,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'perl.svg',
     description: i18n.translate('customIntegrations.languageclients.PerlDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Perl client.',
+      defaultMessage: 'Index data to Elasticsearch with the Perl client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/perl-api/{branch}/index.html`,
   },
@@ -114,8 +96,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'python.svg',
     description: i18n.translate('customIntegrations.languageclients.PythonDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Python client.',
+      defaultMessage: 'Index data to Elasticsearch with the Python client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/python-api/{branch}/index.html`,
   },
@@ -126,8 +107,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'rust.svg',
     description: i18n.translate('customIntegrations.languageclients.RustDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Rust client.',
+      defaultMessage: 'Index data to Elasticsearch with the Rust client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/rust-api/{branch}/index.html`,
   },
@@ -138,8 +118,7 @@ export const integrations: LanguageIntegration[] = [
     }),
     icon: 'java.svg',
     description: i18n.translate('customIntegrations.languageclients.JavaDescription', {
-      defaultMessage:
-        'Start building your custom application on top of Elasticsearch with the official Java client.',
+      defaultMessage: 'Index data to Elasticsearch with the Java client.',
     }),
     docUrlTemplate: `${ELASTICSEARCH_CLIENT_URL}/java-api-client/{branch}/index.html`,
   },