Merge branch 'master' of https://github.com/elastic/kibana into alert…

…ing/pass-all-rule-params

docs/developer/getting-started/monorepo-packages.asciidoc

@@ -82,6 +82,7 @@ yarn kbn watch-bazel
 - @kbn/logging
 - @kbn/securitysolution-constants
 - @kbn/securitysolution-utils
+- @kbn/securitysolution-es-utils
 - @kbn/securitysolution-io-ts-utils
 - @kbn/std
 - @kbn/telemetry-utils

docs/user/discover.asciidoc

@@ -6,7 +6,7 @@
 **_Gain insight to your data._**
 
 *Discover* enables you to quickly search and filter your data, get information
-about structure of the fields, and visualize your data with *Lens* and *Maps*.
+about the structure of the fields, and visualize your data with *Lens* and *Maps*.
 You can customize and save your searches and place them on a dashboard.
 
 ++++
@@ -110,6 +110,43 @@ image:images/document-table.png[Document table with fields for manufacturer, geo
 . To rearrange the table columns, hover the mouse over a
 column header, and then use the move and sort controls.
 
+[float]
+[[add-field-in-discover]]
+=== Add a field
+
+What happens if you forgot to define an important value as a separate field? Or, what if you
+want to combine two fields and treat them as one?
+You can add a field to your index pattern from inside of **Discover**,
+and then use that field for analysis and visualizations,
+the same way you do with other fields.
+
+. Click the ellipsis icon (...), and then click *Add field to index pattern*.
++
+[role="screenshot"]
+image:images/add-field-to-pattern.png[Dropdown menu located next to index pattern field with item for adding a field to an index pattern, width=50%]
+
+. In the *Create field* form, enter `hello` for the name.
+
+. Turn on *Set value*.
+
+. Use the Painless scripting language to define the field:
++
+```ts
+emit("Hello World!");
+```
+
+. Click *Save*.
+
+. In the fields list, search for the *hello* field, and then click it.
++
+You'll see the top values for the field.  The pop-up also includes actions for filtering,
+editing, and deleting the field.
++
+[role="screenshot"]
+image:images/hello-field.png[Top values for the hello field, width=50%]
+
+For more information on adding fields and Painless scripting language examples, refer to <<runtime-fields, Explore your data with runtime fields>>.
+
 
 [float]
 [[search-in-discover]]
@@ -186,7 +223,8 @@ You can bookmark this document and share the link.
 === Save your search for later use
 
 Save your search so you can repeat it later, generate a CSV report, or use it in visualizations, dashboards, and Canvas workpads.
-Saving a search saves the query and the filters.
+Saving a search saves the query text, filters,
+and current view of *Discover*&mdash;the columns selected in the document table, the sort order, and the index pattern.
 
 . In the toolbar, click **Save**.
 

docs/user/reporting/reporting-troubleshooting.asciidoc

@@ -16,6 +16,7 @@ Having trouble? Here are solutions to common problems you might encounter while
 * <<reporting-troubleshooting-puppeteer-debug-logs>>
 * <<reporting-troubleshooting-system-requirements>>
 * <<reporting-troubleshooting-arm-systems>>
+* <<reporting-troubleshooting-maps-ems>>
 
 [float]
 [[reporting-diagnostics]]
@@ -163,3 +164,12 @@ In this case, try increasing the memory for the {kib} instance to 2GB.
 === ARM systems
 
 Chromium is not compatible with ARM RHEL/CentOS.
+
+[float]
+[[reporting-troubleshooting-maps-ems]]
+=== Unable to connect to Elastic Maps Service
+
+https://www.elastic.co/elastic-maps-service[{ems} ({ems-init})] is a service that hosts
+tile layers and vector shapes of administrative boundaries.
+If a report contains a map with a missing basemap layer or administrative boundary, the {kib} server does not have access to {ems-init}.
+See <<maps-connect-to-ems>> for information on how to connect your {kib} server to {ems-init}.

package.json

@@ -137,6 +137,7 @@
     "@kbn/monaco": "link:packages/kbn-monaco",
     "@kbn/securitysolution-constants": "link:bazel-bin/packages/kbn-securitysolution-constants/npm_module",
     "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils/npm_module",
+    "@kbn/securitysolution-es-utils": "link:bazel-bin/packages/kbn-securitysolution-es-utils/npm_module",
     "@kbn/securitysolution-io-ts-utils": "link:bazel-bin/packages/kbn-securitysolution-io-ts-utils/npm_module",
     "@kbn/server-http-tools": "link:packages/kbn-server-http-tools",
     "@kbn/server-route-repository": "link:packages/kbn-server-route-repository",

packages/BUILD.bazel

@@ -26,6 +26,7 @@ filegroup(
       "//packages/kbn-securitysolution-constants:build",
       "//packages/kbn-securitysolution-io-ts-utils:build",
       "//packages/kbn-securitysolution-utils:build",
+      "//packages/kbn-securitysolution-es-utils:build",
       "//packages/kbn-std:build",
       "//packages/kbn-telemetry-tools:build",
       "//packages/kbn-tinymath:build",

packages/kbn-securitysolution-es-utils/BUILD.bazel

@@ -0,0 +1,86 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+
+PKG_BASE_NAME = "kbn-securitysolution-es-utils"
+
+PKG_REQUIRE_NAME = "@kbn/securitysolution-es-utils"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = [
+    "**/*.test.*",
+    "**/*.mock.*",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+  "README.md",
+]
+
+SRC_DEPS = [
+  "@npm//@elastic/elasticsearch",
+  "@npm//@hapi/hapi",
+  "@npm//tslib",
+]
+
+TYPES_DEPS = [
+  "@npm//@types/jest",
+  "@npm//@types/node",
+]
+
+DEPS = SRC_DEPS + TYPES_DEPS
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+      "//:tsconfig.base.json",
+  ],
+)
+
+ts_project(
+  name = "tsc",
+  srcs = SRCS,
+  args = ["--pretty"],
+  declaration = True,
+  declaration_map = True,
+  incremental = True,
+  out_dir = "target",
+  root_dir = "src",
+  source_map = True,
+  tsconfig = ":tsconfig",
+  deps = DEPS,
+)
+
+js_library(
+  name = PKG_BASE_NAME,
+  package_name = PKG_REQUIRE_NAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  visibility = ["//visibility:public"],
+  deps = [":tsc"] + DEPS,
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [
+      ":%s" % PKG_BASE_NAME,
+  ],
+)
+
+filegroup(
+  name = "build",
+  srcs = [
+      ":npm_module",
+  ],
+  visibility = ["//visibility:public"],
+)

packages/kbn-securitysolution-es-utils/README.md

@@ -0,0 +1,6 @@
+# kbn-securitysolution-es-utils
+
+This is the shared security solution elastic search utilities among plugins. This was originally created
+to remove the dependencies between security_solution and other projects such as lists. This should only be
+used within server side code and not client side code since it is all elastic search utilities and packages.
+

packages/kbn-securitysolution-es-utils/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-securitysolution-es-utils'],
+};

packages/kbn-securitysolution-es-utils/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@kbn/securitysolution-es-utils",
+  "version": "1.0.0",
+  "description": "security solution elastic search utilities to use across plugins such lists, security_solution, cases, etc...",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "main": "./target/index.js",
+  "types": "./target/index.d.ts",
+  "private": true
+}

packages/kbn-securitysolution-es-utils/src/bad_request_error/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export class BadRequestError extends Error {}

packages/kbn-securitysolution-es-utils/src/create_boostrap_index/index.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+// See the reference(s) below on explanations about why -000001 was chosen and
+// why the is_write_index is true as well as the bootstrapping step which is needed.
+// Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/applying-policy-to-template.html
+export const createBootstrapIndex = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<unknown> => {
+  return (
+    await esClient.transport.request({
+      path: `/${index}-000001`,
+      method: 'PUT',
+      body: {
+        aliases: {
+          [index]: {
+            is_write_index: true,
+          },
+        },
+      },
+    })
+  ).body;
+};

packages/kbn-securitysolution-es-utils/src/delete_all_index/index.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const deleteAllIndex = async (
+  esClient: ElasticsearchClient,
+  pattern: string,
+  maxAttempts = 5
+): Promise<boolean> => {
+  for (let attempt = 1; ; attempt++) {
+    if (attempt > maxAttempts) {
+      throw new Error(
+        `Failed to delete indexes with pattern [${pattern}] after ${maxAttempts} attempts`
+      );
+    }
+
+    // resolve pattern to concrete index names
+    const { body: resp } = await esClient.indices.getAlias(
+      {
+        index: pattern,
+      },
+      { ignore: [404] }
+    );
+
+    // @ts-expect-error status doesn't exist on response
+    if (resp.status === 404) {
+      return true;
+    }
+
+    const indices = Object.keys(resp) as string[];
+
+    // if no indexes exits then we're done with this pattern
+    if (!indices.length) {
+      return true;
+    }
+
+    // delete the concrete indexes we found and try again until this pattern resolves to no indexes
+    await esClient.indices.delete({
+      index: indices,
+      ignore_unavailable: true,
+    });
+  }
+};

packages/kbn-securitysolution-es-utils/src/delete_policy/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const deletePolicy = async (
+  esClient: ElasticsearchClient,
+  policy: string
+): Promise<unknown> => {
+  return (
+    await esClient.transport.request({
+      path: `/_ilm/policy/${policy}`,
+      method: 'DELETE',
+    })
+  ).body;
+};

packages/kbn-securitysolution-es-utils/src/delete_template/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const deleteTemplate = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<unknown> => {
+  return (
+    await esClient.indices.deleteTemplate({
+      name,
+    })
+  ).body;
+};

packages/kbn-securitysolution-es-utils/src/elasticsearch_client/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// Copied from src/core/server/elasticsearch/client/types.ts
+// as these types aren't part of any package yet. Once they are, remove this completely
+
+import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
+import type {
+  ApiResponse,
+  TransportRequestOptions,
+  TransportRequestParams,
+  TransportRequestPromise,
+} from '@elastic/elasticsearch/lib/Transport';
+
+/**
+ * Client used to query the elasticsearch cluster.
+ * @deprecated At some point use the one from src/core/server/elasticsearch/client/types.ts when it is made into a package. If it never is, then keep using this one.
+ * @public
+ */
+export type ElasticsearchClient = Omit<
+  KibanaClient,
+  'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close'
+> & {
+  transport: {
+    request(
+      params: TransportRequestParams,
+      options?: TransportRequestOptions
+    ): TransportRequestPromise<ApiResponse>;
+  };
+};