Merge branch 'master' into alerting/consumer-based-rbac

* master: (28 commits)
  allow some env settings for ingest manager (#72544)
  Add inspector for VEGA (#70941)
  chore(NA): fix grunt task for test:coverage (#72539)
  Archive e2e test results in ES (#72575)
  preserve 401 errors from new es client (#71248)
  [SIEM][Detections] Updates text for severity and risk_score overrides  (#72244)
  fixing error occurences tooltip (#72425)
  use KibanaClient interface instead of Client for new client interface (#72388)
  [APM] Handle ML errors (#72316)
  [Discover] Improve histogram tests (#72235)
  [ftr/webdriver] retry on all errors, use Rx so that timers are canceled (#72540)
  [pre-req] Move .storybook to storybook; standardize files (#72384)
  [Security_Solution][Resolver][Bug]: Restore breadcrumb background (#72538)
  [ML] Fix annotation detector linking & delayed_data(0) (#72468)
  [Security Solution][Exceptions] - Make esTypes and subType available to index patterns (#72336)
  [SIEM] Uses faster wait from testing-library and removes duplicate older wait idiom (#72509)
  Fix long combo box items breaking out of flex item width (#72512)
  [pipeline/commitStatus] update commit status in baseline-capture job (#72366)
  [Security Solution][Resolver] Update the resolver element ref on scroll events if the position of the element has changed within the page (#72461)
  [Maps] auto-fit to data bounds (#72129)
  ...

.ci/Jenkinsfile_baseline_capture

@@ -4,23 +4,25 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 kibanaPipeline(timeoutMinutes: 120) {
-  ciStats.trackBuild {
-    catchError {
-      parallel([
-        'oss-visualRegression': {
-          workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
-            kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
-          }
-        },
-        'xpack-visualRegression': {
-          workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
-            kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
-          }
-        },
-      ])
-    }
+  githubCommitStatus.trackBuild(params.commit, 'kibana-ci-baseline') {
+    ciStats.trackBuild {
+      catchError {
+        parallel([
+          'oss-visualRegression': {
+            workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
+              kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
+            }
+          },
+          'xpack-visualRegression': {
+            workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
+              kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
+            }
+          },
+        ])
+      }
 
-    kibanaPipeline.sendMail()
-    slackNotifications.onFailure()
+      kibanaPipeline.sendMail()
+      slackNotifications.onFailure()
+    }
   }
 }

.ci/end2end.groovy

@@ -110,6 +110,9 @@ pipeline {
         archiveArtifacts(allowEmptyArchive: true, artifacts: "${E2E_DIR}/kibana.log")
       }
     }
+    cleanup {
+      notifyBuildResult(notifyPRComment: false, analyzeFlakey: false, shouldNotify: false)
+    }
   }
 }
 

.ci/pipeline-library/src/test/githubCommitStatus.groovy

@@ -12,6 +12,7 @@ class GithubCommitStatusTest extends KibanaBasePipelineTest {
 
   interface BuildState {
     Object get(String key)
+    Object has(String key)
   }
 
   interface GithubApi {
@@ -25,6 +26,7 @@ class GithubCommitStatusTest extends KibanaBasePipelineTest {
     buildStateMock = mock(BuildState)
     githubApiMock = mock(GithubApi)
 
+    when(buildStateMock.has('checkoutInfo')).thenReturn(true)
     when(buildStateMock.get('checkoutInfo')).thenReturn([ commit: 'COMMIT_HASH', ])
     when(githubApiMock.post(any(), any())).thenReturn(null)
 

.eslintignore

@@ -33,7 +33,7 @@ target
 /x-pack/plugins/canvas/canvas_plugin
 /x-pack/plugins/canvas/canvas_plugin_src/lib/flot-charts
 /x-pack/plugins/canvas/shareable_runtime/build
-/x-pack/plugins/canvas/storybook
+/x-pack/plugins/canvas/storybook/build
 /x-pack/plugins/monitoring/public/lib/jquery_flot
 /x-pack/plugins/reporting/server/export_types/printable_pdf/server/lib/pdf/assets/**
 /x-pack/legacy/plugins/infra/common/graphql/types.ts

.eslintrc.js

@@ -1222,6 +1222,12 @@ module.exports = {
         ],
       },
     },
+    {
+      files: ['x-pack/plugins/canvas/storybook/**'],
+      rules: {
+        'import/no-extraneous-dependencies': 0,
+      },
+    },
     {
       files: ['x-pack/plugins/canvas/canvas_plugin_src/**/*.js'],
       globals: { canvas: true, $: true },

.sass-lint.yml

@@ -3,6 +3,7 @@ files:
     - 'src/legacy/core_plugins/metrics/**/*.s+(a|c)ss'
     - 'src/plugins/timelion/**/*.s+(a|c)ss'
     - 'src/plugins/vis_type_vislib/**/*.s+(a|c)ss'
+    - 'src/plugins/vis_type_vega/**/*.s+(a|c)ss'
     - 'src/plugins/vis_type_xy/**/*.s+(a|c)ss'
     - 'x-pack/plugins/canvas/**/*.s+(a|c)ss'
     - 'x-pack/plugins/triggers_actions_ui/**/*.s+(a|c)ss'

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.registeronpostauth.md

@@ -14,5 +14,5 @@ registerOnPostAuth: (handler: OnPostAuthHandler) => void;
 
 ## Remarks
 
-The auth state is available at stage via http.auth.get(..) Can register any number of registerOnPreRouting, which are called in sequence (from the first registered to the last). See [OnPostAuthHandler](./kibana-plugin-core-server.onpostauthhandler.md)<!-- -->.
+The auth state is available at stage via http.auth.get(..) Can register any number of registerOnPostAuth, which are called in sequence (from the first registered to the last). See [OnPostAuthHandler](./kibana-plugin-core-server.onpostauthhandler.md)<!-- -->.
 

docs/development/core/server/kibana-plugin-core-server.httpservicesetup.registeronpreauth.md

@@ -14,5 +14,5 @@ registerOnPreAuth: (handler: OnPreAuthHandler) => void;
 
 ## Remarks
 
-Can register any number of registerOnPostAuth, which are called in sequence (from the first registered to the last). See [OnPreRoutingHandler](./kibana-plugin-core-server.onpreroutinghandler.md)<!-- -->.
+Can register any number of registerOnPreAuth, which are called in sequence (from the first registered to the last). See [OnPreAuthHandler](./kibana-plugin-core-server.onpreauthhandler.md)<!-- -->.
 

docs/development/core/server/kibana-plugin-core-server.md

@@ -122,7 +122,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [OnPreAuthToolkit](./kibana-plugin-core-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
 |  [OnPreResponseExtensions](./kibana-plugin-core-server.onpreresponseextensions.md) | Additional data to extend a response. |
 |  [OnPreResponseInfo](./kibana-plugin-core-server.onpreresponseinfo.md) | Response status code. |
-|  [OnPreResponseToolkit](./kibana-plugin-core-server.onpreresponsetoolkit.md) | A tool set defining an outcome of OnPreRouting interceptor for incoming request. |
+|  [OnPreResponseToolkit](./kibana-plugin-core-server.onpreresponsetoolkit.md) | A tool set defining an outcome of OnPreResponse interceptor for incoming request. |
 |  [OnPreRoutingToolkit](./kibana-plugin-core-server.onpreroutingtoolkit.md) | A tool set defining an outcome of OnPreRouting interceptor for incoming request. |
 |  [OpsMetrics](./kibana-plugin-core-server.opsmetrics.md) | Regroups metrics gathered by all the collectors. This contains metrics about the os/runtime, the kibana process and the http server. |
 |  [OpsOsMetrics](./kibana-plugin-core-server.opsosmetrics.md) | OS related metrics |

docs/development/core/server/kibana-plugin-core-server.onpreresponsetoolkit.md

@@ -4,7 +4,7 @@
 
 ## OnPreResponseToolkit interface
 
-A tool set defining an outcome of OnPreRouting interceptor for incoming request.
+A tool set defining an outcome of OnPreResponse interceptor for incoming request.
 
 <b>Signature:</b>
 

package.json

@@ -125,7 +125,7 @@
     "@elastic/apm-rum": "^5.2.0",
     "@elastic/charts": "19.8.1",
     "@elastic/datemath": "5.0.3",
-    "@elastic/elasticsearch": "7.9.0-rc.1",
+    "@elastic/elasticsearch": "7.9.0-rc.2",
     "@elastic/ems-client": "7.9.3",
     "@elastic/eui": "26.3.1",
     "@elastic/filesaver": "1.1.2",

src/core/server/elasticsearch/client/errors.test.ts

@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import {
+  ResponseError,
+  ConnectionError,
+  ConfigurationError,
+} from '@elastic/elasticsearch/lib/errors';
+import { ApiResponse } from '@elastic/elasticsearch';
+import { isResponseError, isUnauthorizedError } from './errors';
+
+const createApiResponseError = ({
+  statusCode = 200,
+  headers = {},
+  body = {},
+}: {
+  statusCode?: number;
+  headers?: Record<string, string>;
+  body?: Record<string, any>;
+} = {}): ApiResponse => {
+  return {
+    body,
+    statusCode,
+    headers,
+    warnings: [],
+    meta: {} as any,
+  };
+};
+
+describe('isResponseError', () => {
+  it('returns `true` when the input is a `ResponseError`', () => {
+    expect(isResponseError(new ResponseError(createApiResponseError()))).toBe(true);
+  });
+
+  it('returns `false` when the input is not a `ResponseError`', () => {
+    expect(isResponseError(new Error('foo'))).toBe(false);
+    expect(isResponseError(new ConnectionError('error', createApiResponseError()))).toBe(false);
+    expect(isResponseError(new ConfigurationError('foo'))).toBe(false);
+  });
+});
+
+describe('isUnauthorizedError', () => {
+  it('returns true when the input is a `ResponseError` and statusCode === 401', () => {
+    expect(
+      isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 401 })))
+    ).toBe(true);
+  });
+
+  it('returns false when the input is a `ResponseError` and statusCode !== 401', () => {
+    expect(
+      isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 200 })))
+    ).toBe(false);
+    expect(
+      isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 403 })))
+    ).toBe(false);
+    expect(
+      isUnauthorizedError(new ResponseError(createApiResponseError({ statusCode: 500 })))
+    ).toBe(false);
+  });
+
+  it('returns `false` when the input is not a `ResponseError`', () => {
+    expect(isUnauthorizedError(new Error('foo'))).toBe(false);
+    expect(isUnauthorizedError(new ConnectionError('error', createApiResponseError()))).toBe(false);
+    expect(isUnauthorizedError(new ConfigurationError('foo'))).toBe(false);
+  });
+});

src/core/server/elasticsearch/client/errors.ts

@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ResponseError } from '@elastic/elasticsearch/lib/errors';
+
+export type UnauthorizedError = ResponseError & {
+  statusCode: 401;
+};
+
+export function isResponseError(error: any): error is ResponseError {
+  return Boolean(error.body && error.statusCode && error.headers);
+}
+
+export function isUnauthorizedError(error: any): error is UnauthorizedError {
+  return isResponseError(error) && error.statusCode === 401;
+}

src/core/server/elasticsearch/client/mocks.test.ts

@@ -49,6 +49,12 @@ describe('Mocked client', () => {
     expectMocked(client.close);
   });
 
+  it('used EventEmitter functions should be mocked', () => {
+    expectMocked(client.on);
+    expectMocked(client.off);
+    expectMocked(client.once);
+  });
+
   it('`child` should be mocked and return a mocked Client', () => {
     expectMocked(client.child);
 

src/core/server/elasticsearch/client/mocks.ts

@@ -54,13 +54,20 @@ const createInternalClientMock = (): DeeplyMockedKeys<Client> => {
 
   mockify(client, omittedProps);
 
-  client.transport = {
+  // client got some read-only (getter) properties
+  // so we need to extend it to override the getter-only props.
+  const mock: any = { ...client };
+
+  mock.transport = {
     request: jest.fn(),
   };
-  client.close = jest.fn().mockReturnValue(Promise.resolve());
-  client.child = jest.fn().mockImplementation(() => createInternalClientMock());
+  mock.close = jest.fn().mockReturnValue(Promise.resolve());
+  mock.child = jest.fn().mockImplementation(() => createInternalClientMock());
+  mock.on = jest.fn();
+  mock.off = jest.fn();
+  mock.once = jest.fn();
 
-  return (client as unknown) as DeeplyMockedKeys<Client>;
+  return (mock as unknown) as DeeplyMockedKeys<Client>;
 };
 
 export type ElasticSearchClientMock = DeeplyMockedKeys<ElasticsearchClient>;

src/core/server/elasticsearch/client/types.ts

@@ -17,11 +17,12 @@
  * under the License.
  */
 
-import type { Client } from '@elastic/elasticsearch';
+import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
 import type {
   ApiResponse,
   TransportRequestOptions,
   TransportRequestParams,
+  TransportRequestPromise,
 } from '@elastic/elasticsearch/lib/Transport';
 
 /**
@@ -30,13 +31,13 @@ import type {
  * @public
  */
 export type ElasticsearchClient = Omit<
-  Client,
-  'connectionPool' | 'transport' | 'serializer' | 'extend' | 'helpers' | 'child' | 'close'
+  KibanaClient,
+  'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close'
 > & {
   transport: {
     request(
       params: TransportRequestParams,
       options?: TransportRequestOptions
-    ): Promise<ApiResponse>;
+    ): TransportRequestPromise<ApiResponse>;
   };
 };

src/core/server/http/integration_tests/core_service.test.mocks.ts

@@ -18,10 +18,12 @@
  */
 import { elasticsearchServiceMock } from '../../elasticsearch/elasticsearch_service.mock';
 
-export const clusterClientMock = jest.fn();
-export const clusterClientInstanceMock = elasticsearchServiceMock.createLegacyScopedClusterClient();
+export const MockLegacyScopedClusterClient = jest.fn();
+export const legacyClusterClientInstanceMock = elasticsearchServiceMock.createLegacyScopedClusterClient();
 jest.doMock('../../elasticsearch/legacy/scoped_cluster_client', () => ({
-  LegacyScopedClusterClient: clusterClientMock.mockImplementation(() => clusterClientInstanceMock),
+  LegacyScopedClusterClient: MockLegacyScopedClusterClient.mockImplementation(
+    () => legacyClusterClientInstanceMock
+  ),
 }));
 
 jest.doMock('elasticsearch', () => {
@@ -34,3 +36,12 @@ jest.doMock('elasticsearch', () => {
     },
   };
 });
+
+export const MockElasticsearchClient = jest.fn();
+jest.doMock('@elastic/elasticsearch', () => {
+  const real = jest.requireActual('@elastic/elasticsearch');
+  return {
+    ...real,
+    Client: MockElasticsearchClient,
+  };
+});