Merge branch 'master' into telemetry_with_new_ml

 Conflicts:
	x-pack/plugins/ml/server/shared_services/providers/modules.ts
	x-pack/plugins/security_solution/server/usage/detections/detections_helpers.ts

docs/settings/ingest-manager-settings.asciidoc

@@ -8,8 +8,7 @@
 experimental[]
 
 You can configure `xpack.ingestManager` settings in your `kibana.yml`. 
-By default, {ingest-manager} is not enabled. You need to
-enable it. To use {fleet}, you also need to configure {kib} and {es} hosts.
+By default, {ingest-manager} is enabled. To use {fleet}, you also need to configure {kib} and {es} hosts.
 
 See the {ingest-guide}/index.html[Ingest Management] docs for more information.
 
@@ -19,7 +18,7 @@ See the {ingest-guide}/index.html[Ingest Management] docs for more information.
 [cols="2*<"]
 |===
 | `xpack.ingestManager.enabled` {ess-icon}
-  | Set to `true` to enable {ingest-manager}. 
+  | Set to `true` (default) to enable {ingest-manager}. 
 | `xpack.ingestManager.fleet.enabled` {ess-icon}
   | Set to `true` (default) to enable {fleet}. 
 |===

x-pack/plugins/actions/README.md

@@ -160,7 +160,7 @@ This is the primary function for an action type. Whenever the action needs to ex
 | config                                  | The decrypted configuration given to an action. This comes from the action saved object that is partially or fully encrypted within the data store. If you would like to validate the config before being passed to the executor, define `validate.config` within the action type.                                                              |
 | params                                  | Parameters for the execution. These will be given at execution time by either an alert or manually provided when calling the plugin provided execute function.                                                                                                                                                                                  |
 | services.callCluster(path, opts)        | Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana but runs in the context of the user who is calling the action when security is enabled.                                                                                                                  |
-| services.getScopedCallCluster           | This function scopes an instance of CallCluster by returning a `callCluster(path, opts)` function that runs in the context of the user who is calling the action when security is enabled. This must only be called with instances of CallCluster provided by core.                                                                             |
+| services.getLegacyScopedClusterClient   | This function returns an instance of the LegacyScopedClusterClient scoped to the user who is calling the action when security is enabled.                                                                                                                                                                                                       |
 | services.savedObjectsClient             | This is an instance of the saved objects client. This provides the ability to do CRUD on any saved objects within the same space the alert lives in.<br><br>The scope of the saved objects client is tied to the user in context calling the execute API or the API key provided to the execute plugin function (only when security isenabled). |
 | services.log(tags, [data], [timestamp]) | Use this to create server logs. (This is the same function as server.log)                                                                                                                                                                                                                                                                       |
 

x-pack/plugins/actions/server/mocks.ts

@@ -38,7 +38,7 @@ const createServicesMock = () => {
     }
   > = {
     callCluster: elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser,
-    getScopedCallCluster: jest.fn(),
+    getLegacyScopedClusterClient: jest.fn(),
     savedObjectsClient: savedObjectsClientMock.create(),
   };
   return mock;

x-pack/plugins/actions/server/plugin.ts

@@ -307,8 +307,8 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
     return (request) => ({
       callCluster: elasticsearch.legacy.client.asScoped(request).callAsCurrentUser,
       savedObjectsClient: getScopedClient(request),
-      getScopedCallCluster(clusterClient: ILegacyClusterClient) {
-        return clusterClient.asScoped(request).callAsCurrentUser;
+      getLegacyScopedClusterClient(clusterClient: ILegacyClusterClient) {
+        return clusterClient.asScoped(request);
       },
     });
   }

x-pack/plugins/actions/server/types.ts

@@ -25,9 +25,7 @@ export type SpaceIdToNamespaceFunction = (spaceId?: string) => string | undefine
 export interface Services {
   callCluster: ILegacyScopedClusterClient['callAsCurrentUser'];
   savedObjectsClient: SavedObjectsClientContract;
-  getScopedCallCluster(
-    clusterClient: ILegacyClusterClient
-  ): ILegacyScopedClusterClient['callAsCurrentUser'];
+  getLegacyScopedClusterClient(clusterClient: ILegacyClusterClient): ILegacyScopedClusterClient;
 }
 
 declare module 'src/core/server' {

x-pack/plugins/alerts/README.md

@@ -103,7 +103,7 @@ This is the primary function for an alert type. Whenever the alert needs to exec
 |---|---|
 |services.callCluster(path, opts)|Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana but in the context of the user who created the alert when security is enabled.|
 |services.savedObjectsClient|This is an instance of the saved objects client. This provides the ability to do CRUD on any saved objects within the same space the alert lives in.<br><br>The scope of the saved objects client is tied to the user who created the alert (only when security isenabled).|
-|services.getScopedCallCluster|This function scopes an instance of CallCluster by returning a `callCluster(path, opts)` function that runs in the context of the user who created the alert when security is enabled. This must only be called with instances of CallCluster provided by core.|
+|services.getLegacyScopedClusterClient|This function returns an instance of the LegacyScopedClusterClient scoped to the user who created the alert when security is enabled.|
 |services.alertInstanceFactory(id)|This [alert instance factory](#alert-instance-factory) creates instances of alerts and must be used in order to execute actions. The id you give to the alert instance factory is a unique identifier to the alert instance.|
 |services.log(tags, [data], [timestamp])|Use this to create server logs. (This is the same function as server.log)|
 |startedAt|The date and time the alert type started execution.|

x-pack/plugins/alerts/server/mocks.ts

@@ -59,7 +59,7 @@ const createAlertServicesMock = () => {
       .fn<jest.Mocked<AlertInstance>, [string]>()
       .mockReturnValue(alertInstanceFactoryMock),
     callCluster: elasticsearchServiceMock.createLegacyScopedClusterClient().callAsCurrentUser,
-    getScopedCallCluster: jest.fn(),
+    getLegacyScopedClusterClient: jest.fn(),
     savedObjectsClient: savedObjectsClientMock.create(),
   };
 };

x-pack/plugins/alerts/server/plugin.ts

@@ -273,8 +273,8 @@ export class AlertingPlugin {
     return (request) => ({
       callCluster: elasticsearch.legacy.client.asScoped(request).callAsCurrentUser,
       savedObjectsClient: this.getScopedClientWithAlertSavedObjectType(savedObjects, request),
-      getScopedCallCluster(clusterClient: ILegacyClusterClient) {
-        return clusterClient.asScoped(request).callAsCurrentUser;
+      getLegacyScopedClusterClient(clusterClient: ILegacyClusterClient) {
+        return clusterClient.asScoped(request);
       },
     });
   }

x-pack/plugins/alerts/server/types.ts

@@ -40,9 +40,7 @@ declare module 'src/core/server' {
 export interface Services {
   callCluster: ILegacyScopedClusterClient['callAsCurrentUser'];
   savedObjectsClient: SavedObjectsClientContract;
-  getScopedCallCluster(
-    clusterClient: ILegacyClusterClient
-  ): ILegacyScopedClusterClient['callAsCurrentUser'];
+  getLegacyScopedClusterClient(clusterClient: ILegacyClusterClient): ILegacyScopedClusterClient;
 }
 
 export interface AlertServices extends Services {

x-pack/plugins/apm/server/lib/helpers/setup_request.ts

@@ -112,7 +112,7 @@ function getMlSetup(context: APMRequestHandlerContext, request: KibanaRequest) {
     return;
   }
   const ml = context.plugins.ml;
-  const mlClient = ml.mlClient.asScoped(request).callAsCurrentUser;
+  const mlClient = ml.mlClient.asScoped(request);
   return {
     mlSystem: ml.mlSystemProvider(mlClient, request),
     anomalyDetectors: ml.anomalyDetectorsProvider(mlClient, request),

x-pack/plugins/event_log/server/es/context.mock.ts

@@ -17,7 +17,7 @@ const createContextMock = () => {
     logger: loggingSystemMock.createLogger(),
     esNames: namesMock.create(),
     initialize: jest.fn(),
-    waitTillReady: jest.fn(),
+    waitTillReady: jest.fn(async () => true),
     esAdapter: clusterClientAdapterMock.create(),
     initialized: true,
   };

x-pack/plugins/event_log/server/es/context.test.ts

@@ -7,9 +7,8 @@
 import { createEsContext } from './context';
 import { LegacyClusterClient, Logger } from '../../../../../src/core/server';
 import { elasticsearchServiceMock, loggingSystemMock } from '../../../../../src/core/server/mocks';
-jest.mock('../lib/../../../../package.json', () => ({
-  version: '1.2.3',
-}));
+jest.mock('../lib/../../../../package.json', () => ({ version: '1.2.3' }));
+jest.mock('./init');
 type EsClusterClient = Pick<jest.Mocked<LegacyClusterClient>, 'callAsInternalUser' | 'asScoped'>;
 
 let logger: Logger;
@@ -92,4 +91,16 @@ describe('createEsContext', () => {
     );
     expect(doesIndexTemplateExist).toBeTruthy();
   });
+
+  test('should handled failed initialization', async () => {
+    jest.requireMock('./init').initializeEs.mockResolvedValue(false);
+    const context = createEsContext({
+      logger,
+      clusterClientPromise: Promise.resolve(clusterClient),
+      indexNameRoot: 'test2',
+    });
+    context.initialize();
+    const success = await context.waitTillReady();
+    expect(success).toBe(false);
+  });
 });

x-pack/plugins/event_log/server/es/context.ts

@@ -64,21 +64,23 @@ class EsContextImpl implements EsContext {
 
     setImmediate(async () => {
       try {
-        await this._initialize();
-        this.logger.debug('readySignal.signal(true)');
-        this.readySignal.signal(true);
+        const success = await this._initialize();
+        this.logger.debug(`readySignal.signal(${success})`);
+        this.readySignal.signal(success);
       } catch (err) {
         this.logger.debug('readySignal.signal(false)');
         this.readySignal.signal(false);
       }
     });
   }
 
+  // waits till the ES initialization is done, returns true if it was successful,
+  // false if it was not successful
   async waitTillReady(): Promise<boolean> {
     return await this.readySignal.wait();
   }
 
-  private async _initialize() {
-    await initializeEs(this);
+  private async _initialize(): Promise<boolean> {
+    return await initializeEs(this);
   }
 }

x-pack/plugins/event_log/server/event_logger.test.ts

@@ -14,25 +14,52 @@ import { delay } from './lib/delay';
 import { EVENT_LOGGED_PREFIX } from './event_logger';
 
 const KIBANA_SERVER_UUID = '424-24-2424';
+const WRITE_LOG_WAIT_MILLIS = 3000;
 
 describe('EventLogger', () => {
   let systemLogger: ReturnType<typeof loggingSystemMock.createLogger>;
-  let esContext: EsContext;
+  let esContext: jest.Mocked<EsContext>;
   let service: IEventLogService;
   let eventLogger: IEventLogger;
 
   beforeEach(() => {
+    jest.resetAllMocks();
     systemLogger = loggingSystemMock.createLogger();
     esContext = contextMock.create();
     service = new EventLogService({
       esContext,
       systemLogger,
-      config: { enabled: true, logEntries: true, indexEntries: false },
+      config: { enabled: true, logEntries: true, indexEntries: true },
       kibanaUUID: KIBANA_SERVER_UUID,
     });
     eventLogger = service.getLogger({});
   });
 
+  test('handles successful initialization', async () => {
+    service.registerProviderActions('test-provider', ['test-action-1']);
+    eventLogger = service.getLogger({
+      event: { provider: 'test-provider', action: 'test-action-1' },
+    });
+
+    eventLogger.logEvent({});
+    await waitForLogEvent(systemLogger);
+    delay(WRITE_LOG_WAIT_MILLIS); // sleep a bit since event logging is async
+    expect(esContext.esAdapter.indexDocument).toHaveBeenCalled();
+  });
+
+  test('handles failed initialization', async () => {
+    service.registerProviderActions('test-provider', ['test-action-1']);
+    eventLogger = service.getLogger({
+      event: { provider: 'test-provider', action: 'test-action-1' },
+    });
+    esContext.waitTillReady.mockImplementation(async () => false);
+
+    eventLogger.logEvent({});
+    await waitForLogEvent(systemLogger);
+    delay(WRITE_LOG_WAIT_MILLIS); // sleep a bit longer since event logging is async
+    expect(esContext.esAdapter.indexDocument).not.toHaveBeenCalled();
+  });
+
   test('method logEvent() writes expected default values', async () => {
     service.registerProviderActions('test-provider', ['test-action-1']);
     eventLogger = service.getLogger({

x-pack/plugins/event_log/server/event_logger.ts

@@ -183,7 +183,12 @@ function indexEventDoc(esContext: EsContext, doc: Doc): void {
 // whew, the thing that actually writes the event log document!
 async function indexLogEventDoc(esContext: EsContext, doc: unknown) {
   esContext.logger.debug(`writing to event log: ${JSON.stringify(doc)}`);
-  await esContext.waitTillReady();
+  const success = await esContext.waitTillReady();
+  if (!success) {
+    esContext.logger.debug(`event log did not initialize correctly, event not written`);
+    return;
+  }
+
   await esContext.esAdapter.indexDocument(doc);
   esContext.logger.debug(`writing to event log complete`);
 }

x-pack/plugins/index_management/public/application/components/template_form/steps/step_logistics.tsx

@@ -62,7 +62,7 @@ function getFieldsMeta(esDocsBase: string) {
       description: (
         <FormattedMessage
           id="xpack.idxMgmt.templateForm.stepLogistics.dataStreamDescription"
-          defaultMessage="Create a data stream instead of an index. {docsLink}"
+          defaultMessage="The template creates data streams instead of indices. {docsLink}"
           values={{
             docsLink: (
               <EuiLink
@@ -73,7 +73,7 @@ function getFieldsMeta(esDocsBase: string) {
                 {i18n.translate(
                   'xpack.idxMgmt.templateForm.stepLogistics.dataStreamDocumentionLink',
                   {
-                    defaultMessage: 'Learn more about data streams.',
+                    defaultMessage: 'Learn more.',
                   }
                 )}
               </EuiLink>

x-pack/plugins/infra/common/http_api/log_analysis/results/index.ts

@@ -10,3 +10,4 @@ export * from './log_entry_category_examples';
 export * from './log_entry_rate';
 export * from './log_entry_examples';
 export * from './log_entry_anomalies';
+export * from './log_entry_anomalies_datasets';

x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_anomalies.ts

@@ -128,6 +128,8 @@ export const getLogEntryAnomaliesRequestPayloadRT = rt.type({
       pagination: paginationRT,
       // Sort properties
       sort: sortRT,
+      // Dataset filters
+      datasets: rt.array(rt.string),
     }),
   ]),
 });

x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_anomalies_datasets.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+import {
+  badRequestErrorRT,
+  forbiddenErrorRT,
+  timeRangeRT,
+  routeTimingMetadataRT,
+} from '../../shared';
+
+export const LOG_ANALYSIS_GET_LOG_ENTRY_ANOMALIES_DATASETS_PATH =
+  '/api/infra/log_analysis/results/log_entry_anomalies_datasets';
+
+/**
+ * request
+ */
+
+export const getLogEntryAnomaliesDatasetsRequestPayloadRT = rt.type({
+  data: rt.type({
+    // the id of the source configuration
+    sourceId: rt.string,
+    // the time range to fetch the anomalies datasets from
+    timeRange: timeRangeRT,
+  }),
+});
+
+export type GetLogEntryAnomaliesDatasetsRequestPayload = rt.TypeOf<
+  typeof getLogEntryAnomaliesDatasetsRequestPayloadRT
+>;
+
+/**
+ * response
+ */
+
+export const getLogEntryAnomaliesDatasetsSuccessReponsePayloadRT = rt.intersection([
+  rt.type({
+    data: rt.type({
+      datasets: rt.array(rt.string),
+    }),
+  }),
+  rt.partial({
+    timing: routeTimingMetadataRT,
+  }),
+]);
+
+export type GetLogEntryAnomaliesDatasetsSuccessResponsePayload = rt.TypeOf<
+  typeof getLogEntryAnomaliesDatasetsSuccessReponsePayloadRT
+>;
+
+export const getLogEntryAnomaliesDatasetsResponsePayloadRT = rt.union([
+  getLogEntryAnomaliesDatasetsSuccessReponsePayloadRT,
+  badRequestErrorRT,
+  forbiddenErrorRT,
+]);
+
+export type GetLogEntryAnomaliesDatasetsReponsePayload = rt.TypeOf<
+  typeof getLogEntryAnomaliesDatasetsResponsePayloadRT
+>;

x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts

@@ -16,11 +16,16 @@ export const LOG_ANALYSIS_GET_LOG_ENTRY_RATE_PATH =
  */
 
 export const getLogEntryRateRequestPayloadRT = rt.type({
-  data: rt.type({
-    bucketDuration: rt.number,
-    sourceId: rt.string,
-    timeRange: timeRangeRT,
-  }),
+  data: rt.intersection([
+    rt.type({
+      bucketDuration: rt.number,
+      sourceId: rt.string,
+      timeRange: timeRangeRT,
+    }),
+    rt.partial({
+      datasets: rt.array(rt.string),
+    }),
+  ]),
 });
 
 export type GetLogEntryRateRequestPayload = rt.TypeOf<typeof getLogEntryRateRequestPayloadRT>;

x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/datasets_selector.tsx → x-pack/plugins/infra/public/components/logging/log_analysis_results/datasets_selector.tsx

@@ -8,7 +8,7 @@ import { EuiComboBox, EuiComboBoxOptionOption } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React, { useCallback, useMemo } from 'react';
 
-import { getFriendlyNameForPartitionId } from '../../../../../../common/log_analysis';
+import { getFriendlyNameForPartitionId } from '../../../../common/log_analysis';
 
 type DatasetOptionProps = EuiComboBoxOptionOption<string>;
 
@@ -51,7 +51,7 @@ export const DatasetsSelector: React.FunctionComponent<{
 };
 
 const datasetFilterPlaceholder = i18n.translate(
-  'xpack.infra.logs.logEntryCategories.datasetFilterPlaceholder',
+  'xpack.infra.logs.analysis.datasetFilterPlaceholder',
   {
     defaultMessage: 'Filter by datasets',
   }

x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/top_categories_section.tsx

@@ -14,7 +14,7 @@ import { BetaBadge } from '../../../../../components/beta_badge';
 import { LoadingOverlayWrapper } from '../../../../../components/loading_overlay_wrapper';
 import { RecreateJobButton } from '../../../../../components/logging/log_analysis_job_status';
 import { AnalyzeInMlButton } from '../../../../../components/logging/log_analysis_results';
-import { DatasetsSelector } from './datasets_selector';
+import { DatasetsSelector } from '../../../../../components/logging/log_analysis_results/datasets_selector';
 import { TopCategoriesTable } from './top_categories_table';
 
 export const TopCategoriesSection: React.FunctionComponent<{