[Security Solutions] Adds a security solutions experimental telemetry…

… preview endpoint and e2e tests (#123989)

## Summary

For e2e testing to be possible we have to enable an experimental preview endpoint:

```ts
GET /internal/security_solution/telemetry
```

You can only enable this endpoint if you edit your `kibana.dev.yml` and add this line:
```yml
xpack.securitySolution.enableExperimental: ['previewTelemetryUrlEnabled']
```

This PR adds the following:
* Pulls up an interface from both `TelemetryEventsSender` and `TelemetryReceiver` 
* Issue in the e2e FTR where we weren't removing lists from agnostic and regular space
* Adds preview capabilities for `DetectionRules`, `SecurityLists`, `Endpoint`, and `Diagnostics`
* 19 FTR e2e tests for `DetectionRules` and `SecurityLists`. But does _NOT_ add any for `Endpoint` or `Diagnostics`
* Fixes a dangling promise that makes the preview deterministic
* Fixes a small issue seen with the trusted applications and endpoint application createLists

To run the test server and runner for FTR:

Start the FTR server and wait for it to be initialized first:
```sh
cd kibana/x-pack
node scripts/functional_tests_server.js --config test/detection_engine_api_integration/security_and_spaces/config.ts
```

In a seperate terminal run:
```sh
cd kibana/x-pack
node scripts/functional_test_runner.js --config test/detection_engine_api_integration/security_and_spaces/config.ts --include test/detection_engine_api_integration/security_and_spaces/tests/telemetry/index.ts
```

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

x-pack/plugins/security_solution/common/constants.ts

@@ -267,6 +267,14 @@ export const DETECTION_ENGINE_RULE_EXECUTION_EVENTS_URL =
 export const detectionEngineRuleExecutionEventsUrl = (ruleId: string) =>
   `${INTERNAL_DETECTION_ENGINE_URL}/rules/${ruleId}/execution/events` as const;
 
+/**
+ * Telemetry detection endpoint for any previews requested of what data we are
+ * providing through UI/UX and for e2e tests.
+ *   curl http//localhost:5601/internal/security_solution/telemetry
+ * to see the contents
+ */
+export const SECURITY_TELEMETRY_URL = `/internal/security_solution/telemetry` as const;
+
 export const TIMELINE_RESOLVE_URL = '/api/timeline/resolve' as const;
 export const TIMELINE_URL = '/api/timeline' as const;
 export const TIMELINES_URL = '/api/timelines' as const;

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -24,6 +24,16 @@ export const allowedExperimentalValues = Object.freeze({
   pendingActionResponsesWithAck: true,
   rulesBulkEditEnabled: true,
   policyListEnabled: false,
+
+  /**
+   * This is used for enabling the end to end tests for the security_solution telemetry.
+   * We disable the telemetry since we don't have specific roles or permissions around it and
+   * we don't want people to be able to violate security by getting access to whole documents
+   * around telemetry they should not.
+   * @see telemetry_detection_rules_preview_route.ts
+   * @see test/detection_engine_api_integration/security_and_spaces/tests/telemetry/README.md
+   */
+  previewTelemetryUrlEnabled: false,
 });
 
 type ExperimentalConfigKeys = Array<keyof ExperimentalFeatures>;

x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/open_close_signals_route.ts

@@ -20,7 +20,7 @@ import {
   DETECTION_ENGINE_SIGNALS_STATUS_URL,
 } from '../../../../../common/constants';
 import { buildSiemResponse } from '../utils';
-import { TelemetryEventsSender } from '../../../telemetry/sender';
+import { ITelemetryEventsSender } from '../../../telemetry/sender';
 import { INSIGHTS_CHANNEL } from '../../../telemetry/constants';
 import { SetupPlugins } from '../../../../plugin';
 import { buildRouteValidation } from '../../../../utils/build_validation/route_validation';
@@ -33,7 +33,7 @@ export const setSignalsStatusRoute = (
   router: SecuritySolutionPluginRouter,
   logger: Logger,
   security: SetupPlugins['security'],
-  sender: TelemetryEventsSender
+  sender: ITelemetryEventsSender
 ) => {
   router.post(
     {

x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/telemetry_detection_rules_preview_route.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger } from 'src/core/server';
+
+import { SECURITY_TELEMETRY_URL } from '../../../../../common/constants';
+import type { SecuritySolutionPluginRouter } from '../../../../types';
+import { ITelemetryReceiver } from '../../../telemetry/receiver';
+import { ITelemetryEventsSender } from '../../../telemetry/sender';
+import { getDetectionRulesPreview } from './utils/get_detecton_rules_preview';
+import { getSecurityListsPreview } from './utils/get_security_lists_preview';
+import { getEndpointPreview } from './utils/get_endpoint_preview';
+import { getDiagnosticsPreview } from './utils/get_diagnostics_preview';
+
+export const telemetryDetectionRulesPreviewRoute = (
+  router: SecuritySolutionPluginRouter,
+  logger: Logger,
+  telemetryReceiver: ITelemetryReceiver,
+  telemetrySender: ITelemetryEventsSender
+) => {
+  router.get(
+    {
+      path: SECURITY_TELEMETRY_URL,
+      validate: false,
+      options: {
+        tags: ['access:securitySolution'],
+      },
+    },
+    async (context, request, response) => {
+      const detectionRules = await getDetectionRulesPreview({
+        logger,
+        telemetryReceiver,
+        telemetrySender,
+      });
+
+      const securityLists = await getSecurityListsPreview({
+        logger,
+        telemetryReceiver,
+        telemetrySender,
+      });
+
+      const endpoints = await getEndpointPreview({
+        logger,
+        telemetryReceiver,
+        telemetrySender,
+      });
+
+      const diagnostics = await getDiagnosticsPreview({
+        logger,
+        telemetryReceiver,
+        telemetrySender,
+      });
+
+      return response.ok({
+        body: {
+          detection_rules: detectionRules,
+          security_lists: securityLists,
+          endpoints,
+          diagnostics,
+        },
+      });
+    }
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_detecton_rules_preview.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger } from 'src/core/server';
+
+import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender';
+import { ITelemetryReceiver } from '../../../../telemetry/receiver';
+import { ITelemetryEventsSender } from '../../../../telemetry/sender';
+import { createTelemetryDetectionRuleListsTaskConfig } from '../../../../telemetry/tasks/detection_rule';
+import { parseNdjson } from './parse_ndjson';
+
+export const getDetectionRulesPreview = async ({
+  logger,
+  telemetryReceiver,
+  telemetrySender,
+}: {
+  logger: Logger;
+  telemetryReceiver: ITelemetryReceiver;
+  telemetrySender: ITelemetryEventsSender;
+}): Promise<object[][]> => {
+  const taskExecutionPeriod = {
+    last: new Date(0).toISOString(),
+    current: new Date().toISOString(),
+  };
+
+  const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender);
+  const task = createTelemetryDetectionRuleListsTaskConfig(1000);
+  await task.runTask(
+    'detection-rules-preview',
+    logger,
+    telemetryReceiver,
+    taskSender,
+    taskExecutionPeriod
+  );
+  const messages = taskSender.getSentMessages();
+  return parseNdjson(messages);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_diagnostics_preview.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger } from 'src/core/server';
+
+import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender';
+import { ITelemetryReceiver } from '../../../../telemetry/receiver';
+import { ITelemetryEventsSender } from '../../../../telemetry/sender';
+import { createTelemetryDiagnosticsTaskConfig } from '../../../../telemetry/tasks/diagnostic';
+import { parseNdjson } from './parse_ndjson';
+
+export const getDiagnosticsPreview = async ({
+  logger,
+  telemetryReceiver,
+  telemetrySender,
+}: {
+  logger: Logger;
+  telemetryReceiver: ITelemetryReceiver;
+  telemetrySender: ITelemetryEventsSender;
+}): Promise<object[][]> => {
+  const taskExecutionPeriod = {
+    last: new Date(0).toISOString(),
+    current: new Date().toISOString(),
+  };
+
+  const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender);
+  const task = createTelemetryDiagnosticsTaskConfig();
+  await task.runTask(
+    'diagnostics-preview',
+    logger,
+    telemetryReceiver,
+    taskSender,
+    taskExecutionPeriod
+  );
+  const messages = taskSender.getSentMessages();
+  return parseNdjson(messages);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_endpoint_preview.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger } from 'src/core/server';
+
+import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender';
+import { ITelemetryReceiver } from '../../../../telemetry/receiver';
+import { ITelemetryEventsSender } from '../../../../telemetry/sender';
+import { createTelemetryEndpointTaskConfig } from '../../../../telemetry/tasks/endpoint';
+import { parseNdjson } from './parse_ndjson';
+
+export const getEndpointPreview = async ({
+  logger,
+  telemetryReceiver,
+  telemetrySender,
+}: {
+  logger: Logger;
+  telemetryReceiver: ITelemetryReceiver;
+  telemetrySender: ITelemetryEventsSender;
+}): Promise<object[][]> => {
+  const taskExecutionPeriod = {
+    last: new Date(0).toISOString(),
+    current: new Date().toISOString(),
+  };
+
+  const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender);
+  const task = createTelemetryEndpointTaskConfig(1000);
+  await task.runTask(
+    'endpoint-preview',
+    logger,
+    telemetryReceiver,
+    taskSender,
+    taskExecutionPeriod
+  );
+  const messages = taskSender.getSentMessages();
+  return parseNdjson(messages);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/get_security_lists_preview.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Logger } from 'src/core/server';
+
+import { PreviewTelemetryEventsSender } from '../../../../telemetry/preview_sender';
+import { ITelemetryReceiver } from '../../../../telemetry/receiver';
+import { ITelemetryEventsSender } from '../../../../telemetry/sender';
+import { createTelemetrySecurityListTaskConfig } from '../../../../telemetry/tasks/security_lists';
+import { parseNdjson } from './parse_ndjson';
+
+export const getSecurityListsPreview = async ({
+  logger,
+  telemetryReceiver,
+  telemetrySender,
+}: {
+  logger: Logger;
+  telemetryReceiver: ITelemetryReceiver;
+  telemetrySender: ITelemetryEventsSender;
+}): Promise<object[][]> => {
+  const taskExecutionPeriod = {
+    last: new Date(0).toISOString(),
+    current: new Date().toISOString(),
+  };
+
+  const taskSender = new PreviewTelemetryEventsSender(logger, telemetrySender);
+  const task = createTelemetrySecurityListTaskConfig(1000);
+  await task.runTask(
+    'security-lists-preview',
+    logger,
+    telemetryReceiver,
+    taskSender,
+    taskExecutionPeriod
+  );
+  const messages = taskSender.getSentMessages();
+  return parseNdjson(messages);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/routes/telemetry/utils/parse_ndjson.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const parseNdjson = (messages: string[]): object[][] => {
+  return messages.map((message) => {
+    const splitByNewLine = message.split('\n');
+    const linesParsed = splitByNewLine.flatMap<object>((line) => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {
+        return [];
+      }
+    });
+    return linesParsed;
+  });
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -37,7 +37,7 @@ import {
 import { ExperimentalFeatures } from '../../../../common/experimental_features';
 import { IEventLogService } from '../../../../../event_log/server';
 import { AlertsFieldMap, RulesFieldMap } from '../../../../common/field_maps';
-import { TelemetryEventsSender } from '../../telemetry/sender';
+import { ITelemetryEventsSender } from '../../telemetry/sender';
 import { RuleExecutionLoggerFactory } from '../rule_execution_log';
 import { commonParamsCamelToSnake } from '../schemas/rule_converters';
 
@@ -128,6 +128,6 @@ export interface CreateRuleOptions {
   experimentalFeatures: ExperimentalFeatures;
   logger: Logger;
   ml?: SetupPlugins['ml'];
-  eventsTelemetry?: TelemetryEventsSender | undefined;
+  eventsTelemetry?: ITelemetryEventsSender | undefined;
   version: string;
 }

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts

@@ -17,7 +17,7 @@ import { getFilter } from '../get_filter';
 import { getInputIndex } from '../get_input_output_index';
 import { searchAfterAndBulkCreate } from '../search_after_bulk_create';
 import { RuleRangeTuple, BulkCreate, WrapHits } from '../types';
-import { TelemetryEventsSender } from '../../../telemetry/sender';
+import { ITelemetryEventsSender } from '../../../telemetry/sender';
 import { BuildRuleMessage } from '../rule_messages';
 import { CompleteRule, SavedQueryRuleParams, QueryRuleParams } from '../../schemas/rule_schemas';
 import { ExperimentalFeatures } from '../../../../../common/experimental_features';
@@ -48,7 +48,7 @@ export const queryExecutor = async ({
   version: string;
   searchAfterSize: number;
   logger: Logger;
-  eventsTelemetry: TelemetryEventsSender | undefined;
+  eventsTelemetry: ITelemetryEventsSender | undefined;
   buildRuleMessage: BuildRuleMessage;
   bulkCreate: BulkCreate;
   wrapHits: WrapHits;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts

@@ -15,7 +15,7 @@ import {
 import { ListClient } from '../../../../../../lists/server';
 import { getInputIndex } from '../get_input_output_index';
 import { RuleRangeTuple, BulkCreate, WrapHits } from '../types';
-import { TelemetryEventsSender } from '../../../telemetry/sender';
+import { ITelemetryEventsSender } from '../../../telemetry/sender';
 import { BuildRuleMessage } from '../rule_messages';
 import { createThreatSignals } from '../threat_mapping/create_threat_signals';
 import { CompleteRule, ThreatRuleParams } from '../../schemas/rule_schemas';
@@ -46,7 +46,7 @@ export const threatMatchExecutor = async ({
   version: string;
   searchAfterSize: number;
   logger: Logger;
-  eventsTelemetry: TelemetryEventsSender | undefined;
+  eventsTelemetry: ITelemetryEventsSender | undefined;
   experimentalFeatures: ExperimentalFeatures;
   buildRuleMessage: BuildRuleMessage;
   bulkCreate: BulkCreate;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { TelemetryEventsSender } from '../../telemetry/sender';
+import { ITelemetryEventsSender } from '../../telemetry/sender';
 import { TelemetryEvent } from '../../telemetry/types';
 import { BuildRuleMessage } from './rule_messages';
 import { SignalSearchResponse, SignalSource } from './types';
@@ -29,7 +29,7 @@ export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEve
 
 export function sendAlertTelemetryEvents(
   logger: Logger,
-  eventsTelemetry: TelemetryEventsSender | undefined,
+  eventsTelemetry: ITelemetryEventsSender | undefined,
   filteredEvents: SignalSearchResponse,
   buildRuleMessage: BuildRuleMessage
 ) {