From e091be3c63705ca3ea543d85cb7c8f81fe17cf9d Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Thu, 26 Nov 2020 12:09:34 +0000 Subject: [PATCH 01/19] Port @tsg's work on task manager. Remove 2nd var to track telemetry opt in. Add ES client to start querying index. Use query to get docs from a dummy index. Change how index is queried. Get diagnostic alerts to send to staging cluster. Record last timestamp. PoC on telemetry opt in via 2 processes. Revert to original solution --- .../server/lib/telemetry/sender.ts | 73 ++++++++++++- .../server/lib/telemetry/task.ts | 100 ++++++++++++++++++ .../security_solution/server/plugin.ts | 4 +- 3 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/task.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 88ce963757f6d..c2f6f49e2e087 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -14,6 +14,11 @@ import { TelemetryPluginStart, TelemetryPluginSetup, } from '../../../../../../src/plugins/telemetry/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../task_manager/server'; +import { TelemetryDiagTask } from './task'; export type SearchTypes = | string @@ -56,24 +61,39 @@ export class TelemetryEventsSender { private isSending = false; private queue: TelemetryEvent[] = []; private isOptedIn?: boolean = true; // Assume true until the first check + private wasOptedInChecked?: boolean = false; // False until the first check + private diagTask?: TelemetryDiagTask; constructor(logger: Logger) { this.logger = logger.get('telemetry_events'); } - public setup(telemetrySetup?: TelemetryPluginSetup) { + public setup(telemetrySetup?: TelemetryPluginSetup, taskManager?: TaskManagerSetupContract) { this.telemetrySetup = telemetrySetup; + + if (taskManager) { + this.diagTask = new TelemetryDiagTask(this.logger, taskManager, this); + } } - public start(core?: CoreStart, telemetryStart?: TelemetryPluginStart) { + public start( + core?: CoreStart, + telemetryStart?: TelemetryPluginStart, + taskManager?: TaskManagerStartContract + ) { this.telemetryStart = telemetryStart; this.core = core; - this.logger.debug(`Starting task`); + this.logger.debug(`Starting local task`); setTimeout(() => { this.sendIfDue(); this.intervalId = setInterval(() => this.sendIfDue(), this.checkIntervalMs); }, this.initialCheckDelayMs); + + if (taskManager && this.diagTask) { + this.logger.debug(`Starting diag task`); + this.diagTask.start(taskManager); + } } public stop() { @@ -82,6 +102,35 @@ export class TelemetryEventsSender { } } + public async fetchDiagnosticAlerts(fromTimestamp: string, toTimestamp: string) { + const query = { + index: 'pete-hampton-test-index*', + ignore_unavailable: true, + size: 100, + body: { + query: { + match_all: {}, + }, + /* + query: { + range: { + '@timestamp': { + gt: fromTimestamp, + lte: toTimestamp, + }, + }, + }, + */ + }, + }; + + if (!this.core) { + throw Error('could not fetch diagnostic alerts. core is not available'); + } + const callCluster = this.core.elasticsearch.legacy.client.callAsInternalUser; + return callCluster('search', query); + } + public queueTelemetryEvents(events: TelemetryEvent[]) { const qlength = this.queue.length; @@ -109,6 +158,17 @@ export class TelemetryEventsSender { }); } + // this returns a cached version of the telemetry opt-in. The cache is updated + // in the periodically executed sendIfDue function. This function returns false + // until the first execution of that function. + public isTelemetryOptedIn(): boolean { + if (!this.wasOptedInChecked) { + return false; + } else { + return this.isOptedIn === true; + } + } + private async sendIfDue() { if (this.isSending) { return; @@ -121,9 +181,8 @@ export class TelemetryEventsSender { try { this.isSending = true; - // Checking opt-in status is relatively expensive (calls a saved-object), so - // we only check it when we have things to send. this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); + this.wasOptedInChecked = true; if (!this.isOptedIn) { this.logger.debug(`Telemetry is not opted-in.`); this.queue = []; @@ -150,6 +209,7 @@ export class TelemetryEventsSender { })); this.queue = []; + /* await this.sendEvents( toSend, telemetryUrl, @@ -157,6 +217,7 @@ export class TelemetryEventsSender { clusterInfo.version?.number, licenseInfo?.uid ); + */ } catch (err) { this.logger.warn(`Error sending telemetry events data: ${err}`); this.queue = []; @@ -245,6 +306,8 @@ const allowlistEventFields: AllowlistFields = { '@timestamp': true, agent: true, Endpoint: true, + Ransomware: true, + data_stream: true, ecs: true, elastic: true, event: true, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts new file mode 100644 index 0000000000000..3d2afa2e458da --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import moment from 'moment'; +import { Logger } from 'src/core/server'; +import { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../../task_manager/server'; +import { TelemetryEventsSender, TelemetryEvent } from './sender'; + +export const TelemetryDiagTaskConstants = { + TIMEOUT: '1m', + TYPE: 'security:telemetry-diagnostics', + INTERVAL: '10s', // TODO: update to 5m + VERSION: '1.0.0', +}; + +export class TelemetryDiagTask { + private readonly logger: Logger; + private readonly sender: TelemetryEventsSender; + private lastQueryTimestamp?: string; + + constructor( + logger: Logger, + taskManager: TaskManagerSetupContract, + sender: TelemetryEventsSender + ) { + this.logger = logger; + this.sender = sender; + + taskManager.registerTaskDefinitions({ + [TelemetryDiagTaskConstants.TYPE]: { + title: 'Security Solution Telemetry Diagnostics task', + timeout: TelemetryDiagTaskConstants.TIMEOUT, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + run: async () => { + await this.runTask(taskInstance.id); + }, + cancel: async () => {}, + }; + }, + }, + }); + } + + public start = async (taskManager: TaskManagerStartContract) => { + try { + await taskManager.ensureScheduled({ + id: this.getTaskId(), + taskType: TelemetryDiagTaskConstants.TYPE, + scope: ['securitySolution'], + schedule: { + interval: TelemetryDiagTaskConstants.INTERVAL, + }, + state: {}, + params: { version: TelemetryDiagTaskConstants.VERSION }, + }); + } catch (e) { + this.logger.error(`Error scheduling task, received ${e.message}`); + } + }; + + private getTaskId = (): string => { + return `${TelemetryDiagTaskConstants.TYPE}:${TelemetryDiagTaskConstants.VERSION}`; + }; + + public runTask = async (taskId: string) => { + // Check that this task is current + if (taskId !== this.getTaskId()) { + this.logger.debug(`Outdated task running: ${taskId}`); + return; + } + + const isOptedIn = this.sender.isTelemetryOptedIn(); + if (!isOptedIn) { + this.logger.debug(`Telemetry is not opted-in.`); + return; + } + + const fetchFromTimestamp = + this.lastQueryTimestamp || moment.utc().subtract(5, 'm').toISOString(); + const fetchToTimestamp = moment.utc().toISOString(); + const response = await this.sender.fetchDiagnosticAlerts(fetchFromTimestamp, fetchToTimestamp); + this.lastQueryTimestamp = fetchToTimestamp; + + const hits = response.hits?.hits || []; + if (!Array.isArray(hits) || !hits.length) { + this.logger.debug('no diagnostic alerts retrieved'); + return; + } + + const alerts: TelemetryEvent[] = hits.map((h) => h._source); + this.sender.queueTelemetryEvents(alerts); + }; +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 10e817bea0282..78cbb47b01695 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -316,7 +316,7 @@ export class Plugin implements IPlugin Date: Fri, 4 Dec 2020 17:16:21 +0000 Subject: [PATCH 02/19] Update on agreed method. Fixes race condition. --- .../server/lib/telemetry/sender.ts | 18 ++++-------------- .../server/lib/telemetry/task.ts | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index c2f6f49e2e087..1d4054afc6820 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -61,7 +61,6 @@ export class TelemetryEventsSender { private isSending = false; private queue: TelemetryEvent[] = []; private isOptedIn?: boolean = true; // Assume true until the first check - private wasOptedInChecked?: boolean = false; // False until the first check private diagTask?: TelemetryDiagTask; constructor(logger: Logger) { @@ -158,15 +157,9 @@ export class TelemetryEventsSender { }); } - // this returns a cached version of the telemetry opt-in. The cache is updated - // in the periodically executed sendIfDue function. This function returns false - // until the first execution of that function. - public isTelemetryOptedIn(): boolean { - if (!this.wasOptedInChecked) { - return false; - } else { - return this.isOptedIn === true; - } + public async isTelemetryOptedIn() { + this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); + return this.isOptedIn === true; } private async sendIfDue() { @@ -181,8 +174,7 @@ export class TelemetryEventsSender { try { this.isSending = true; - this.isOptedIn = await this.telemetryStart?.getIsOptedIn(); - this.wasOptedInChecked = true; + this.isOptedIn = await this.isTelemetryOptedIn(); if (!this.isOptedIn) { this.logger.debug(`Telemetry is not opted-in.`); this.queue = []; @@ -209,7 +201,6 @@ export class TelemetryEventsSender { })); this.queue = []; - /* await this.sendEvents( toSend, telemetryUrl, @@ -217,7 +208,6 @@ export class TelemetryEventsSender { clusterInfo.version?.number, licenseInfo?.uid ); - */ } catch (err) { this.logger.warn(`Error sending telemetry events data: ${err}`); this.queue = []; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 3d2afa2e458da..255c6e43f45a1 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -76,7 +76,7 @@ export class TelemetryDiagTask { return; } - const isOptedIn = this.sender.isTelemetryOptedIn(); + const isOptedIn = await this.sender.isTelemetryOptedIn(); if (!isOptedIn) { this.logger.debug(`Telemetry is not opted-in.`); return; From ae8e2f4a63c3d323722e49fb2cef82bbcdf29164 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Fri, 4 Dec 2020 19:27:26 +0000 Subject: [PATCH 03/19] Expand wildcards. --- .../security_solution/server/lib/telemetry/sender.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 1d4054afc6820..cd9196e09983e 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -103,14 +103,12 @@ export class TelemetryEventsSender { public async fetchDiagnosticAlerts(fromTimestamp: string, toTimestamp: string) { const query = { + // logs-diagnostic.endpoint.collection-default + expand_wildcards: 'open,hidden', index: 'pete-hampton-test-index*', ignore_unavailable: true, size: 100, body: { - query: { - match_all: {}, - }, - /* query: { range: { '@timestamp': { @@ -119,7 +117,6 @@ export class TelemetryEventsSender { }, }, }, - */ }, }; From 76cb626ee9be7573c340fd5a666d391d6174f96d Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Mon, 7 Dec 2020 08:29:46 +0000 Subject: [PATCH 04/19] stage. --- x-pack/plugins/security_solution/server/lib/telemetry/sender.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index cd9196e09983e..dc27bc0840b44 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -103,8 +103,8 @@ export class TelemetryEventsSender { public async fetchDiagnosticAlerts(fromTimestamp: string, toTimestamp: string) { const query = { - // logs-diagnostic.endpoint.collection-default expand_wildcards: 'open,hidden', + // logs-diagnostic.endpoint.collection-default index: 'pete-hampton-test-index*', ignore_unavailable: true, size: 100, From 2d685a4ffb032eb4edef2be672ebcb415e0e4329 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Mon, 7 Dec 2020 13:51:13 +0000 Subject: [PATCH 05/19] Add rule.ruleset collection. --- .../security_solution/server/lib/telemetry/sender.ts | 12 +++++------- .../security_solution/server/lib/telemetry/task.ts | 11 +++-------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index dc27bc0840b44..eb992cac20f9c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -101,7 +101,7 @@ export class TelemetryEventsSender { } } - public async fetchDiagnosticAlerts(fromTimestamp: string, toTimestamp: string) { + public async fetchDiagnosticAlerts() { const query = { expand_wildcards: 'open,hidden', // logs-diagnostic.endpoint.collection-default @@ -110,12 +110,7 @@ export class TelemetryEventsSender { size: 100, body: { query: { - range: { - '@timestamp': { - gt: fromTimestamp, - lte: toTimestamp, - }, - }, + match_all: {}, }, }, }; @@ -298,6 +293,9 @@ const allowlistEventFields: AllowlistFields = { ecs: true, elastic: true, event: true, + rule: { + ruleset: true, + }, file: { name: true, path: true, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 255c6e43f45a1..5413fa1e53b4d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import moment from 'moment'; import { Logger } from 'src/core/server'; import { ConcreteTaskInstance, @@ -15,14 +14,13 @@ import { TelemetryEventsSender, TelemetryEvent } from './sender'; export const TelemetryDiagTaskConstants = { TIMEOUT: '1m', TYPE: 'security:telemetry-diagnostics', - INTERVAL: '10s', // TODO: update to 5m + INTERVAL: '2m', // TODO: update to 5m VERSION: '1.0.0', }; export class TelemetryDiagTask { private readonly logger: Logger; private readonly sender: TelemetryEventsSender; - private lastQueryTimestamp?: string; constructor( logger: Logger, @@ -70,6 +68,7 @@ export class TelemetryDiagTask { }; public runTask = async (taskId: string) => { + this.logger.debug(`running task ${taskId}`); // Check that this task is current if (taskId !== this.getTaskId()) { this.logger.debug(`Outdated task running: ${taskId}`); @@ -82,11 +81,7 @@ export class TelemetryDiagTask { return; } - const fetchFromTimestamp = - this.lastQueryTimestamp || moment.utc().subtract(5, 'm').toISOString(); - const fetchToTimestamp = moment.utc().toISOString(); - const response = await this.sender.fetchDiagnosticAlerts(fetchFromTimestamp, fetchToTimestamp); - this.lastQueryTimestamp = fetchToTimestamp; + const response = await this.sender.fetchDiagnosticAlerts(); const hits = response.hits?.hits || []; if (!Array.isArray(hits) || !hits.length) { From 7becfb7ace16f80b7ab09d41feabadf04848b250 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Mon, 7 Dec 2020 17:28:02 +0000 Subject: [PATCH 06/19] Update telemetry sender with correct query for loading diag alerts. --- .../server/lib/telemetry/sender.ts | 29 +++++++++++++------ .../server/lib/telemetry/task.ts | 12 ++++---- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index eb992cac20f9c..5f43d05858e0d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -83,16 +83,16 @@ export class TelemetryEventsSender { this.telemetryStart = telemetryStart; this.core = core; + if (taskManager && this.diagTask) { + this.logger.debug(`Starting diag task`); + this.diagTask.start(taskManager); + } + this.logger.debug(`Starting local task`); setTimeout(() => { this.sendIfDue(); this.intervalId = setInterval(() => this.sendIfDue(), this.checkIntervalMs); }, this.initialCheckDelayMs); - - if (taskManager && this.diagTask) { - this.logger.debug(`Starting diag task`); - this.diagTask.start(taskManager); - } } public stop() { @@ -104,14 +104,25 @@ export class TelemetryEventsSender { public async fetchDiagnosticAlerts() { const query = { expand_wildcards: 'open,hidden', - // logs-diagnostic.endpoint.collection-default - index: 'pete-hampton-test-index*', + index: 'logs-endpoint.diagnostic.collection-default*', ignore_unavailable: true, - size: 100, + size: this.maxQueueSize, body: { query: { - match_all: {}, + range: { + 'event.ingested': { + gte: 'now-5m', + lt: 'now', + }, + }, }, + sort: [ + { + 'event.ingested': { + order: 'asc', + }, + }, + ], }, }; diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 5413fa1e53b4d..6b79c3d792ba9 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -13,8 +13,8 @@ import { TelemetryEventsSender, TelemetryEvent } from './sender'; export const TelemetryDiagTaskConstants = { TIMEOUT: '1m', - TYPE: 'security:telemetry-diagnostics', - INTERVAL: '2m', // TODO: update to 5m + TYPE: 'security:endpoint-diagnostics', + INTERVAL: '5m', VERSION: '1.0.0', }; @@ -68,8 +68,7 @@ export class TelemetryDiagTask { }; public runTask = async (taskId: string) => { - this.logger.debug(`running task ${taskId}`); - // Check that this task is current + this.logger.debug(`Running task ${taskId}`); if (taskId !== this.getTaskId()) { this.logger.debug(`Outdated task running: ${taskId}`); return; @@ -89,7 +88,8 @@ export class TelemetryDiagTask { return; } - const alerts: TelemetryEvent[] = hits.map((h) => h._source); - this.sender.queueTelemetryEvents(alerts); + const diagAlerts: TelemetryEvent[] = hits.map((h) => h._source); + this.logger.debug(`Received ${diagAlerts.length} diagnostic alerts`); + this.sender.queueTelemetryEvents(diagAlerts); }; } From 1974c94792ef7ac6dc0027f5dc5074089393f961 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Tue, 8 Dec 2020 11:58:50 +0000 Subject: [PATCH 07/19] Add similar task tests to endpont artifact work. --- .../server/lib/telemetry/task.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts new file mode 100644 index 0000000000000..3003312b35715 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { loggingSystemMock } from 'src/core/server/mocks'; + +import { taskManagerMock } from '../../../../task_manager/server/mocks'; +import { TaskStatus } from '../../../task_manager/server'; + +import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './task'; +import { TelemetryEventsSender } from './sender'; + +describe('test', () => { + let logger: ReturnType; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + }); + + describe('basic diagnostic alert telemetry sanity checks', () => { + test('task can register', () => { + const telemetryDiagTask = new TelemetryDiagTask( + logger, + taskManagerMock.createSetup(), + new TelemetryEventsSender(logger) + ); + + expect(telemetryDiagTask).toBeInstanceOf(TelemetryDiagTask); + }); + }); + + test('diagnostic task should be registered', () => { + const mockTaskManager = taskManagerMock.createSetup(); + new TelemetryDiagTask(logger, mockTaskManager, new TelemetryEventsSender(logger)); + + expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); + }); + + test('task should be scheduled', async () => { + const mockTaskManagerSetup = taskManagerMock.createSetup(); + const telemetryDiagTask = new TelemetryDiagTask( + logger, + mockTaskManagerSetup, + new TelemetryEventsSender(logger) + ); + + const mockTaskManagerStart = taskManagerMock.createStart(); + await telemetryDiagTask.start(mockTaskManagerStart); + expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled(); + }); + + test('task should run', async () => { + const mockContext = new TelemetryEventsSender(logger); + const mockTaskManager = taskManagerMock.createSetup(); + const telemetryDiagTask = new TelemetryDiagTask(logger, mockTaskManager, mockContext); + + const mockTaskInstance = { + id: TelemetryDiagTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryDiagTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][TelemetryDiagTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(telemetryDiagTask.runTask).toHaveBeenCalled(); + }); +}); From 0a13f63be8524be8a022bc8cf2eaab774b0e562f Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Tue, 8 Dec 2020 13:08:58 +0000 Subject: [PATCH 08/19] Fix broken import statement. --- .../plugins/security_solution/server/lib/telemetry/task.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 3003312b35715..329a28d75d401 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -6,7 +6,7 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { taskManagerMock } from '../../../../task_manager/server/mocks'; -import { TaskStatus } from '../../../task_manager/server'; +import { TaskStatus } from '../../../../task_manager/server'; import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './task'; import { TelemetryEventsSender } from './sender'; From c709fc6488258a835ed461f066747ddfc78c574f Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Tue, 8 Dec 2020 14:58:07 +0000 Subject: [PATCH 09/19] Create sender mocks. --- .../server/lib/telemetry/mocks.ts | 36 +++++++++++++++++++ .../server/lib/telemetry/task.test.ts | 5 +-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts new file mode 100644 index 0000000000000..ccd6d1799568c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TelemetryEventsSender } from './sender'; +import { TelemetryDiagTask } from './task'; + +/** + * Creates a mocked Telemetry Events Sender + */ +export const createMockTelemetryEventsSender = (): jest.Mocked => { + return ({ + setup: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + fetchDiagnosticAlerts: jest.fn(), + queueTelemetryEvents: jest.fn(), + processEvents: jest.fn(), + isTelemetryOptedIn: jest.fn(), + sendIfDue: jest.fn(), + fetchClusterInfo: jest.fn(), + fetchTelemetryUrl: jest.fn(), + fetchLicenseInfo: jest.fn(), + copyLicenseFields: jest.fn(), + sendEvents: jest.fn(), + } as unknown) as jest.Mocked; +}; + +/** + * Creates a mocked Telemetry Diagnostic Task + */ +export class MockTelemetryDiagnosticTask extends TelemetryDiagTask { + public runTask = jest.fn(); +} diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 329a28d75d401..d726295a42392 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -10,6 +10,7 @@ import { TaskStatus } from '../../../../task_manager/server'; import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './task'; import { TelemetryEventsSender } from './sender'; +import { createMockTelemetryEventsSender, MockTelemetryDiagnosticTask } from './mocks'; describe('test', () => { let logger: ReturnType; @@ -51,9 +52,9 @@ describe('test', () => { }); test('task should run', async () => { - const mockContext = new TelemetryEventsSender(logger); + const mockContext = createMockTelemetryEventsSender(); const mockTaskManager = taskManagerMock.createSetup(); - const telemetryDiagTask = new TelemetryDiagTask(logger, mockTaskManager, mockContext); + const telemetryDiagTask = new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockContext); const mockTaskInstance = { id: TelemetryDiagTaskConstants.TYPE, From b8d9ddb584a380af24623a5509dfd385fb339d14 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Tue, 8 Dec 2020 16:33:43 +0000 Subject: [PATCH 10/19] Update test to check for func call. --- .../server/lib/telemetry/mocks.ts | 6 ++-- .../server/lib/telemetry/task.test.ts | 35 ++++++++++++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts index ccd6d1799568c..b8908b1ad30c3 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks.ts @@ -10,7 +10,9 @@ import { TelemetryDiagTask } from './task'; /** * Creates a mocked Telemetry Events Sender */ -export const createMockTelemetryEventsSender = (): jest.Mocked => { +export const createMockTelemetryEventsSender = ( + enableTelemtry: boolean +): jest.Mocked => { return ({ setup: jest.fn(), start: jest.fn(), @@ -18,7 +20,7 @@ export const createMockTelemetryEventsSender = (): jest.Mocked { @@ -24,7 +23,7 @@ describe('test', () => { const telemetryDiagTask = new TelemetryDiagTask( logger, taskManagerMock.createSetup(), - new TelemetryEventsSender(logger) + createMockTelemetryEventsSender(true) ); expect(telemetryDiagTask).toBeInstanceOf(TelemetryDiagTask); @@ -33,7 +32,7 @@ describe('test', () => { test('diagnostic task should be registered', () => { const mockTaskManager = taskManagerMock.createSetup(); - new TelemetryDiagTask(logger, mockTaskManager, new TelemetryEventsSender(logger)); + new TelemetryDiagTask(logger, mockTaskManager, createMockTelemetryEventsSender(true)); expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled(); }); @@ -43,7 +42,7 @@ describe('test', () => { const telemetryDiagTask = new TelemetryDiagTask( logger, mockTaskManagerSetup, - new TelemetryEventsSender(logger) + createMockTelemetryEventsSender(true) ); const mockTaskManagerStart = taskManagerMock.createStart(); @@ -52,7 +51,7 @@ describe('test', () => { }); test('task should run', async () => { - const mockContext = createMockTelemetryEventsSender(); + const mockContext = createMockTelemetryEventsSender(true); const mockTaskManager = taskManagerMock.createSetup(); const telemetryDiagTask = new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockContext); @@ -76,4 +75,30 @@ describe('test', () => { await taskRunner.run(); expect(telemetryDiagTask.runTask).toHaveBeenCalled(); }); + + test('task should not query elastic if telemetry is not opted in', async () => { + const mockSender = createMockTelemetryEventsSender(false); + const mockTaskManager = taskManagerMock.createSetup(); + const telemetryDiagTask = new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockSender); + + const mockTaskInstance = { + id: TelemetryDiagTaskConstants.TYPE, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: TelemetryDiagTaskConstants.TYPE, + }; + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][TelemetryDiagTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + await taskRunner.run(); + expect(mockSender.fetchDiagnosticAlerts).not.toHaveBeenCalled(); + }); }); From f0d226e2de68035961025b812bcfd0e6c851e8d1 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Tue, 8 Dec 2020 20:17:16 +0000 Subject: [PATCH 11/19] Update unused reference. --- .../plugins/security_solution/server/lib/telemetry/task.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index cd4171de839fc..e499f15af2b3c 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -79,7 +79,7 @@ describe('test', () => { test('task should not query elastic if telemetry is not opted in', async () => { const mockSender = createMockTelemetryEventsSender(false); const mockTaskManager = taskManagerMock.createSetup(); - const telemetryDiagTask = new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockSender); + new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockSender); const mockTaskInstance = { id: TelemetryDiagTaskConstants.TYPE, From 5638da9c7ddbd3e952cfa65baf24f947a3613c4e Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 11:57:52 +0000 Subject: [PATCH 12/19] record last run. --- .../server/lib/telemetry/sender.ts | 8 ++-- .../server/lib/telemetry/task.test.ts | 45 +++++++++++++++++++ .../server/lib/telemetry/task.ts | 32 +++++++++++-- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 5f43d05858e0d..42f00536464a3 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -101,7 +101,7 @@ export class TelemetryEventsSender { } } - public async fetchDiagnosticAlerts() { + public async fetchDiagnosticAlerts(executeFrom: string, executeTo: string) { const query = { expand_wildcards: 'open,hidden', index: 'logs-endpoint.diagnostic.collection-default*', @@ -111,15 +111,15 @@ export class TelemetryEventsSender { query: { range: { 'event.ingested': { - gte: 'now-5m', - lt: 'now', + gte: executeFrom, + lt: executeTo, }, }, }, sort: [ { 'event.ingested': { - order: 'asc', + order: 'desc', }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index e499f15af2b3c..4411ad1538c7d 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { moment } from 'moment'; import { loggingSystemMock } from 'src/core/server/mocks'; import { taskManagerMock } from '../../../../task_manager/server/mocks'; @@ -101,4 +102,48 @@ describe('test', () => { await taskRunner.run(); expect(mockSender.fetchDiagnosticAlerts).not.toHaveBeenCalled(); }); + + test('test -5 mins is returned when there is no previous task run', async () => { + const telemetryDiagTask = new TelemetryDiagTask( + logger, + taskManagerMock.createSetup(), + createMockTelemetryEventsSender(true) + ); + + const executeTo = moment().utc().toISOString(); + const executeFrom = undefined; + const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(5, 'minutes')); + }); + + test('test -6 mins is returned when there was a previous task run', async () => { + const telemetryDiagTask = new TelemetryDiagTask( + logger, + taskManagerMock.createSetup(), + createMockTelemetryEventsSender(true) + ); + + const executeTo = moment().utc().toISOString(); + const executeFrom = moment(executeTo).subtract(6, 'minutes'); + const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(executeFrom); + }); + + // it's possible if Kibana is down for a prolonged period the stored lastRun would have drifted + // if that is the case we will just roll it back to a 10 min search window + test('test 10 mins is returned when previous task run took longer than 10 minutes', async () => { + const telemetryDiagTask = new TelemetryDiagTask( + logger, + taskManagerMock.createSetup(), + createMockTelemetryEventsSender(true) + ); + + const executeTo = moment().utc().toISOString(); + const executeFrom = moment(executeTo).subtract(142, 'minutes'); + const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); + + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes')); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 6b79c3d792ba9..37fdaebf61c77 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import moment from 'moment'; import { Logger } from 'src/core/server'; import { ConcreteTaskInstance, @@ -37,7 +38,18 @@ export class TelemetryDiagTask { createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { return { run: async () => { - await this.runTask(taskInstance.id); + const executeTo = moment().utc().toISOString(); + const executeFrom = this.getLastExecutionTimestamp( + executeTo, + taskInstance.state?.lastExecutionTimestamp + ); + await this.runTask(taskInstance.id, executeFrom, executeTo); + + return { + state: { + lastExecutionTimestamp: executeTo, + }, + }; }, cancel: async () => {}, }; @@ -46,6 +58,20 @@ export class TelemetryDiagTask { }); } + public getLastExecutionTimestamp(executeTo: string, lastExecutionTimestamp?: string) { + if (lastExecutionTimestamp === undefined) { + this.logger.debug(`No last execution timestamp defined`); + return moment(executeTo).subtract(5, 'minutes').toISOString(); + } + + if (moment(lastExecutionTimestamp).diff(executeTo) >= 10) { + this.logger.debug(`last execution timestamp was greater than 10 minutes`); + return moment(executeTo).subtract(10, 'minutes').toISOString(); + } + + return lastExecutionTimestamp; + } + public start = async (taskManager: TaskManagerStartContract) => { try { await taskManager.ensureScheduled({ @@ -67,7 +93,7 @@ export class TelemetryDiagTask { return `${TelemetryDiagTaskConstants.TYPE}:${TelemetryDiagTaskConstants.VERSION}`; }; - public runTask = async (taskId: string) => { + public runTask = async (taskId: string, searchFrom: string, searchTo: string) => { this.logger.debug(`Running task ${taskId}`); if (taskId !== this.getTaskId()) { this.logger.debug(`Outdated task running: ${taskId}`); @@ -80,7 +106,7 @@ export class TelemetryDiagTask { return; } - const response = await this.sender.fetchDiagnosticAlerts(); + const response = await this.sender.fetchDiagnosticAlerts(searchFrom, searchTo); const hits = response.hits?.hits || []; if (!Array.isArray(hits) || !hits.length) { From 201813234422c68b8f308721580053ece64b10fb Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 12:02:24 +0000 Subject: [PATCH 13/19] Update index. --- x-pack/plugins/security_solution/server/lib/telemetry/sender.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 42f00536464a3..1f05b6507a0a2 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -104,7 +104,7 @@ export class TelemetryEventsSender { public async fetchDiagnosticAlerts(executeFrom: string, executeTo: string) { const query = { expand_wildcards: 'open,hidden', - index: 'logs-endpoint.diagnostic.collection-default*', + index: 'logs-endpoint.diagnostic.collection-*', ignore_unavailable: true, size: this.maxQueueSize, body: { From 43558f7c9aae8ef7611feae3de330233970197cb Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 12:37:41 +0000 Subject: [PATCH 14/19] fix import --- .../plugins/security_solution/server/lib/telemetry/task.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 4411ad1538c7d..7de29b8269899 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { moment } from 'moment'; +import moment from 'moment'; import { loggingSystemMock } from 'src/core/server/mocks'; import { taskManagerMock } from '../../../../task_manager/server/mocks'; From 07cdf3486ebbb4a8b802f8544b380266d98ce110 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 13:02:54 +0000 Subject: [PATCH 15/19] Fix test. --- .../security_solution/server/lib/telemetry/task.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 7de29b8269899..11fe9745d2022 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -125,7 +125,7 @@ describe('test', () => { ); const executeTo = moment().utc().toISOString(); - const executeFrom = moment(executeTo).subtract(6, 'minutes'); + const executeFrom = moment(executeTo).subtract(6, 'minutes').toISOString(); const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); expect(newExecuteFrom).toEqual(executeFrom); @@ -141,7 +141,7 @@ describe('test', () => { ); const executeTo = moment().utc().toISOString(); - const executeFrom = moment(executeTo).subtract(142, 'minutes'); + const executeFrom = moment(executeTo).subtract(142, 'minutes').toISOString(); const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes')); From 1ddd2efb8c7d9f3552fb4b43ef477f8954fdf718 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 13:28:48 +0000 Subject: [PATCH 16/19] test fix. --- .../plugins/security_solution/server/lib/telemetry/task.test.ts | 2 +- x-pack/plugins/security_solution/server/lib/telemetry/task.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index 11fe9745d2022..c634ed482acdf 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -114,7 +114,7 @@ describe('test', () => { const executeFrom = undefined; const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); - expect(newExecuteFrom).toEqual(moment(executeTo).subtract(5, 'minutes')); + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(5, 'minutes').toISOString()); }); test('test -6 mins is returned when there was a previous task run', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 37fdaebf61c77..16b29b5b540cd 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -64,7 +64,7 @@ export class TelemetryDiagTask { return moment(executeTo).subtract(5, 'minutes').toISOString(); } - if (moment(lastExecutionTimestamp).diff(executeTo) >= 10) { + if (moment(executeTo).diff(lastExecutionTimestamp) >= 10) { this.logger.debug(`last execution timestamp was greater than 10 minutes`); return moment(executeTo).subtract(10, 'minutes').toISOString(); } From d14f8f99a84c82a885548207cc0e829574548d56 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 13:48:40 +0000 Subject: [PATCH 17/19] Pass unit to time diff calc. --- x-pack/plugins/security_solution/server/lib/telemetry/task.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts index 16b29b5b540cd..28b8524f64516 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.ts @@ -64,7 +64,7 @@ export class TelemetryDiagTask { return moment(executeTo).subtract(5, 'minutes').toISOString(); } - if (moment(executeTo).diff(lastExecutionTimestamp) >= 10) { + if (moment(executeTo).diff(lastExecutionTimestamp, 'minutes') >= 10) { this.logger.debug(`last execution timestamp was greater than 10 minutes`); return moment(executeTo).subtract(10, 'minutes').toISOString(); } From 4c17199ddab8d74e090fe62a9aaea1a8a32a4e03 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 15:43:52 +0000 Subject: [PATCH 18/19] Tests should pass now hopefully. --- .../plugins/security_solution/server/lib/telemetry/task.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts index c634ed482acdf..d5856cef8c029 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts @@ -144,6 +144,6 @@ describe('test', () => { const executeFrom = moment(executeTo).subtract(142, 'minutes').toISOString(); const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom); - expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes')); + expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes').toISOString()); }); }); From b70772939553fd93b6f75ea7f849121814478475 Mon Sep 17 00:00:00 2001 From: Pete Hampton Date: Wed, 9 Dec 2020 17:40:17 +0000 Subject: [PATCH 19/19] Add additional process fields to allowlist. --- .../security_solution/server/lib/telemetry/sender.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 1f05b6507a0a2..65c69966f7caa 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -329,6 +329,8 @@ const allowlistEventFields: AllowlistFields = { executable: true, command_line: true, hash: true, + pid: true, + uptime: true, Ext: { code_signature: true, }, @@ -340,6 +342,12 @@ const allowlistEventFields: AllowlistFields = { Ext: { code_signature: true, }, + uptime: true, + pid: true, + ppid: true, + }, + token: { + integrity_level_name: true, }, }, };