From e0ca5645d0901e4dfcfaac19b293ab0201710970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Zaffarano?= Date: Wed, 7 Aug 2024 15:13:13 +0200 Subject: [PATCH] [Telemetry][Security Solution] Fix flaky tests (#190044) ## Summary Fixes: https://github.com/elastic/kibana/issues/188234 https://github.com/elastic/kibana/issues/187719 and https://github.com/elastic/kibana/issues/178918 The flakiness was while calculating the Detection Rules task invocations. It could have two different RCs: 1) The code didn't retry in case the task wasn't executed yet, which makes sense in a CI environment, which is slower than a dev environment; 2) The timestamp to filter out requests was calculated after the task was triggered, and if the task is executed fast enough, it could lead to empty responses because of that. (cherry picked from commit 358e104ebcc2bfcfeec3457b2e913d071d27745d) --- .../integration_tests/telemetry.test.ts | 97 ++++++++----------- 1 file changed, 43 insertions(+), 54 deletions(-) diff --git a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts index efe17ebe41296..558f7e7ade2f6 100644 --- a/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts +++ b/x-pack/plugins/security_solution/server/integration_tests/telemetry.test.ts @@ -148,8 +148,7 @@ describe('telemetry tasks', () => { }); }); - // FLAKY: https://github.com/elastic/kibana/issues/187719 - describe.skip('detection-rules', () => { + describe('detection-rules', () => { it('should execute when scheduled', async () => { await mockAndScheduleDetectionRulesTask(); @@ -169,8 +168,7 @@ describe('telemetry tasks', () => { }); it('should send task metrics', async () => { - const task = await mockAndScheduleDetectionRulesTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleDetectionRulesTask(); const requests = await getTaskMetricsRequests(task, started); @@ -181,13 +179,10 @@ describe('telemetry tasks', () => { }); }); - // FLAKY: https://github.com/elastic/kibana/issues/178918 - // FLAKY: https://github.com/elastic/kibana/issues/187720 - describe.skip('sender configuration', () => { + describe('sender configuration', () => { it('should use legacy sender by default', async () => { // launch a random task and verify it uses the new configuration - const task = await mockAndScheduleDetectionRulesTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleDetectionRulesTask(); const requests = await getTaskMetricsRequests(task, started); expect(requests.length).toBeGreaterThan(0); @@ -216,8 +211,7 @@ describe('telemetry tasks', () => { expect(found).toBeFalsy(); }); - const task = await mockAndScheduleDetectionRulesTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleDetectionRulesTask(); const requests = await getTaskMetricsRequests(task, started); expect(requests.length).toBeGreaterThan(0); @@ -333,8 +327,7 @@ describe('telemetry tasks', () => { Promise.reject(Error(errorMessage)) ); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const requests = await getTaskMetricsRequests(task, started); @@ -359,8 +352,7 @@ describe('telemetry tasks', () => { agentClient.listAgents = jest.fn((_) => Promise.reject(Error(errorMessage))); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -399,8 +391,7 @@ describe('telemetry tasks', () => { }) ); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -432,8 +423,7 @@ describe('telemetry tasks', () => { telemetryReceiver.fetchPolicyConfigs = jest.fn((_) => Promise.reject(Error(errorMessage))); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -476,8 +466,7 @@ describe('telemetry tasks', () => { } as unknown as AgentPolicy); }); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -510,8 +499,7 @@ describe('telemetry tasks', () => { return Promise.reject(Error(errorMessage)); }); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -543,8 +531,7 @@ describe('telemetry tasks', () => { return Promise.resolve(new Map()); }); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -577,8 +564,7 @@ describe('telemetry tasks', () => { return Promise.reject(Error(errorMessage)); }); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -613,8 +599,7 @@ describe('telemetry tasks', () => { return Promise.resolve(new Map()); }); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -661,8 +646,7 @@ describe('telemetry tasks', () => { return esClient.search(query); }); - const task = await mockAndScheduleEndpointTask(); - const started = performance.now(); + const [task, started] = await mockAndScheduleEndpointTask(); const endpointMetaRequests = await getEndpointMetaRequests(); @@ -686,8 +670,7 @@ describe('telemetry tasks', () => { }); }); - // FLAKY: https://github.com/elastic/kibana/issues/188234 - describe.skip('telemetry-prebuilt-rule-alerts', () => { + describe('telemetry-prebuilt-rule-alerts', () => { it('should execute when scheduled', async () => { await mockAndSchedulePrebuiltRulesTask(); @@ -720,8 +703,7 @@ describe('telemetry tasks', () => { telemetryReceiver.fetchPrebuiltRuleAlertsBatch = mockedGenerator; - const task = await mockAndSchedulePrebuiltRulesTask(); - const started = performance.now(); + const [task, started] = await mockAndSchedulePrebuiltRulesTask(); const requests = await getTaskMetricsRequests(task, started); @@ -779,7 +761,7 @@ describe('telemetry tasks', () => { }); } - async function mockAndScheduleDetectionRulesTask(): Promise { + async function mockAndScheduleDetectionRulesTask(): Promise<[SecurityTelemetryTask, number]> { const task = getTelemetryTask(tasks, 'security:telemetry-detection-rules'); // create some data @@ -795,50 +777,52 @@ describe('telemetry tasks', () => { exceptionsListItem.push(exceptionListItem); // schedule task to run ASAP - await eventually(async () => { + return eventually(async () => { + const started = performance.now(); await taskManagerPlugin.runSoon(task.getTaskId()); + return [task, started]; }); - - return task; } - async function mockAndScheduleEndpointTask(): Promise { + async function mockAndScheduleEndpointTask(): Promise<[SecurityTelemetryTask, number]> { const task = getTelemetryTask(tasks, 'security:endpoint-meta-telemetry'); await mockEndpointData(esClient, kibanaServer.coreStart.savedObjects); // schedule task to run ASAP - await eventually(async () => { + return eventually(async () => { + const started = performance.now(); await taskManagerPlugin.runSoon(task.getTaskId()); + return [task, started]; }); - - return task; } - async function mockAndSchedulePrebuiltRulesTask(): Promise { + async function mockAndSchedulePrebuiltRulesTask(): Promise<[SecurityTelemetryTask, number]> { const task = getTelemetryTask(tasks, 'security:telemetry-prebuilt-rule-alerts'); await mockPrebuiltRulesData(esClient); // schedule task to run ASAP - await eventually(async () => { + return eventually(async () => { + const started = performance.now(); await taskManagerPlugin.runSoon(task.getTaskId()); + return [task, started]; }); - - return task; } - async function mockAndScheduleEndpointDiagnosticsTask(): Promise { + async function mockAndScheduleEndpointDiagnosticsTask(): Promise< + [SecurityTelemetryTask, number] + > { const task = getTelemetryTask(tasks, 'security:endpoint-diagnostics'); await createMockedEndpointAlert(kibanaServer.coreStart.elasticsearch.client.asInternalUser); // schedule task to run ASAP - await eventually(async () => { + return eventually(async () => { + const started = performance.now(); await taskManagerPlugin.runSoon(task.getTaskId()); + return [task, started]; }); - - return task; } function mockAxiosGet(bufferConfig: unknown = fakeBufferAndSizesConfigAsyncDisabled) { @@ -875,6 +859,7 @@ describe('telemetry tasks', () => { requestConfig: AxiosRequestConfig | undefined; }> > { + const taskType = getTelemetryTaskType(task); return eventually(async () => { const calls = mockedAxiosPost.mock.calls.flatMap(([url, data, config]) => { return (data as string).split('\n').map((body) => { @@ -884,20 +869,24 @@ describe('telemetry tasks', () => { const requests = calls.filter(({ url, body }) => { return ( - body.indexOf(getTelemetryTaskType(task)) !== -1 && + body.indexOf(taskType) !== -1 && url.startsWith(ENDPOINT_STAGING) && url.endsWith('task-metrics') ); }); expect(requests.length).toBeGreaterThan(0); - return requests + const filtered = requests .map((r) => { return { taskMetric: JSON.parse(r.body) as TaskMetric, requestConfig: r.config, }; }) - .filter((t) => t.taskMetric.start_time >= olderThan); + .filter((t) => { + return t.taskMetric.start_time >= olderThan; + }); + expect(filtered.length).toBeGreaterThan(0); + return filtered; }); } });