From a3d71f2534a56dd070982d6d8c60e56aa43ae2fa Mon Sep 17 00:00:00 2001 From: Ash <1849116+ashokaditya@users.noreply.github.com> Date: Wed, 1 Nov 2023 21:33:36 +0100 Subject: [PATCH] [Security Solution][Endpoint] Unskip flaky tests (document signing & response console - isolate) (#169539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Split test to avoid long running times. Fixes https://github.com/elastic/kibana/issues/168296 https://github.com/elastic/kibana/issues/168390 **Flaky test runner** - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3754 x 150 (3 failures for document signing: 7, 101, 141) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3809 x 50 (all pass for document_signing.cy.ts) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3835 x 50 (2 fails for document signing and 1 fail for response console release due to timeout on response delay) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3842 x 25 (all pass for document_singing.cy.ts and response_console_actions.cy.ts) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3843 x 25 (3 fails for document_signing.cy.ts related to timeout for action response) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3844 x 100 (all pass for release, 3 failures for document signing due to time out waiting for action response) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3848 x 50 ( all pass, 1 unrelated failure related to `endpoints_list_response_console.cy·ts`) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/3849 x 50 (all pass) ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../response_actions/document_signing.cy.ts | 84 ++++++++++++++++++ .../response_console/execute.cy.ts | 2 +- .../response_console/file_operations.cy.ts | 2 +- .../response_console/isolation.cy.ts | 21 +++-- .../response_console/process_operations.cy.ts | 2 +- .../response_console_actions.cy.ts | 87 ------------------- .../cypress/support/response_actions.ts | 3 +- .../management/cypress/tasks/isolate.ts | 23 +++++ 8 files changed, 123 insertions(+), 101 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts delete mode 100644 x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console_actions.cy.ts diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts new file mode 100644 index 0000000000000..b806323726018 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/document_signing.cy.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PolicyData } from '../../../../../common/endpoint/types'; +import type { CreateAndEnrollEndpointHostResponse } from '../../../../../scripts/endpoint/common/endpoint_host_services'; +import { + openResponseConsoleFromEndpointList, + performCommandInputChecks, + submitCommand, + waitForEndpointListPageToBeLoaded, +} from '../../tasks/response_console'; +import type { IndexedFleetEndpointPolicyResponse } from '../../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; +import { createAgentPolicyTask, getEndpointIntegrationVersion } from '../../tasks/fleet'; +import { checkEndpointListForOnlyUnIsolatedHosts } from '../../tasks/isolate'; + +import { login } from '../../tasks/login'; +import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; +import { createEndpointHost } from '../../tasks/create_endpoint_host'; +import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; + +describe('Document signing:', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { + let indexedPolicy: IndexedFleetEndpointPolicyResponse; + let policy: PolicyData; + let createdHost: CreateAndEnrollEndpointHostResponse; + + before(() => { + getEndpointIntegrationVersion().then((version) => + createAgentPolicyTask(version).then((data) => { + indexedPolicy = data; + policy = indexedPolicy.integrationPolicies[0]; + + return enableAllPolicyProtections(policy.id).then(() => { + // Create and enroll a new Endpoint host + return createEndpointHost(policy.policy_id).then((host) => { + createdHost = host as CreateAndEnrollEndpointHostResponse; + }); + }); + }) + ); + }); + + after(() => { + if (createdHost) { + cy.task('destroyEndpointHost', createdHost); + } + + if (indexedPolicy) { + cy.task('deleteIndexedFleetEndpointPolicies', indexedPolicy); + } + + if (createdHost) { + deleteAllLoadedEndpointData({ endpointAgentIds: [createdHost.agentId] }); + } + }); + + beforeEach(() => { + login(); + }); + + it('should fail if data tampered', () => { + waitForEndpointListPageToBeLoaded(createdHost.hostname); + checkEndpointListForOnlyUnIsolatedHosts(); + openResponseConsoleFromEndpointList(); + performCommandInputChecks('isolate'); + + // stop host so that we ensure tamper happens before endpoint processes the action + cy.task('stopEndpointHost', createdHost.hostname); + // get action doc before we submit command, so we know when the new action doc is indexed + cy.task('getLatestActionDoc').then((previousActionDoc) => { + submitCommand(); + cy.task('tamperActionDoc', previousActionDoc); + }); + cy.task('startEndpointHost', createdHost.hostname); + + const actionValidationErrorMsg = + 'Fleet action response error: Failed to validate action signature; check Endpoint logs for details'; + // wait for 3 minutes for the response to be indexed + cy.contains(actionValidationErrorMsg, { timeout: 180000 }).should('exist'); + }); +}); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts index 9e0fab431fe3a..dad573bb09c2b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/execute.cy.ts @@ -26,7 +26,7 @@ describe('Response console', { tags: ['@ess', '@serverless'] }, () => { login(); }); - describe('Execute operations: execute', () => { + describe('Execute operations:', () => { const homeFilePath = process.env.CI || true ? '/home/vagrant' : `/home/ubuntu`; let indexedPolicy: IndexedFleetEndpointPolicyResponse; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts index a2355647bf238..ccea07c117820 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/file_operations.cy.ts @@ -26,7 +26,7 @@ describe('Response console', { tags: ['@ess', '@serverless'] }, () => { login(); }); - describe('File operations: get-file and upload', () => { + describe('File operations:', () => { const homeFilePath = process.env.CI || true ? '/home/vagrant' : `/home/ubuntu`; const fileContent = 'This is a test file for the get-file command.'; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/isolation.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/isolation.cy.ts index e41232e8fcf65..b748dca85f517 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/isolation.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/isolation.cy.ts @@ -19,6 +19,7 @@ import { createAgentPolicyTask, getEndpointIntegrationVersion } from '../../../t import { checkEndpointListForOnlyIsolatedHosts, checkEndpointListForOnlyUnIsolatedHosts, + isolateHostFromEndpointList, } from '../../../tasks/isolate'; import { login } from '../../../tasks/login'; @@ -31,7 +32,7 @@ describe('Response console', { tags: ['@ess', '@serverless'] }, () => { login(); }); - describe('Host Isolation: isolate and release an endpoint', () => { + describe('Host Isolation:', () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; @@ -66,28 +67,30 @@ describe('Response console', { tags: ['@ess', '@serverless'] }, () => { } }); - it('should isolate host from response console', () => { - const command = 'isolate'; + it('should release an isolated host from response console', () => { + const command = 'release'; waitForEndpointListPageToBeLoaded(createdHost.hostname); - checkEndpointListForOnlyUnIsolatedHosts(); + // isolate the host first + isolateHostFromEndpointList(); + checkEndpointListForOnlyIsolatedHosts(); openResponseConsoleFromEndpointList(); performCommandInputChecks(command); submitCommand(); waitForCommandToBeExecuted(command); waitForEndpointListPageToBeLoaded(createdHost.hostname); - checkEndpointListForOnlyIsolatedHosts(); + checkEndpointListForOnlyUnIsolatedHosts(); }); - it('should release host from response console', () => { - const command = 'release'; + it('should isolate a host from response console', () => { + const command = 'isolate'; waitForEndpointListPageToBeLoaded(createdHost.hostname); - checkEndpointListForOnlyIsolatedHosts(); + checkEndpointListForOnlyUnIsolatedHosts(); openResponseConsoleFromEndpointList(); performCommandInputChecks(command); submitCommand(); waitForCommandToBeExecuted(command); waitForEndpointListPageToBeLoaded(createdHost.hostname); - checkEndpointListForOnlyUnIsolatedHosts(); + checkEndpointListForOnlyIsolatedHosts(); }); }); }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts index 09a8ecc3363ef..3faf7373f5569 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/process_operations.cy.ts @@ -28,7 +28,7 @@ describe('Response console', { tags: ['@ess', '@serverless'] }, () => { login(); }); - describe('Processes operations: list, kill and suspend process', () => { + describe('Processes operations:', () => { let cronPID: string; let newCronPID: string; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console_actions.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console_actions.cy.ts deleted file mode 100644 index d78513f62eaf3..0000000000000 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console_actions.cy.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PolicyData } from '../../../../../common/endpoint/types'; -import type { CreateAndEnrollEndpointHostResponse } from '../../../../../scripts/endpoint/common/endpoint_host_services'; -import { - openResponseConsoleFromEndpointList, - performCommandInputChecks, - submitCommand, - waitForEndpointListPageToBeLoaded, -} from '../../tasks/response_console'; -import type { IndexedFleetEndpointPolicyResponse } from '../../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; -import { createAgentPolicyTask, getEndpointIntegrationVersion } from '../../tasks/fleet'; -import { checkEndpointListForOnlyUnIsolatedHosts } from '../../tasks/isolate'; - -import { login } from '../../tasks/login'; -import { enableAllPolicyProtections } from '../../tasks/endpoint_policy'; -import { createEndpointHost } from '../../tasks/create_endpoint_host'; -import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data'; - -describe('Response console', { tags: ['@ess', '@serverless'] }, () => { - beforeEach(() => { - login(); - }); - - // TODO: open PR https://github.com/elastic/kibana/pull/169539 - // FLAKY: https://github.com/elastic/kibana/issues/168296 - describe.skip('document signing', () => { - let indexedPolicy: IndexedFleetEndpointPolicyResponse; - let policy: PolicyData; - let createdHost: CreateAndEnrollEndpointHostResponse; - - before(() => { - getEndpointIntegrationVersion().then((version) => - createAgentPolicyTask(version).then((data) => { - indexedPolicy = data; - policy = indexedPolicy.integrationPolicies[0]; - - return enableAllPolicyProtections(policy.id).then(() => { - // Create and enroll a new Endpoint host - return createEndpointHost(policy.policy_id).then((host) => { - createdHost = host as CreateAndEnrollEndpointHostResponse; - }); - }); - }) - ); - }); - - after(() => { - if (createdHost) { - cy.task('destroyEndpointHost', createdHost); - } - - if (indexedPolicy) { - cy.task('deleteIndexedFleetEndpointPolicies', indexedPolicy); - } - - if (createdHost) { - deleteAllLoadedEndpointData({ endpointAgentIds: [createdHost.agentId] }); - } - }); - - it('should fail if data tampered', () => { - waitForEndpointListPageToBeLoaded(createdHost.hostname); - checkEndpointListForOnlyUnIsolatedHosts(); - openResponseConsoleFromEndpointList(); - performCommandInputChecks('isolate'); - - // stop host so that we ensure tamper happens before endpoint processes the action - cy.task('stopEndpointHost', createdHost.hostname); - // get action doc before we submit command so we know when the new action doc is indexed - cy.task('getLatestActionDoc').then((previousActionDoc) => { - submitCommand(); - cy.task('tamperActionDoc', previousActionDoc); - }); - cy.task('startEndpointHost', createdHost.hostname); - - const actionValidationErrorMsg = - 'Fleet action response error: Failed to validate action signature; check Endpoint logs for details'; - cy.contains(actionValidationErrorMsg, { timeout: 120000 }).should('exist'); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts b/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts index b22e50c660d83..baaf0e12c10e2 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/response_actions.ts @@ -48,10 +48,9 @@ export const responseActionTasks = ( const signed = get(newActionDoc, '_source.signed'); const signedDataBuffer = Buffer.from(signed.data, 'base64'); const signedDataJson = JSON.parse(signedDataBuffer.toString()); - const tamperedAgentsList = [...signedDataJson.agents, 'anotheragent']; const tamperedData = { ...signedDataJson, - agents: tamperedAgentsList, + comment: 'tampered data', }; const tamperedDataString = Buffer.from(JSON.stringify(tamperedData), 'utf8').toString( 'base64' diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/isolate.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/isolate.ts index e13bb832adce5..10aec51af291d 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/isolate.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/isolate.ts @@ -41,6 +41,29 @@ export const isolateHostWithComment = (comment: string, hostname: string): void cy.getByTestSubj('host_isolation_comment').type(comment); }; +export const isolateHostFromEndpointList = (index: number = 0): void => { + // open the action menu and click isolate action + cy.getByTestSubj('endpointTableRowActions').eq(index).click(); + cy.getByTestSubj('isolateLink').click(); + // isolation form, click confirm button + cy.getByTestSubj('hostIsolateConfirmButton').click(); + // return to endpoint details + cy.getByTestSubj('hostIsolateSuccessCompleteButton').click(); + // close details flyout + cy.getByTestSubj('euiFlyoutCloseButton').click(); + + // ensure the host is isolated, wait for 3 minutes for the host to be isolated + cy.wait(18000); + + cy.getByTestSubj('endpointListTable').within(() => { + cy.get('tbody tr') + .eq(index) + .within(() => { + cy.get('td').eq(1).should('contain.text', 'Isolated'); + }); + }); +}; + export const releaseHostWithComment = (comment: string, hostname: string): void => { cy.contains(`${hostname} is currently isolated.`); cy.getByTestSubj('endpointHostIsolationForm');