From 25a0551034de4a83e94a67b5226a4d50036b5a22 Mon Sep 17 00:00:00 2001 From: Wafaa Nasr Date: Wed, 9 Aug 2023 16:40:55 +0200 Subject: [PATCH] [Security Solution] [Flaky test] Fix double ftr test flakiness (#162990) ## Summary - Addresses https://github.com/elastic/kibana/issues/155122 ## Investigations *First attempt* > While investigating the three failing pipelines and examining the Artifacts, specifically target/test_failures/*.log, I discovered the root cause to be > Did not get an expected 200 "ok" when waiting for a list item (waitForListItem) yet. Retrying until we get a 200 "ok". body: {"message":"list_id: \"list_items.txt\" item of 1.0 does not exist","status_code":404}, status: 404 > > ![image](https://github.com/elastic/kibana/assets/12671903/ad3c4db9-e340-42c0-8a61-63e5387b9b83) > *Second attempt* > Upon incorporating additional logs to pinpoint the underlying issue, it was realized that the test will persist in its attempts until the preceding error is rectified, and it was determined that the list_items.txt file contains the correct item once located. > > The problem seemed to be associated with incomplete indexing of all indices signalsOpen.hits.hits when the test verifies against the anticipated value. > > To investigate the matter, two iterations were conducted, totaling 490 attempts. During these iterations, logs, and warnings were deliberately inserted to facilitate the tracing of the issue. > > https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2770#0189bb29-c258-4586-986a-2ac5e128eb52 > https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2768#0189baeb-8f09-488e-936f-69e72d089787 > > In this [Atifact](https://s3.amazonaws.com/buildkiteartifacts.com/e0f3970e-3a75-4621-919f-e6c773e2bb12/fedcad1e-0f36-4d55-bdc7-e1edbecf91fe/0189bb29-495f-4b7e-b34e-558ff6c02ce2/0189bb29-c258-4586-986a-2ac5e128eb52/target/test_failures/0189bb29-c258-4586-986a-2ac5e128eb52_d98d35afa0abd1ad03e28a68882291c4.log?response-content-type=text%2Fplain&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=ASIAQPCP3C7LXS7MSJGH%2F20230808%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230808T102039Z&X-Amz-Expires=600&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEOH%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLWVhc3QtMSJHMEUCIQDYG9De6RYDlVNcO0JGqO6Qc4dpzCQG1s%2BgaXTSimWc9QIgfkZgXlnVcScJCO56vz2VQx1buVTNEDEaVYhYmDgupEgq%2BgMIiv%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FARAAGgwwMzIzNzk3MDUzMDMiDOz2Gl2kfcuTHVaWyirOA4iHfkFBI7KBoy%2F97CuLMAxGLNIR8h4FtLCsY664lMhgDGd%2FBVi82zu%2B2Y5DAGEboOmb7C1q%2FX6CyesjDKP7sknXdfT7q8B4jBRK4QcVw2AAvC9%2BqMeN37eCfRXWT%2FrQAEdaWzxkJxk4BtFcATds3LJmmAlbYOT2L7vItmGzGs%2FnkRYevX%2Fq6hwxWqmOKNNot9hZ9sMJ3TZvGsA3045459BfyfObwIlzb13Pj6V7%2FMyTb87RjQazFiK%2Bb7qOXB%2B7ItaVAcpG7diXzPAif%2F%2FEiJedTHtBVh4LsMIKQtiASbcTh4X90kYtVMUcH%2BSG0B41WjWYPok0C3tHfQnGhGBIbSnqvMGmWgwGlhHNF1G7nIDYyWs32PQ3M3WmTGQuLIwp2hPzk4wFtgH%2FDQPWevRFRT%2FhzYJf05hyBT9BMcST9eeKTqEqs52PhCHDlCgo7K5bGjT86OO16MzEKdA77HRro8GqIh6mgTMyQ5gKpH38ueUWvQZnLix4T5HYw0EyJgRLRfiVIpKcH0jRobaFwKSfSYc%2FZ2FPHsAK31weedhy6tafwU7elNYiAaMRv4AQhB%2FlrGWYd6IjOrdgSFowlkd1m5%2FrYTY3k7%2F8D0vsxgjUETDX9semBjqlAWkLq%2FLaRV5KokMNXfT3lCVNJzpYeN1%2FdnInE%2BQn3Ub86kRgMOfTN2OgDHkuYSKsQ95tFHH49NfRn6rTw8tI2Tjs3pToqp65gCYh9MzUKTd2EWsgqdNn6y%2BbHRGFJxd7cacrX9Ghr4XyNme5aAeWAVYNXSqJ1KIbtWmRcwwhThAJHxWk5%2FoBwScZRUSYGWmkhaxnZXDjXg6LBuH7uZOJjf%2FLuFVdtQ%3D%3D&X-Amz-SignedHeaders=host&X-Amz-Signature=8a44c3a5e3b88b632097dc344ecec1ba6d6cb8e80e10d9058e96883e43d925b5) > > ![image](https://github.com/elastic/kibana/assets/12671903/bc58d389-3586-4343-a9f9-9577331d4876) > > ## Solution By modifying the third parameter of the `await waitForSignalsToBePresent(supertest, log, 1, [id]);` function to a value of 3 instead of the initial 1, we signify the number of alerts we need to await before conducting assertions. This change is made to accommodate the assertion of three alerts being generated, accounting for the exclusion of a single 1.0 value. --- .../group7/exception_operators_data_types/double.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group7/exception_operators_data_types/double.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group7/exception_operators_data_types/double.ts index 05d821b5932fb..39a878a82f896 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group7/exception_operators_data_types/double.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group7/exception_operators_data_types/double.ts @@ -33,8 +33,7 @@ export default ({ getService }: FtrProviderContext) => { const log = getService('log'); const es = getService('es'); - // Failing: See https://github.com/elastic/kibana/issues/155122 - describe.skip('Rule exception operators for data type double', () => { + describe('Rule exception operators for data type double', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/double'); await esArchiver.load('x-pack/test/functional/es_archives/rule_exceptions/double_as_string'); @@ -490,7 +489,6 @@ export default ({ getService }: FtrProviderContext) => { expect(hits).to.eql([]); }); }); - describe('working against string values in the data set', () => { it('will return 3 results if we have a list that includes 1 double', async () => { await importFile(supertest, log, 'double', ['1.0'], 'list_items.txt'); @@ -509,7 +507,7 @@ export default ({ getService }: FtrProviderContext) => { ], ]); await waitForRuleSuccess({ supertest, log, id }); - await waitForSignalsToBePresent(supertest, log, 1, [id]); + await waitForSignalsToBePresent(supertest, log, 3, [id]); const signalsOpen = await getSignalsById(supertest, log, id); const hits = signalsOpen.hits.hits.map((hit) => hit._source?.double).sort(); expect(hits).to.eql(['1.1', '1.2', '1.3']);