From 18e76dca68472a53ce70855343bb9b7f6b26e4bc Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 20 Jul 2020 10:35:53 -0600 Subject: [PATCH 1/2] Unskips tests by adding additional checks that should reduce flakiness. One test that failed was due to a promition issue --- .../basic/tests/add_prepackaged_rules.ts | 14 ++++++++++++-- .../tests/add_prepackaged_rules.ts | 14 ++++++++++++-- .../security_and_spaces/tests/create_rules_bulk.ts | 3 +-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts b/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts index 3340ac49b2d2d..ae2c84afd6571 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts @@ -13,6 +13,7 @@ import { deleteAllAlerts, deleteAllTimelines, deleteSignalsIndex, + waitFor, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -20,8 +21,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); - // FLAKY: https://github.com/elastic/kibana/issues/71867 - describe.skip('add_prepackaged_rules', () => { + describe('add_prepackaged_rules', () => { describe('validation errors', () => { it('should give an error that the index must exist first if it does not exist before adding prepackaged rules', async () => { const { body } = await supertest @@ -91,6 +91,16 @@ export default ({ getService }: FtrProviderContext): void => { .send() .expect(200); + // NOTE: I call the GET call until eventually it becomes consistent and that the number of rules to install are zero. + // This is to reduce flakiness where it can for a short period of time try to install the same rule twice an + await waitFor(async () => { + const { body } = await supertest + .get(`${DETECTION_ENGINE_PREPACKAGED_URL}/_status`) + .set('kbn-xsrf', 'true') + .expect(200); + return body.rules_not_installed === 0; + }); + const { body } = await supertest .put(DETECTION_ENGINE_PREPACKAGED_URL) .set('kbn-xsrf', 'true') diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts index 7671b1bd49744..ae2c84afd6571 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts @@ -13,6 +13,7 @@ import { deleteAllAlerts, deleteAllTimelines, deleteSignalsIndex, + waitFor, } from '../../utils'; // eslint-disable-next-line import/no-default-export @@ -20,8 +21,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); - // FLAKY: https://github.com/elastic/kibana/issues/71814 - describe.skip('add_prepackaged_rules', () => { + describe('add_prepackaged_rules', () => { describe('validation errors', () => { it('should give an error that the index must exist first if it does not exist before adding prepackaged rules', async () => { const { body } = await supertest @@ -91,6 +91,16 @@ export default ({ getService }: FtrProviderContext): void => { .send() .expect(200); + // NOTE: I call the GET call until eventually it becomes consistent and that the number of rules to install are zero. + // This is to reduce flakiness where it can for a short period of time try to install the same rule twice an + await waitFor(async () => { + const { body } = await supertest + .get(`${DETECTION_ENGINE_PREPACKAGED_URL}/_status`) + .set('kbn-xsrf', 'true') + .expect(200); + return body.rules_not_installed === 0; + }); + const { body } = await supertest .put(DETECTION_ENGINE_PREPACKAGED_URL) .set('kbn-xsrf', 'true') diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules_bulk.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules_bulk.ts index b59fd1b744e97..52865e43be750 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules_bulk.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_rules_bulk.ts @@ -29,8 +29,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); - // Failing ES promotion: https://github.com/elastic/kibana/issues/71612 - describe.skip('create_rules_bulk', () => { + describe('create_rules_bulk', () => { describe('validation errors', () => { it('should give a 200 even if the index does not exist as all bulks return a 200 but have an error of 409 bad request in the body', async () => { const { body } = await supertest From 709b61b27d3ed6eb65583e0406a2840075af97bb Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Mon, 20 Jul 2020 10:42:34 -0600 Subject: [PATCH 2/2] Updates the comments --- .../basic/tests/add_prepackaged_rules.ts | 2 +- .../security_and_spaces/tests/add_prepackaged_rules.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts b/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts index ae2c84afd6571..a022b7c79c079 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/add_prepackaged_rules.ts @@ -92,7 +92,7 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); // NOTE: I call the GET call until eventually it becomes consistent and that the number of rules to install are zero. - // This is to reduce flakiness where it can for a short period of time try to install the same rule twice an + // This is to reduce flakiness where it can for a short period of time try to install the same rule twice. await waitFor(async () => { const { body } = await supertest .get(`${DETECTION_ENGINE_PREPACKAGED_URL}/_status`) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts index ae2c84afd6571..40456737b8761 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/add_prepackaged_rules.ts @@ -92,7 +92,7 @@ export default ({ getService }: FtrProviderContext): void => { .expect(200); // NOTE: I call the GET call until eventually it becomes consistent and that the number of rules to install are zero. - // This is to reduce flakiness where it can for a short period of time try to install the same rule twice an + // This is to reduce flakiness where it can for a short period of time try to install the same rule the same rule twice. await waitFor(async () => { const { body } = await supertest .get(`${DETECTION_ENGINE_PREPACKAGED_URL}/_status`)