From bd6faf13c6fe25f1c1bf18770d800a9ad87f1ba6 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 27 Dec 2023 18:59:47 +0100 Subject: [PATCH 01/18] Unskip tests --- .../install_latest_bundled_prebuilt_rules.ts | 2 +- .../prebuilt_rules/management/fleet_integration.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index 60e0399df53f..a5899f59d82a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -29,7 +29,7 @@ export default ({ getService }: FtrProviderContext): void => { /* attempt to install it from the local file system. The API response from EPM provides /* us with the information of whether the package was installed from the registry or /* from a package that was bundled with Kibana */ - describe.skip('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => { + describe('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => { beforeEach(async () => { await deleteAllRules(supertest, log); await deleteAllPrebuiltRuleAssets(es); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index 5453ff5b34c7..0eff0a25c2cb 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -33,8 +33,7 @@ export default ({ getService }: FtrProviderContext): void => { * Unlike other tests that use mocks, this test uses actual rules from the * package storage and checks that they are installed. */ - // TODO: Fix and unskip https://github.com/elastic/kibana/issues/172107 - it.skip('should install prebuilt rules from the package storage', async () => { + it('should install prebuilt rules from the package storage', async () => { // Verify that status is empty before package installation const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); expect(statusBeforePackageInstallation.rules_installed).toBe(0); From 16f39079079ee8ecfc82ec274ff0e362030770c1 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 27 Dec 2023 19:10:40 +0100 Subject: [PATCH 02/18] Clear cache after getting status --- .../default_license/actions/update_actions.ts | 2 +- .../role_based_rule_exceptions_workflows.ts | 2 +- .../install_latest_bundled_prebuilt_rules.ts | 4 +- .../prerelease_packages.ts | 6 +-- .../install_large_prebuilt_rules_package.ts | 4 +- .../management/fleet_integration.ts | 12 +++-- .../management/get_prebuilt_rules_status.ts | 52 +++++++++---------- .../get_prebuilt_timelines_status.ts | 4 +- .../install_and_upgrade_prebuilt_rules.ts | 36 ++++++------- .../update_prebuilt_rules_package.ts | 6 +-- ...get_prebuilt_rules_and_timelines_status.ts | 5 ++ .../get_prebuilt_rules_status.ts | 5 ++ 12 files changed, 77 insertions(+), 61 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts index 333e76550a74..3b8689b47fcb 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts @@ -218,7 +218,7 @@ export default ({ getService }: FtrProviderContext) => { ); await updateRule(supertest, ruleToUpdate); - const status = await getPrebuiltRulesAndTimelinesStatus(supertest); + const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(status.rules_not_installed).toBe(0); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts index 870df90d3e47..a0b7145dbc95 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts @@ -526,7 +526,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - const status = await getPrebuiltRulesAndTimelinesStatus(supertest); + const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(status.rules_not_installed).toEqual(0); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index a5899f59d82a..5414e9f5152a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -52,7 +52,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should install prebuilt rules from the package that comes bundled with Kibana', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -71,7 +71,7 @@ export default ({ getService }: FtrProviderContext): void => { await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBeGreaterThan(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts index 448e325892a5..4f48868e2464 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should install latest stable version and ignore prerelease packages', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -59,7 +59,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(prebuiltRulesFleetPackage.status).toBe(200); // Get status of our prebuilt rules (nothing should be instaled yet) - const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBe(1); // 1 rule in package 99.0.0 expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -68,7 +68,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Verify that status is updated after package installation - const statusAfterRulesInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterRulesInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_installed).toBe(1); // 1 rule in package 99.0.0 expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts index e059cab5ae64..1c563b545a47 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts @@ -31,7 +31,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should install a package containing 15000 prebuilt rules without crashing', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusBeforePackageInstallation.rules_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_updated).toBe(0); @@ -40,7 +40,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusAfterPackageInstallation.rules_installed).toBe(750); expect(statusAfterPackageInstallation.rules_not_installed).toBe(0); expect(statusAfterPackageInstallation.rules_not_updated).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index 0eff0a25c2cb..b230e4a796bf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -35,7 +35,10 @@ export default ({ getService }: FtrProviderContext): void => { */ it('should install prebuilt rules from the package storage', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusBeforePackageInstallation.rules_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_updated).toBe(0); @@ -47,7 +50,10 @@ export default ({ getService }: FtrProviderContext): void => { }); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusAfterPackageInstallation.rules_installed).toBe(0); expect(statusAfterPackageInstallation.rules_not_installed).toBeGreaterThan(0); expect(statusAfterPackageInstallation.rules_not_updated).toBe(0); @@ -58,7 +64,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.rules_updated).toBe(0); // Verify that status is updated after rules installation - const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusAfterRuleInstallation.rules_installed).toBe(response.rules_installed); expect(statusAfterRuleInstallation.rules_not_installed).toBe(0); expect(statusAfterRuleInstallation.rules_not_updated).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts index 16dba2761694..bfbe4f00cb32 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts @@ -36,7 +36,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should return empty structure when no prebuilt rule assets', async () => { - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: 0, @@ -48,7 +48,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should not update the prebuilt rule status when a custom rule is added', async () => { await createRule(supertest, log, getSimpleRule()); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: 0, @@ -68,7 +68,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: RULES_COUNT, @@ -81,7 +81,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -95,7 +95,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); await deleteRule(supertest, 'rule-1'); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT - 1, num_prebuilt_rules_to_install: 1, @@ -115,7 +115,7 @@ export default ({ getService }: FtrProviderContext): void => { ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -137,7 +137,7 @@ export default ({ getService }: FtrProviderContext): void => { // Upgrade all rules await upgradePrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -156,7 +156,7 @@ export default ({ getService }: FtrProviderContext): void => { // Recreate the rules without bumping any versions await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -179,7 +179,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: RULES_COUNT, @@ -192,7 +192,7 @@ export default ({ getService }: FtrProviderContext): void => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -206,7 +206,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); await deleteRule(supertest, 'rule-1'); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT - 1, num_prebuilt_rules_to_install: 1, @@ -224,7 +224,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -245,7 +245,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -271,7 +271,7 @@ export default ({ getService }: FtrProviderContext): void => { // Upgrade the rule await upgradePrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -291,7 +291,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should return empty structure when no rules package installed', async () => { - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, @@ -304,7 +304,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should show that one custom rule is installed when a custom rule is added', async () => { await createRule(supertest, log, getSimpleRule()); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 1, rules_installed: 0, @@ -324,7 +324,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, @@ -338,7 +338,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRulesAndTimelines(es, supertest); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -352,7 +352,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); await deleteRule(supertest, 'rule-1'); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT - 1, @@ -372,7 +372,7 @@ export default ({ getService }: FtrProviderContext): void => { ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -391,7 +391,7 @@ export default ({ getService }: FtrProviderContext): void => { // Recreate the rules without bumping any versions await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -413,7 +413,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, @@ -427,7 +427,7 @@ export default ({ getService }: FtrProviderContext): void => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRulesAndTimelines(es, supertest); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -441,7 +441,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); await deleteRule(supertest, 'rule-1'); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT - 1, @@ -459,7 +459,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -480,7 +480,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts index 9acef16bfbeb..cf7a2c961e06 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts @@ -23,7 +23,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should return the number of timeline templates available to install', async () => { - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ timelines_installed: 0, @@ -36,7 +36,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of installed timeline templates after installing them', async () => { await installPrebuiltRulesAndTimelines(es, supertest); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ timelines_installed: expect.any(Number), timelines_not_installed: 0, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts index a75c8f87bd78..4a85fe68fcba 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts @@ -80,7 +80,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(1); // Call the install prebuilt rules again and check that the missing rule was installed @@ -102,7 +102,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_updated).toBe(1); // Call the install prebuilt rules again and check that the outdated rule was updated @@ -117,7 +117,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(0); expect(statusResponse.rules_not_updated).toBe(0); @@ -162,7 +162,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(1); // Call the install prebuilt rules again and check that the missing rule was installed @@ -183,7 +183,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(1); @@ -199,7 +199,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -258,7 +258,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(0); // Call the install prebuilt rules again and check that no rules were installed @@ -276,7 +276,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(1); // Call the install prebuilt rules endpoint again and check that the missing rule was installed @@ -296,7 +296,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_updated).toBe(1); // Call the install prebuilt rules again and check that the outdated rule was updated @@ -304,7 +304,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.rules_installed).toBe(0); expect(response.rules_updated).toBe(1); - const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(_statusResponse.rules_not_installed).toBe(0); expect(_statusResponse.rules_not_updated).toBe(0); }); @@ -323,7 +323,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_updated).toBe(1); expect(statusResponse.rules_not_installed).toBe(0); @@ -332,7 +332,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.rules_installed).toBe(0); expect(response.rules_updated).toBe(1); - const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(_statusResponse.rules_not_updated).toBe(0); expect(_statusResponse.rules_not_installed).toBe(0); }); @@ -366,7 +366,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); // Call the install prebuilt rules again and check that no rules were installed @@ -384,7 +384,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(1); // Call the install prebuilt rules endpoint again and check that the missing rule was installed @@ -404,7 +404,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that the prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(1); // Call the upgrade prebuilt rules endpoint and check that the outdated rule was updated @@ -412,7 +412,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.summary.succeeded).toBe(1); expect(response.summary.total).toBe(1); - const status = await getPrebuiltRulesStatus(supertest); + const status = await getPrebuiltRulesStatus(es, supertest); expect(status.stats.num_prebuilt_rules_to_install).toBe(0); expect(status.stats.num_prebuilt_rules_to_upgrade).toBe(0); }); @@ -431,7 +431,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that the prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(1); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); @@ -440,7 +440,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.summary.succeeded).toBe(1); expect(response.summary.total).toBe(1); - const status = await getPrebuiltRulesStatus(supertest); + const status = await getPrebuiltRulesStatus(es, supertest); expect(status.stats.num_prebuilt_rules_to_install).toBe(0); expect(status.stats.num_prebuilt_rules_to_upgrade).toBe(0); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts index 981bfd712678..d6eb09d1411a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts @@ -96,7 +96,7 @@ export default ({ getService }: FtrProviderContext): void => { // PART 1: Install prebuilt rules from the previous minor version as the current version // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -112,7 +112,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(installPreviousPackageResponse.items.length).toBeGreaterThan(0); // Verify that status is updated after the installation of package "N-1" - const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBeGreaterThan(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -162,7 +162,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0); // Verify status after intallation of the latest package - const statusAfterLatestPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterLatestPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect( statusAfterLatestPackageInstallation.stats.num_prebuilt_rules_installed ).toBeGreaterThan(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts index 2d03e597dc5a..742e11d10e67 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts @@ -10,6 +10,8 @@ import { PREBUILT_RULES_STATUS_URL, } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; +import type { Client } from '@elastic/elasticsearch'; +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; /** * (LEGACY) @@ -18,6 +20,7 @@ import type SuperTest from 'supertest'; * @param supertest The supertest deps */ export const getPrebuiltRulesAndTimelinesStatus = async ( + es: Client, supertest: SuperTest.SuperTest ): Promise => { const response = await supertest @@ -27,5 +30,7 @@ export const getPrebuiltRulesAndTimelinesStatus = async ( .send() .expect(200); + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); + return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts index 0f785f8a7745..b0e5b714a83e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts @@ -10,6 +10,8 @@ import { GetPrebuiltRulesStatusResponseBody, } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; +import type { Client } from '@elastic/elasticsearch'; +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; /** * Helper to retrieve the prebuilt rules status @@ -17,6 +19,7 @@ import type SuperTest from 'supertest'; * @param supertest The supertest deps */ export const getPrebuiltRulesStatus = async ( + es: Client, supertest: SuperTest.SuperTest ): Promise => { const response = await supertest @@ -27,5 +30,7 @@ export const getPrebuiltRulesStatus = async ( .send() .expect(200); + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); + return response.body; }; From 7ad1ab3acf9c72b85a1cdae46782725e646fccdb Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 00:22:38 +0100 Subject: [PATCH 03/18] Changes --- .../role_based_rule_exceptions_workflows.ts | 1 + .../install_large_prebuilt_rules_package.ts | 10 ++- .../management/fleet_integration.ts | 1 + .../prebuilt_rules/management/index.ts | 6 +- .../rule_bulk_actions/perform_bulk_action.ts | 1 + .../perform_bulk_action_dry_run.ts | 1 + .../get_rule_management_filters.ts | 1 + .../detections_response/utils/index.ts | 1 + .../detections_response/utils/retry.ts | 45 +++++++++++ ...get_prebuilt_rules_and_timelines_status.ts | 5 +- .../get_prebuilt_rules_status.ts | 5 +- .../install_prebuilt_rules_fleet_package.ts | 76 ++++++++++++++----- 12 files changed, 125 insertions(+), 28 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts index a0b7145dbc95..b114766a7f67 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts @@ -73,6 +73,7 @@ export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const log = getService('log'); const es = getService('es'); + const retry = getService('retry'); // TODO: add a new service const config = getService('config'); const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts index 1c563b545a47..20f0e214c105 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts @@ -31,7 +31,10 @@ export default ({ getService }: FtrProviderContext): void => { it('should install a package containing 15000 prebuilt rules without crashing', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusBeforePackageInstallation.rules_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_updated).toBe(0); @@ -40,7 +43,10 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusAfterPackageInstallation.rules_installed).toBe(750); expect(statusAfterPackageInstallation.rules_not_installed).toBe(0); expect(statusAfterPackageInstallation.rules_not_updated).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index b230e4a796bf..b3f1c4537357 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -20,6 +20,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); describe('@ess @serverless @skipInQA install_prebuilt_rules_from_real_package', () => { beforeEach(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts index 52a745e0e797..ed87c0e3b4dd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts @@ -9,9 +9,9 @@ import { FtrProviderContext } from '../../../../../ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { describe('Detection Engine API - Prebuilt Rules Management', function () { - loadTestFile(require.resolve('./get_prebuilt_rules_status')); - loadTestFile(require.resolve('./get_prebuilt_timelines_status')); - loadTestFile(require.resolve('./install_and_upgrade_prebuilt_rules')); + // loadTestFile(require.resolve('./get_prebuilt_rules_status')); + // loadTestFile(require.resolve('./get_prebuilt_timelines_status')); + // loadTestFile(require.resolve('./install_and_upgrade_prebuilt_rules')); loadTestFile(require.resolve('./fleet_integration')); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts index f7e48ac30a6e..28e805983865 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts @@ -43,6 +43,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const log = getService('log'); const esArchiver = getService('esArchiver'); + const retry = getService('retry'); // TODO: add a new service const config = getService('config'); const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts index 216d00533e10..88943aca5d5c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts @@ -29,6 +29,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); const es = getService('es'); + const retry = getService('retry'); const postDryRunBulkAction = () => supertest diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts index 37853b865e16..eb583a075683 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts @@ -22,6 +22,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); const log = getService('log'); + const retry = getService('retry'); describe('@ess @serverless get_rule_management_filters', () => { beforeEach(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts index 0fed526f9ef3..415569827b85 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts @@ -25,3 +25,4 @@ export * from './wait_for_index_to_populate'; export * from './get_stats'; export * from './get_detection_metrics_from_body'; export * from './get_stats_url'; +export * from './retry'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts new file mode 100644 index 000000000000..6362149daf4a --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RetryService } from '@kbn/ftr-common-functional-services'; + +/* + * Retry wrapper for async supertests, with a maximum number of retries + */ +export const retry = async ({ + test, + retryService, + retries, +}: { + test: () => Promise; + retryService: RetryService; + retries: number; +}): Promise => { + let retryAttempt = 0; + const response = await retryService.try( + async () => { + if (retryAttempt > retries) { + // Log error message if we reached the maximum number of retries + // but don't throw an error, return it to break the retry loop. + return new Error('Reached maximum number of retries for test.'); + } + + retryAttempt = retryAttempt + 1; + + return test(); + }, + undefined, + 200 + ); + + // Now throw the error in order to fail the test. + if (response instanceof Error) { + throw response; + } + + return response; +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts index 742e11d10e67..c7b16bc3fc17 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts @@ -23,6 +23,9 @@ export const getPrebuiltRulesAndTimelinesStatus = async ( es: Client, supertest: SuperTest.SuperTest ): Promise => { + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); + const response = await supertest .get(PREBUILT_RULES_STATUS_URL) .set('kbn-xsrf', 'true') @@ -30,7 +33,5 @@ export const getPrebuiltRulesAndTimelinesStatus = async ( .send() .expect(200); - await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); - return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts index b0e5b714a83e..45cbc3e17873 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts @@ -22,6 +22,9 @@ export const getPrebuiltRulesStatus = async ( es: Client, supertest: SuperTest.SuperTest ): Promise => { + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); + const response = await supertest .get(GET_PREBUILT_RULES_STATUS_URL) .set('kbn-xsrf', 'true') @@ -30,7 +33,5 @@ export const getPrebuiltRulesStatus = async ( .send() .expect(200); - await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); - return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index cc899ecc1dcc..7dbe4ee6c9a1 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -6,10 +6,16 @@ */ import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; -import { epmRouteService } from '@kbn/fleet-plugin/common'; +import { + BulkInstallPackageInfo, + BulkInstallPackagesResponse, + epmRouteService, +} from '@kbn/fleet-plugin/common'; import type { Client } from '@elastic/elasticsearch'; +import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; - +import { RetryService } from '@kbn/ftr-common-functional-services'; +import { retry } from '../../retry'; /** * Installs the `security_detection_engine` package via fleet API. This will * create real `security-rule` asset saved objects from the package. @@ -23,32 +29,64 @@ export const installPrebuiltRulesFleetPackage = async ({ supertest, version, overrideExistingPackage, + retryService, }: { es: Client; supertest: SuperTest.SuperTest; version?: string; overrideExistingPackage: boolean; -}): Promise => { + retryService: RetryService; +}): Promise => { if (version) { // Install a specific version - await supertest - .post(epmRouteService.getInstallPath('security_detection_engine', version)) - .set('kbn-xsrf', 'true') - .send({ - force: overrideExistingPackage, - }) - .expect(200); + const response = await retry({ + test: async () => { + const testResponse = await supertest + .post(epmRouteService.getInstallPath('security_detection_engine', version)) + .set('kbn-xsrf', 'true') + .send({ + force: overrideExistingPackage, + }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: 1, + }); + + return response as InstallPackageResponse; } else { // Install the latest version - await supertest - .post(epmRouteService.getBulkInstallPath()) - .query({ prerelease: true }) - .set('kbn-xsrf', 'true') - .send({ - packages: ['security_detection_engine'], - force: overrideExistingPackage, - }) - .expect(200); + const response = await retry({ + test: async () => { + const testResponse = await supertest + .post(epmRouteService.getBulkInstallPath()) + .query({ prerelease: true }) + .set('kbn-xsrf', 'true') + .send({ + packages: ['security_detection_engine'], + force: overrideExistingPackage, + }) + .expect(200); + + const body = testResponse.body as BulkInstallPackagesResponse; + + // First and only item in the response should be the security_detection_engine package + expect(body.items[0]).toBeDefined(); + expect((body.items[0] as BulkInstallPackageInfo).result.assets).toBeDefined(); + // Should have installed at least 1 security-rule asset + expect((body.items[0] as BulkInstallPackageInfo).result.assets?.length).toBeGreaterThan(0); + + return body; + }, + retryService, + retries: 1, + }); + + return response as BulkInstallPackagesResponse; } // Before we proceed, we need to refresh saved object indices. From 8a6f22b9cecfd776f100ee0b47c6fe4a09074833 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 00:43:18 +0100 Subject: [PATCH 04/18] Create reusable refresh indices --- .../role_based_rule_exceptions_workflows.ts | 1 - .../install_latest_bundled_prebuilt_rules.ts | 4 +- .../prerelease_packages.ts | 4 +- .../management/fleet_integration.ts | 1 + .../update_prebuilt_rules_package.ts | 7 ++- .../rule_bulk_actions/perform_bulk_action.ts | 1 - .../perform_bulk_action_dry_run.ts | 1 - .../get_rule_management_filters.ts | 1 - .../install_fleet_package_by_url.ts | 62 +++++++++++++------ .../install_prebuilt_rules_fleet_package.ts | 2 +- 10 files changed, 56 insertions(+), 28 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts index b114766a7f67..a0b7145dbc95 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts @@ -73,7 +73,6 @@ export default ({ getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); const log = getService('log'); const es = getService('es'); - const retry = getService('retry'); // TODO: add a new service const config = getService('config'); const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index 5414e9f5152a..54424b39ecd9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -22,6 +22,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); // FLAKY: https://github.com/elastic/kibana/issues/171380 /* This test simulates an air-gapped environment in which the user doesn't have access to EPR. @@ -60,7 +61,8 @@ export default ({ getService }: FtrProviderContext): void => { const bundledInstallResponse = await installPrebuiltRulesPackageByVersion( es, supertest, - '99.0.0' + '99.0.0', + retry ); // As opposed to "registry" diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts index 4f48868e2464..f9450929cf53 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts @@ -22,6 +22,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); /* This test makes use of the mock packages created in the '/fleet_bundled_packages' folder, /* in order to assert that, in production environments, the latest stable version of the package @@ -47,7 +48,8 @@ export default ({ getService }: FtrProviderContext): void => { // Install package without specifying version to check if latest stable version is installed const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI( es, - supertest + supertest, + retry ); expect(fleetPackageInstallationResponse.items.length).toBe(1); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index b3f1c4537357..b8f29010d5a0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -48,6 +48,7 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, overrideExistingPackage: true, + retryService: retry, }); // Verify that status is updated after package installation diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts index d6eb09d1411a..2711f28ce88e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts @@ -29,6 +29,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); let currentVersion: string; let previousVersion: string; @@ -105,7 +106,8 @@ export default ({ getService }: FtrProviderContext): void => { const installPreviousPackageResponse = await installPrebuiltRulesPackageByVersion( es, supertest, - previousVersion + previousVersion, + retry ); expect(installPreviousPackageResponse._meta.install_source).toBe('registry'); @@ -157,7 +159,8 @@ export default ({ getService }: FtrProviderContext): void => { const installLatestPackageResponse = await installPrebuiltRulesPackageByVersion( es, supertest, - currentVersion + currentVersion, + retry ); expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts index 28e805983865..f7e48ac30a6e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action.ts @@ -43,7 +43,6 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const log = getService('log'); const esArchiver = getService('esArchiver'); - const retry = getService('retry'); // TODO: add a new service const config = getService('config'); const ELASTICSEARCH_USERNAME = config.get('servers.kibana.username'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts index 88943aca5d5c..216d00533e10 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_bulk_actions/perform_bulk_action_dry_run.ts @@ -29,7 +29,6 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); const es = getService('es'); - const retry = getService('retry'); const postDryRunBulkAction = () => supertest diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts index eb583a075683..37853b865e16 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_management/get_rule_management_filters.ts @@ -22,7 +22,6 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const es = getService('es'); const log = getService('log'); - const retry = getService('retry'); describe('@ess @serverless get_rule_management_filters', () => { beforeEach(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 259369346cc8..03f974445983 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -9,6 +9,8 @@ import type SuperTest from 'supertest'; import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; +import { RetryService } from '@kbn/ftr-common-functional-services'; +import { retry } from '../..'; /** * Installs latest available non-prerelease prebuilt rules package `security_detection_engine`. @@ -21,15 +23,26 @@ import { epmRouteService } from '@kbn/fleet-plugin/common'; export const installPrebuiltRulesPackageViaFleetAPI = async ( es: Client, - supertest: SuperTest.SuperTest + supertest: SuperTest.SuperTest, + retryService: RetryService ): Promise => { - const fleetResponse = await supertest - .post(`/api/fleet/epm/packages/security_detection_engine`) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(200); + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(`/api/fleet/epm/packages/security_detection_engine`) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: 2, + }); // Before we proceed, we need to refresh saved object indices. // At the previous step we installed the Fleet package with prebuilt detection rules. @@ -47,11 +60,11 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( // fix it deterministically we have to refresh saved object indices and wait until it's done. await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - return fleetResponse.body as InstallPackageResponse; + return fleetResponse as InstallPackageResponse; }; /** * Installs prebuilt rules package `security_detection_engine`, passing in the version - * of the package as a parameter to the utl. + * of the package as a parameter to the url. * * @param es Elasticsearch client * @param supertest SuperTest instance @@ -62,17 +75,28 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( export const installPrebuiltRulesPackageByVersion = async ( es: Client, supertest: SuperTest.SuperTest, - version: string + version: string, + retryService: RetryService ): Promise => { - const fleetResponse = await supertest - .post(epmRouteService.getInstallPath('security_detection_engine', version)) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(200); + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(epmRouteService.getInstallPath('security_detection_engine', version)) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: 2, + }); await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - return fleetResponse.body as InstallPackageResponse; + return fleetResponse as InstallPackageResponse; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index 7dbe4ee6c9a1..6a5f7fbf2ae9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -77,7 +77,7 @@ export const installPrebuiltRulesFleetPackage = async ({ // First and only item in the response should be the security_detection_engine package expect(body.items[0]).toBeDefined(); expect((body.items[0] as BulkInstallPackageInfo).result.assets).toBeDefined(); - // Should have installed at least 1 security-rule asset + // Endpoint call should have installed at least 1 security-rule asset expect((body.items[0] as BulkInstallPackageInfo).result.assets?.length).toBeGreaterThan(0); return body; From b927f5df3216b5268e57b92dd63ba7a852abbaf5 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 00:48:10 +0100 Subject: [PATCH 05/18] Changes --- .../utils/rules/es_indices.ts | 26 +++++++++++++++++++ .../detections_response/utils/rules/index.ts | 1 + .../install_prebuilt_rules_fleet_package.ts | 23 +++++----------- 3 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts new file mode 100644 index 000000000000..8ee99be7e09e --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import type { Client } from '@elastic/elasticsearch'; + +export const refreshSavedObjectIndices = async (es: Client) => { + // In cases such as when installing the prebuilt detection rules SO of type 'security-rule', + // The savedObjectsClient does this with a call with explicit `refresh: false`. + // So, despite of the fact that the endpoint waits until the prebuilt rule will be + // successfully indexed, it doesn't wait until they become "visible" for subsequent read + // operations. + // And this is usually what we do next in integration tests: we read these SOs with utility + // function such as getPrebuiltRulesAndTimelinesStatus(). + // This can cause race condition between a write and subsequent read operation, and to + // fix it deterministically we have to refresh saved object indices and wait until it's done. + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + + // Additionally, we need to clear the cache to ensure that the next read operation will + // not return stale data. + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts index 90f3ae07871c..db0695546dcd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts @@ -13,6 +13,7 @@ export * from './create_non_security_rule'; export * from './downgrade_immutable_rule'; export * from './delete_all_rules'; export * from './delete_rule'; +export * from './es_indices'; export * from './fetch_rule'; export * from './find_immutable_rule_by_id'; export * from './get_simple_rule'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index 6a5f7fbf2ae9..c59978993742 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; import { BulkInstallPackageInfo, BulkInstallPackagesResponse, @@ -16,6 +15,8 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; import { RetryService } from '@kbn/ftr-common-functional-services'; import { retry } from '../../retry'; +import { refreshSavedObjectIndices } from '../..'; + /** * Installs the `security_detection_engine` package via fleet API. This will * create real `security-rule` asset saved objects from the package. @@ -57,6 +58,8 @@ export const installPrebuiltRulesFleetPackage = async ({ retries: 1, }); + await refreshSavedObjectIndices(es); + return response as InstallPackageResponse; } else { // Install the latest version @@ -86,24 +89,10 @@ export const installPrebuiltRulesFleetPackage = async ({ retries: 1, }); + await refreshSavedObjectIndices(es); + return response as BulkInstallPackagesResponse; } - - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the Fleet package with prebuilt detection rules. - // Prebuilt rules are assets that Fleet indexes as saved objects of a certain type. - // Fleet does this via a savedObjectsClient.import() call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule assets will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // Now, the time left until the next refresh can be anything from 0 to the default value, and - // it depends on the time when savedObjectsClient.import() call happens relative to the time of - // the next refresh. Also, probably the refresh time can be delayed when ES is under load? - // Anyway, this can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); }; /** From 077a2e70c91298b5ec71044157d182f4cba0d033 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 00:51:40 +0100 Subject: [PATCH 06/18] Use common util --- ...get_prebuilt_rules_and_timelines_status.ts | 5 ++--- .../get_prebuilt_rules_status.ts | 5 ++--- .../install_fleet_package_by_url.ts | 21 +++---------------- .../prebuilt_rules/install_prebuilt_rules.ts | 14 ++----------- .../install_prebuilt_rules_and_timelines.ts | 14 ++----------- .../prebuilt_rules/upgrade_prebuilt_rules.ts | 15 ++----------- 6 files changed, 13 insertions(+), 61 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts index c7b16bc3fc17..7133f221070d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; import type { Client } from '@elastic/elasticsearch'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../es_indices'; /** * (LEGACY) @@ -23,8 +23,7 @@ export const getPrebuiltRulesAndTimelinesStatus = async ( es: Client, supertest: SuperTest.SuperTest ): Promise => { - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); const response = await supertest .get(PREBUILT_RULES_STATUS_URL) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts index 45cbc3e17873..416be5f441c5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; import type { Client } from '@elastic/elasticsearch'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../es_indices'; /** * Helper to retrieve the prebuilt rules status @@ -22,8 +22,7 @@ export const getPrebuiltRulesStatus = async ( es: Client, supertest: SuperTest.SuperTest ): Promise => { - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); const response = await supertest .get(GET_PREBUILT_RULES_STATUS_URL) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 03f974445983..0da50e16925d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -6,11 +6,10 @@ */ import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; import { RetryService } from '@kbn/ftr-common-functional-services'; -import { retry } from '../..'; +import { refreshSavedObjectIndices, retry } from '../..'; /** * Installs latest available non-prerelease prebuilt rules package `security_detection_engine`. @@ -44,21 +43,7 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( retries: 2, }); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the Fleet package with prebuilt detection rules. - // Prebuilt rules are assets that Fleet indexes as saved objects of a certain type. - // Fleet does this via a savedObjectsClient.import() call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule assets will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // Now, the time left until the next refresh can be anything from 0 to the default value, and - // it depends on the time when savedObjectsClient.import() call happens relative to the time of - // the next refresh. Also, probably the refresh time can be delayed when ES is under load? - // Anyway, this can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return fleetResponse as InstallPackageResponse; }; @@ -96,7 +81,7 @@ export const installPrebuiltRulesPackageByVersion = async ( retries: 2, }); - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return fleetResponse as InstallPackageResponse; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts index 308fef271e98..ed3d3ef6f0af 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts @@ -12,7 +12,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../es_indices'; /** * Installs available prebuilt rules in Kibana. Rules are @@ -47,17 +47,7 @@ export const installPrebuiltRules = async ( .send(payload) .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race conditions between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts index 776af6074e07..35f711fe043d 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../es_indices'; /** * (LEGACY) @@ -40,17 +40,7 @@ export const installPrebuiltRulesAndTimelines = async ( .send() .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts index caadba2619a7..abcc91fb3ae0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts @@ -12,7 +12,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../es_indices'; /** * Upgrades available prebuilt rules in Kibana. @@ -43,18 +43,7 @@ export const upgradePrebuiltRules = async ( .send(payload) .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we upgraded the prebuilt rules, which, under the hoods, installs new versions - // of the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race conditions between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; From 0cea7f7a656b33256f8ee06afee428fc6f17fc6d Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 01:00:25 +0100 Subject: [PATCH 07/18] Changes --- .../services/retry/retry.ts | 7 ++++- .../services/retry/retry_for_success.ts | 11 ++++++-- .../basic/tests/coverage_overview.ts | 3 ++- .../utils/es_indices.ts | 26 +++++++++++++++++++ .../utils/index.ts | 1 + .../install_prebuilt_rules_and_timelines.ts | 14 ++-------- 6 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/utils/es_indices.ts diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts index 231a829225db..6162c3a33432 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts @@ -27,12 +27,17 @@ export class RetryService extends FtrService { }); } - public async try(block: () => Promise, onFailureBlock?: () => Promise) { + public async try( + block: () => Promise, + onFailureBlock?: () => Promise, + retryDelay?: number + ) { return await retryForSuccess(this.log, { timeout: this.config.get('timeouts.try'), methodName: 'retry.try', block, onFailureBlock, + retryDelay, }); } diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts index 088f6fedc85f..f59d76028b9f 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts @@ -44,14 +44,21 @@ interface Options { onFailureBlock?: () => Promise; onFailure?: ReturnType; accept?: (v: T) => boolean; + retryDelay?: number; } export async function retryForSuccess(log: ToolingLog, options: Options) { - const { timeout, methodName, block, onFailureBlock, accept = returnTrue } = options; + const { + timeout, + methodName, + block, + onFailureBlock, + accept = returnTrue, + retryDelay = 502, + } = options; const { onFailure = defaultOnFailure(methodName) } = options; const start = Date.now(); - const retryDelay = 502; const criticalWebDriverErrors = ['NoSuchSessionError', 'NoSuchWindowError']; let lastError; diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts b/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts index b40033b5eb17..7a550efc5b51 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts @@ -27,6 +27,7 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); const es = getService('es'); + const retry = getService('retry'); describe('coverage_overview', () => { beforeEach(async () => { @@ -438,7 +439,7 @@ export default ({ getService }: FtrProviderContext): void => { threat: generateThreatArray(1), }), ]); - await installPrebuiltRulesAndTimelines(es, supertest); + await installPrebuiltRulesAndTimelines(es, supertest, retry); const expectedRule = await createRule(supertest, log, { ...getSimpleRule('rule-1'), diff --git a/x-pack/test/detection_engine_api_integration/utils/es_indices.ts b/x-pack/test/detection_engine_api_integration/utils/es_indices.ts new file mode 100644 index 000000000000..8ee99be7e09e --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/es_indices.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import type { Client } from '@elastic/elasticsearch'; + +export const refreshSavedObjectIndices = async (es: Client) => { + // In cases such as when installing the prebuilt detection rules SO of type 'security-rule', + // The savedObjectsClient does this with a call with explicit `refresh: false`. + // So, despite of the fact that the endpoint waits until the prebuilt rule will be + // successfully indexed, it doesn't wait until they become "visible" for subsequent read + // operations. + // And this is usually what we do next in integration tests: we read these SOs with utility + // function such as getPrebuiltRulesAndTimelinesStatus(). + // This can cause race condition between a write and subsequent read operation, and to + // fix it deterministically we have to refresh saved object indices and wait until it's done. + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + + // Additionally, we need to clear the cache to ensure that the next read operation will + // not return stale data. + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); +}; diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index 1938a069a2f5..7adbab2aa129 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -14,6 +14,7 @@ export * from './create_signals_index'; export * from './delete_all_rules'; export * from './delete_all_alerts'; export * from './delete_all_timelines'; +export * from './es_indices'; export * from './get_complex_rule'; export * from './get_complex_rule_output'; export * from './get_simple_rule'; diff --git a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts index 776af6074e07..35f711fe043d 100644 --- a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts +++ b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../es_indices'; /** * (LEGACY) @@ -40,17 +40,7 @@ export const installPrebuiltRulesAndTimelines = async ( .send() .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; From d09d338b6b13021bcb721cfded535fe683ff8986 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 3 Jan 2024 00:11:41 +0000 Subject: [PATCH 08/18] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test/security_solution_api_integration/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index 77b7f288a5b0..18e019202355 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -40,5 +40,6 @@ "@kbn/datemath", "@kbn/safer-lodash-set", "@kbn/stack-connectors-plugin", + "@kbn/ftr-common-functional-services", ] } From 6154f3050f7406e612a690a80778e64c8d5b2156 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 11:40:37 +0100 Subject: [PATCH 09/18] Add missing expect --- .../rules/prebuilt_rules/install_fleet_package_by_url.ts | 1 + .../prebuilt_rules/install_prebuilt_rules_fleet_package.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 0da50e16925d..5eb1c01babad 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -9,6 +9,7 @@ import type SuperTest from 'supertest'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; import { RetryService } from '@kbn/ftr-common-functional-services'; +import expect from 'expect'; import { refreshSavedObjectIndices, retry } from '../..'; /** diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index c59978993742..bfc807a85ba2 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -14,6 +14,7 @@ import type { Client } from '@elastic/elasticsearch'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; import { RetryService } from '@kbn/ftr-common-functional-services'; +import expect from 'expect'; import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../..'; @@ -55,7 +56,7 @@ export const installPrebuiltRulesFleetPackage = async ({ return testResponse.body; }, retryService, - retries: 1, + retries: 2, }); await refreshSavedObjectIndices(es); @@ -86,7 +87,7 @@ export const installPrebuiltRulesFleetPackage = async ({ return body; }, retryService, - retries: 1, + retries: 2, }); await refreshSavedObjectIndices(es); From dfd90017d557109d32cadb117c35e05f1de2ec7c Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 13:01:16 +0100 Subject: [PATCH 10/18] Fix wrong util call --- .../basic/tests/coverage_overview.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts b/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts index 7a550efc5b51..b40033b5eb17 100644 --- a/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts +++ b/x-pack/test/detection_engine_api_integration/basic/tests/coverage_overview.ts @@ -27,7 +27,6 @@ export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); const es = getService('es'); - const retry = getService('retry'); describe('coverage_overview', () => { beforeEach(async () => { @@ -439,7 +438,7 @@ export default ({ getService }: FtrProviderContext): void => { threat: generateThreatArray(1), }), ]); - await installPrebuiltRulesAndTimelines(es, supertest, retry); + await installPrebuiltRulesAndTimelines(es, supertest); const expectedRule = await createRule(supertest, log, { ...getSimpleRule('rule-1'), From 1c1ebd85e8e7a9f1b9e7fb6ed84104b35223bd8f Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 3 Jan 2024 16:18:49 +0100 Subject: [PATCH 11/18] Changes --- .../install_latest_bundled_prebuilt_rules.ts | 1 - .../default_license/prebuilt_rules/management/index.ts | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index 54424b39ecd9..d2908f148690 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -24,7 +24,6 @@ export default ({ getService }: FtrProviderContext): void => { const log = getService('log'); const retry = getService('retry'); - // FLAKY: https://github.com/elastic/kibana/issues/171380 /* This test simulates an air-gapped environment in which the user doesn't have access to EPR. /* We first download the package from the registry as done during build time, and then /* attempt to install it from the local file system. The API response from EPM provides diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts index ed87c0e3b4dd..52a745e0e797 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/index.ts @@ -9,9 +9,9 @@ import { FtrProviderContext } from '../../../../../ftr_provider_context'; export default ({ loadTestFile }: FtrProviderContext): void => { describe('Detection Engine API - Prebuilt Rules Management', function () { - // loadTestFile(require.resolve('./get_prebuilt_rules_status')); - // loadTestFile(require.resolve('./get_prebuilt_timelines_status')); - // loadTestFile(require.resolve('./install_and_upgrade_prebuilt_rules')); + loadTestFile(require.resolve('./get_prebuilt_rules_status')); + loadTestFile(require.resolve('./get_prebuilt_timelines_status')); + loadTestFile(require.resolve('./install_and_upgrade_prebuilt_rules')); loadTestFile(require.resolve('./fleet_integration')); }); }; From 7202291469f40c967f7f9e1acb7c383a18493cef Mon Sep 17 00:00:00 2001 From: jpdjere Date: Thu, 4 Jan 2024 15:26:08 +0100 Subject: [PATCH 12/18] Added configurable timeput --- .../services/retry/retry.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts index 6162c3a33432..b0ea9ab97fba 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts @@ -30,10 +30,11 @@ export class RetryService extends FtrService { public async try( block: () => Promise, onFailureBlock?: () => Promise, - retryDelay?: number + retryDelay?: number, + timeout?: number ) { return await retryForSuccess(this.log, { - timeout: this.config.get('timeouts.try'), + timeout: timeout ?? this.config.get('timeouts.try'), methodName: 'retry.try', block, onFailureBlock, From 1a9782a1f3e444bb997c0a90749811c2912e7e96 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Thu, 4 Jan 2024 15:28:33 +0100 Subject: [PATCH 13/18] Pass timeout --- .../test_suites/detections_response/utils/retry.ts | 5 ++++- .../rules/prebuilt_rules/install_fleet_package_by_url.ts | 2 ++ .../prebuilt_rules/install_prebuilt_rules_fleet_package.ts | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts index 6362149daf4a..b545519b7df3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -14,10 +14,12 @@ export const retry = async ({ test, retryService, retries, + timeout, }: { test: () => Promise; retryService: RetryService; retries: number; + timeout?: number; }): Promise => { let retryAttempt = 0; const response = await retryService.try( @@ -33,7 +35,8 @@ export const retry = async ({ return test(); }, undefined, - 200 + 200, + timeout ); // Now throw the error in order to fail the test. diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 5eb1c01babad..6c28869799de 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -42,6 +42,7 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( }, retryService, retries: 2, + timeout: 120000, }); await refreshSavedObjectIndices(es); @@ -80,6 +81,7 @@ export const installPrebuiltRulesPackageByVersion = async ( }, retryService, retries: 2, + timeout: 120000, }); await refreshSavedObjectIndices(es); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index bfc807a85ba2..d9b6901f2728 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -57,6 +57,7 @@ export const installPrebuiltRulesFleetPackage = async ({ }, retryService, retries: 2, + timeout: 120000, }); await refreshSavedObjectIndices(es); @@ -88,6 +89,7 @@ export const installPrebuiltRulesFleetPackage = async ({ }, retryService, retries: 2, + timeout: 120000, }); await refreshSavedObjectIndices(es); From 0c33f0ee77ccc7736b5d9781c227939ca0b7d02f Mon Sep 17 00:00:00 2001 From: jpdjere Date: Thu, 4 Jan 2024 18:33:18 +0100 Subject: [PATCH 14/18] Fix --- .../services/retry/retry.ts | 9 +++++---- .../test_suites/detections_response/utils/retry.ts | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts index b0ea9ab97fba..383e37511ef8 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts @@ -17,24 +17,25 @@ export class RetryService extends FtrService { public async tryForTime( timeout: number, block: () => Promise, - onFailureBlock?: () => Promise + onFailureBlock?: () => Promise, + retryDelay?: number ) { return await retryForSuccess(this.log, { timeout, methodName: 'retry.tryForTime', block, onFailureBlock, + retryDelay, }); } public async try( block: () => Promise, onFailureBlock?: () => Promise, - retryDelay?: number, - timeout?: number + retryDelay?: number ) { return await retryForSuccess(this.log, { - timeout: timeout ?? this.config.get('timeouts.try'), + timeout: this.config.get('timeouts.try'), methodName: 'retry.try', block, onFailureBlock, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts index b545519b7df3..cbe31ead0034 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -19,10 +19,11 @@ export const retry = async ({ test: () => Promise; retryService: RetryService; retries: number; - timeout?: number; + timeout: number; }): Promise => { let retryAttempt = 0; - const response = await retryService.try( + const response = await retryService.tryForTime( + timeout, async () => { if (retryAttempt > retries) { // Log error message if we reached the maximum number of retries @@ -35,8 +36,7 @@ export const retry = async ({ return test(); }, undefined, - 200, - timeout + 200 ); // Now throw the error in order to fail the test. From 02d486e3f4380c58ecb593db48d3886e5d694da2 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Mon, 8 Jan 2024 22:25:06 +0100 Subject: [PATCH 15/18] Added docs to retry function --- .../utils/es_indices.ts | 26 ---------- .../utils/index.ts | 2 - .../install_prebuilt_rules_and_timelines.ts | 46 ----------------- .../utils/refresh_index.ts | 26 ++++++++++ .../detections_response/utils/retry.ts | 49 ++++++++++++++++--- .../utils/rules/es_indices.ts | 26 ---------- ...get_prebuilt_rules_and_timelines_status.ts | 2 +- .../get_prebuilt_rules_status.ts | 2 +- .../install_fleet_package_by_url.ts | 16 +++--- .../prebuilt_rules/install_prebuilt_rules.ts | 2 +- .../install_prebuilt_rules_and_timelines.ts | 2 +- .../install_prebuilt_rules_fleet_package.ts | 17 ++++--- .../prebuilt_rules/upgrade_prebuilt_rules.ts | 2 +- 13 files changed, 92 insertions(+), 126 deletions(-) delete mode 100644 x-pack/test/detection_engine_api_integration/utils/es_indices.ts delete mode 100644 x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts delete mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts diff --git a/x-pack/test/detection_engine_api_integration/utils/es_indices.ts b/x-pack/test/detection_engine_api_integration/utils/es_indices.ts deleted file mode 100644 index 8ee99be7e09e..000000000000 --- a/x-pack/test/detection_engine_api_integration/utils/es_indices.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; -import type { Client } from '@elastic/elasticsearch'; - -export const refreshSavedObjectIndices = async (es: Client) => { - // In cases such as when installing the prebuilt detection rules SO of type 'security-rule', - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - - // Additionally, we need to clear the cache to ensure that the next read operation will - // not return stale data. - await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); -}; diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index 7adbab2aa129..f01b3e09e161 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -14,7 +14,6 @@ export * from './create_signals_index'; export * from './delete_all_rules'; export * from './delete_all_alerts'; export * from './delete_all_timelines'; -export * from './es_indices'; export * from './get_complex_rule'; export * from './get_complex_rule_output'; export * from './get_simple_rule'; @@ -29,7 +28,6 @@ export * from './update_rule'; export * from './wait_for'; export * from './wait_for_rule_status'; export * from './prebuilt_rules/create_prebuilt_rule_saved_objects'; -export * from './prebuilt_rules/install_prebuilt_rules_and_timelines'; export * from './get_simple_rule_update'; export * from './get_simple_ml_rule_update'; export * from './create_non_security_rule'; diff --git a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts deleted file mode 100644 index 35f711fe043d..000000000000 --- a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - InstallPrebuiltRulesAndTimelinesResponse, - PREBUILT_RULES_URL, -} from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; -import type { Client } from '@elastic/elasticsearch'; -import type SuperTest from 'supertest'; -import { refreshSavedObjectIndices } from '../es_indices'; - -/** - * (LEGACY) - * Installs all prebuilt rules and timelines available in Kibana. Rules are - * installed from the security-rule saved objects. - * This is a legacy endpoint and has been replaced by: - * POST /internal/detection_engine/prebuilt_rules/installation/_perform - * - * - No rules will be installed if there are no security-rule assets (e.g., the - * package is not installed or mocks are not created). - * - * - If some prebuilt rules are already installed, they will be upgraded in case - * there are newer versions of them in security-rule assets. - * - * @param supertest SuperTest instance - * @returns Install prebuilt rules response - */ -export const installPrebuiltRulesAndTimelines = async ( - es: Client, - supertest: SuperTest.SuperTest -): Promise => { - const response = await supertest - .put(PREBUILT_RULES_URL) - .set('kbn-xsrf', 'true') - .set('elastic-api-version', '2023-10-31') - .send() - .expect(200); - - await refreshSavedObjectIndices(es); - - return response.body; -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts index f888216cb6ee..16439adb00e3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts @@ -6,6 +6,7 @@ */ import type { Client } from '@elastic/elasticsearch'; +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; /** * Refresh an index, making changes available to search. @@ -17,3 +18,28 @@ export const refreshIndex = async (es: Client, index?: string) => { index, }); }; + +/** + * Refresh an index, making changes available to search. + * Reusable utility which refreshes all saved object indices, to make them available for search, especially + * useful when needing to perform a search on an index that has just been written to. + * + * An example of this when installing the prebuilt detection rules SO of type 'security-rule': + * the savedObjectsClient does this with a call with explicit `refresh: false`. + * So, despite of the fact that the endpoint waits until the prebuilt rule will be + * successfully indexed, it doesn't wait until they become "visible" for subsequent read + * operations. + * + * Additionally, this method clears the cache for all saved object indices. This helps in cases in which + * saved object is read, then written to, and then read again, and the second read returns stale data. + * @param es The Elasticsearch client + */ +export const refreshSavedObjectIndices = async (es: Client) => { + // Refresh indices to prevent a race condition between a write and subsequent read operation. To + // fix it deterministically we have to refresh saved object indices and wait until it's done. + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + + // Additionally, we need to clear the cache to ensure that the next read operation will + // not return stale data. + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts index cbe31ead0034..dafd16aaa9f5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -7,20 +7,53 @@ import { RetryService } from '@kbn/ftr-common-functional-services'; -/* - * Retry wrapper for async supertests, with a maximum number of retries +/** + * Retry wrapper for async supertests, with a maximum number of retries. + * You can pass in a function that executes a supertest test, and make assertions + * on the response. If the test fails, it will retry the test the number of retries + * that are passed in. + * + * Example usage: + * ```ts + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(`/api/fleet/epm/packages/security_detection_engine`) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); + * ``` + * @param test The function containing a test to run + * @param retryService The retry service to use + * @param retries The maximum number of retries + * @param timeout The timeout for each retry + * @param retryDelay The delay between each retry + * @returns The response from the test */ export const retry = async ({ test, retryService, - retries, - timeout, + retries = 2, + timeout = 30000, + retryDelay = 200, }: { test: () => Promise; retryService: RetryService; - retries: number; - timeout: number; -}): Promise => { + retries?: number; + timeout?: number; + retryDelay?: number; +}): Promise => { let retryAttempt = 0; const response = await retryService.tryForTime( timeout, @@ -36,7 +69,7 @@ export const retry = async ({ return test(); }, undefined, - 200 + retryDelay ); // Now throw the error in order to fail the test. diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts deleted file mode 100644 index 8ee99be7e09e..000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/es_indices.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; -import type { Client } from '@elastic/elasticsearch'; - -export const refreshSavedObjectIndices = async (es: Client) => { - // In cases such as when installing the prebuilt detection rules SO of type 'security-rule', - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); - - // Additionally, we need to clear the cache to ensure that the next read operation will - // not return stale data. - await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts index 7133f221070d..7f683ca9994b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; import type { Client } from '@elastic/elasticsearch'; -import { refreshSavedObjectIndices } from '../es_indices'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * (LEGACY) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts index 416be5f441c5..da044637fc77 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; import type { Client } from '@elastic/elasticsearch'; -import { refreshSavedObjectIndices } from '../es_indices'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * Helper to retrieve the prebuilt rules status diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 6c28869799de..988d73660d0e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -10,7 +10,11 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; import { RetryService } from '@kbn/ftr-common-functional-services'; import expect from 'expect'; -import { refreshSavedObjectIndices, retry } from '../..'; +import { retry } from '../../retry'; +import { refreshSavedObjectIndices } from '../../refresh_index'; + +const MAX_RETRIES = 2; +const ATTEMPT_TIMEOUT = 120000; /** * Installs latest available non-prerelease prebuilt rules package `security_detection_engine`. @@ -41,13 +45,13 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( return testResponse.body; }, retryService, - retries: 2, - timeout: 120000, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, }); await refreshSavedObjectIndices(es); - return fleetResponse as InstallPackageResponse; + return fleetResponse; }; /** * Installs prebuilt rules package `security_detection_engine`, passing in the version @@ -80,8 +84,8 @@ export const installPrebuiltRulesPackageByVersion = async ( return testResponse.body; }, retryService, - retries: 2, - timeout: 120000, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, }); await refreshSavedObjectIndices(es); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts index ed3d3ef6f0af..499f97877bf1 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts @@ -12,7 +12,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { refreshSavedObjectIndices } from '../es_indices'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * Installs available prebuilt rules in Kibana. Rules are diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts index 35f711fe043d..c83e8693f239 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { refreshSavedObjectIndices } from '../es_indices'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * (LEGACY) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index d9b6901f2728..592406e8c339 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -16,7 +16,10 @@ import type SuperTest from 'supertest'; import { RetryService } from '@kbn/ftr-common-functional-services'; import expect from 'expect'; import { retry } from '../../retry'; -import { refreshSavedObjectIndices } from '../..'; +import { refreshSavedObjectIndices } from '../../refresh_index'; + +const MAX_RETRIES = 2; +const ATTEMPT_TIMEOUT = 120000; /** * Installs the `security_detection_engine` package via fleet API. This will @@ -56,13 +59,13 @@ export const installPrebuiltRulesFleetPackage = async ({ return testResponse.body; }, retryService, - retries: 2, - timeout: 120000, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, }); await refreshSavedObjectIndices(es); - return response as InstallPackageResponse; + return response; } else { // Install the latest version const response = await retry({ @@ -88,13 +91,13 @@ export const installPrebuiltRulesFleetPackage = async ({ return body; }, retryService, - retries: 2, - timeout: 120000, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, }); await refreshSavedObjectIndices(es); - return response as BulkInstallPackagesResponse; + return response; } }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts index abcc91fb3ae0..c22aa9106a27 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts @@ -12,7 +12,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { refreshSavedObjectIndices } from '../es_indices'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * Upgrades available prebuilt rules in Kibana. From 51348ddb87b21f0bae413f9641061d3dcd7d8ce1 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 9 Jan 2024 00:15:43 +0100 Subject: [PATCH 16/18] Undo deletion --- .../install_prebuilt_rules_and_timelines.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts diff --git a/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts new file mode 100644 index 000000000000..776af6074e07 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/utils/prebuilt_rules/install_prebuilt_rules_and_timelines.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + InstallPrebuiltRulesAndTimelinesResponse, + PREBUILT_RULES_URL, +} from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; +import type { Client } from '@elastic/elasticsearch'; +import type SuperTest from 'supertest'; +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; + +/** + * (LEGACY) + * Installs all prebuilt rules and timelines available in Kibana. Rules are + * installed from the security-rule saved objects. + * This is a legacy endpoint and has been replaced by: + * POST /internal/detection_engine/prebuilt_rules/installation/_perform + * + * - No rules will be installed if there are no security-rule assets (e.g., the + * package is not installed or mocks are not created). + * + * - If some prebuilt rules are already installed, they will be upgraded in case + * there are newer versions of them in security-rule assets. + * + * @param supertest SuperTest instance + * @returns Install prebuilt rules response + */ +export const installPrebuiltRulesAndTimelines = async ( + es: Client, + supertest: SuperTest.SuperTest +): Promise => { + const response = await supertest + .put(PREBUILT_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send() + .expect(200); + + // Before we proceed, we need to refresh saved object indices. + // At the previous step we installed the prebuilt detection rules SO of type 'security-rule'. + // The savedObjectsClient does this with a call with explicit `refresh: false`. + // So, despite of the fact that the endpoint waits until the prebuilt rule will be + // successfully indexed, it doesn't wait until they become "visible" for subsequent read + // operations. + // And this is usually what we do next in integration tests: we read these SOs with utility + // function such as getPrebuiltRulesAndTimelinesStatus(). + // This can cause race condition between a write and subsequent read operation, and to + // fix it deterministically we have to refresh saved object indices and wait until it's done. + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + + return response.body; +}; From e50b062bb26bd46dcb0db92de0f35764d60878a1 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 9 Jan 2024 00:16:32 +0100 Subject: [PATCH 17/18] Index --- x-pack/test/detection_engine_api_integration/utils/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/detection_engine_api_integration/utils/index.ts b/x-pack/test/detection_engine_api_integration/utils/index.ts index f01b3e09e161..1938a069a2f5 100644 --- a/x-pack/test/detection_engine_api_integration/utils/index.ts +++ b/x-pack/test/detection_engine_api_integration/utils/index.ts @@ -28,6 +28,7 @@ export * from './update_rule'; export * from './wait_for'; export * from './wait_for_rule_status'; export * from './prebuilt_rules/create_prebuilt_rule_saved_objects'; +export * from './prebuilt_rules/install_prebuilt_rules_and_timelines'; export * from './get_simple_rule_update'; export * from './get_simple_ml_rule_update'; export * from './create_non_security_rule'; From f2d95fffbfac2950c6aafb6e46c2610ffd0ff8c0 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 9 Jan 2024 09:02:13 +0100 Subject: [PATCH 18/18] Index --- .../test_suites/detections_response/utils/rules/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts index db0695546dcd..90f3ae07871c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/index.ts @@ -13,7 +13,6 @@ export * from './create_non_security_rule'; export * from './downgrade_immutable_rule'; export * from './delete_all_rules'; export * from './delete_rule'; -export * from './es_indices'; export * from './fetch_rule'; export * from './find_immutable_rule_by_id'; export * from './get_simple_rule';