diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts index 231a829225db..383e37511ef8 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts @@ -17,22 +17,29 @@ export class RetryService extends FtrService { public async tryForTime( timeout: number, block: () => Promise, - onFailureBlock?: () => Promise + onFailureBlock?: () => Promise, + retryDelay?: number ) { return await retryForSuccess(this.log, { timeout, methodName: 'retry.tryForTime', block, onFailureBlock, + retryDelay, }); } - public async try(block: () => Promise, onFailureBlock?: () => Promise) { + public async try( + block: () => Promise, + onFailureBlock?: () => Promise, + retryDelay?: number + ) { return await retryForSuccess(this.log, { timeout: this.config.get('timeouts.try'), methodName: 'retry.try', block, onFailureBlock, + retryDelay, }); } diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts index 088f6fedc85f..f59d76028b9f 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts @@ -44,14 +44,21 @@ interface Options { onFailureBlock?: () => Promise; onFailure?: ReturnType; accept?: (v: T) => boolean; + retryDelay?: number; } export async function retryForSuccess(log: ToolingLog, options: Options) { - const { timeout, methodName, block, onFailureBlock, accept = returnTrue } = options; + const { + timeout, + methodName, + block, + onFailureBlock, + accept = returnTrue, + retryDelay = 502, + } = options; const { onFailure = defaultOnFailure(methodName) } = options; const start = Date.now(); - const retryDelay = 502; const criticalWebDriverErrors = ['NoSuchSessionError', 'NoSuchWindowError']; let lastError; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts index 333e76550a74..3b8689b47fcb 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts @@ -218,7 +218,7 @@ export default ({ getService }: FtrProviderContext) => { ); await updateRule(supertest, ruleToUpdate); - const status = await getPrebuiltRulesAndTimelinesStatus(supertest); + const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(status.rules_not_installed).toBe(0); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts index 870df90d3e47..a0b7145dbc95 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts @@ -526,7 +526,7 @@ export default ({ getService }: FtrProviderContext) => { }) .expect(200); - const status = await getPrebuiltRulesAndTimelinesStatus(supertest); + const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(status.rules_not_installed).toEqual(0); }); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index 60e0399df53f..d2908f148690 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -22,14 +22,14 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); - // FLAKY: https://github.com/elastic/kibana/issues/171380 /* This test simulates an air-gapped environment in which the user doesn't have access to EPR. /* We first download the package from the registry as done during build time, and then /* attempt to install it from the local file system. The API response from EPM provides /* us with the information of whether the package was installed from the registry or /* from a package that was bundled with Kibana */ - describe.skip('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => { + describe('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => { beforeEach(async () => { await deleteAllRules(supertest, log); await deleteAllPrebuiltRuleAssets(es); @@ -52,7 +52,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should install prebuilt rules from the package that comes bundled with Kibana', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -60,7 +60,8 @@ export default ({ getService }: FtrProviderContext): void => { const bundledInstallResponse = await installPrebuiltRulesPackageByVersion( es, supertest, - '99.0.0' + '99.0.0', + retry ); // As opposed to "registry" @@ -71,7 +72,7 @@ export default ({ getService }: FtrProviderContext): void => { await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBeGreaterThan(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts index 448e325892a5..f9450929cf53 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts @@ -22,6 +22,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); /* This test makes use of the mock packages created in the '/fleet_bundled_packages' folder, /* in order to assert that, in production environments, the latest stable version of the package @@ -39,7 +40,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should install latest stable version and ignore prerelease packages', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -47,7 +48,8 @@ export default ({ getService }: FtrProviderContext): void => { // Install package without specifying version to check if latest stable version is installed const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI( es, - supertest + supertest, + retry ); expect(fleetPackageInstallationResponse.items.length).toBe(1); @@ -59,7 +61,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(prebuiltRulesFleetPackage.status).toBe(200); // Get status of our prebuilt rules (nothing should be instaled yet) - const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBe(1); // 1 rule in package 99.0.0 expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -68,7 +70,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Verify that status is updated after package installation - const statusAfterRulesInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterRulesInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_installed).toBe(1); // 1 rule in package 99.0.0 expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts index e059cab5ae64..20f0e214c105 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts @@ -31,7 +31,10 @@ export default ({ getService }: FtrProviderContext): void => { it('should install a package containing 15000 prebuilt rules without crashing', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusBeforePackageInstallation.rules_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_updated).toBe(0); @@ -40,7 +43,10 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusAfterPackageInstallation.rules_installed).toBe(750); expect(statusAfterPackageInstallation.rules_not_installed).toBe(0); expect(statusAfterPackageInstallation.rules_not_updated).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index 5453ff5b34c7..b8f29010d5a0 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -20,6 +20,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); describe('@ess @serverless @skipInQA install_prebuilt_rules_from_real_package', () => { beforeEach(async () => { @@ -33,10 +34,12 @@ export default ({ getService }: FtrProviderContext): void => { * Unlike other tests that use mocks, this test uses actual rules from the * package storage and checks that they are installed. */ - // TODO: Fix and unskip https://github.com/elastic/kibana/issues/172107 - it.skip('should install prebuilt rules from the package storage', async () => { + it('should install prebuilt rules from the package storage', async () => { // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusBeforePackageInstallation.rules_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_installed).toBe(0); expect(statusBeforePackageInstallation.rules_not_updated).toBe(0); @@ -45,10 +48,14 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, overrideExistingPackage: true, + retryService: retry, }); // Verify that status is updated after package installation - const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus( + es, + supertest + ); expect(statusAfterPackageInstallation.rules_installed).toBe(0); expect(statusAfterPackageInstallation.rules_not_installed).toBeGreaterThan(0); expect(statusAfterPackageInstallation.rules_not_updated).toBe(0); @@ -59,7 +66,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.rules_updated).toBe(0); // Verify that status is updated after rules installation - const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusAfterRuleInstallation.rules_installed).toBe(response.rules_installed); expect(statusAfterRuleInstallation.rules_not_installed).toBe(0); expect(statusAfterRuleInstallation.rules_not_updated).toBe(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts index 16dba2761694..bfbe4f00cb32 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_rules_status.ts @@ -36,7 +36,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should return empty structure when no prebuilt rule assets', async () => { - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: 0, @@ -48,7 +48,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should not update the prebuilt rule status when a custom rule is added', async () => { await createRule(supertest, log, getSimpleRule()); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: 0, @@ -68,7 +68,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: RULES_COUNT, @@ -81,7 +81,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -95,7 +95,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); await deleteRule(supertest, 'rule-1'); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT - 1, num_prebuilt_rules_to_install: 1, @@ -115,7 +115,7 @@ export default ({ getService }: FtrProviderContext): void => { ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -137,7 +137,7 @@ export default ({ getService }: FtrProviderContext): void => { // Upgrade all rules await upgradePrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -156,7 +156,7 @@ export default ({ getService }: FtrProviderContext): void => { // Recreate the rules without bumping any versions await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -179,7 +179,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: 0, num_prebuilt_rules_to_install: RULES_COUNT, @@ -192,7 +192,7 @@ export default ({ getService }: FtrProviderContext): void => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -206,7 +206,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); await deleteRule(supertest, 'rule-1'); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT - 1, num_prebuilt_rules_to_install: 1, @@ -224,7 +224,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -245,7 +245,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -271,7 +271,7 @@ export default ({ getService }: FtrProviderContext): void => { // Upgrade the rule await upgradePrebuiltRules(es, supertest); - const { stats } = await getPrebuiltRulesStatus(supertest); + const { stats } = await getPrebuiltRulesStatus(es, supertest); expect(stats).toMatchObject({ num_prebuilt_rules_installed: RULES_COUNT, num_prebuilt_rules_to_install: 0, @@ -291,7 +291,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should return empty structure when no rules package installed', async () => { - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, @@ -304,7 +304,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should show that one custom rule is installed when a custom rule is added', async () => { await createRule(supertest, log, getSimpleRule()); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 1, rules_installed: 0, @@ -324,7 +324,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, @@ -338,7 +338,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRulesAndTimelines(es, supertest); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -352,7 +352,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); await deleteRule(supertest, 'rule-1'); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT - 1, @@ -372,7 +372,7 @@ export default ({ getService }: FtrProviderContext): void => { ruleAssetSavedObjects[0]['security-rule'].version += 1; await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -391,7 +391,7 @@ export default ({ getService }: FtrProviderContext): void => { // Recreate the rules without bumping any versions await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -413,7 +413,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of rules available to install', async () => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, @@ -427,7 +427,7 @@ export default ({ getService }: FtrProviderContext): void => { await createHistoricalPrebuiltRuleAssetSavedObjects(es, getRuleAssetSavedObjects()); await installPrebuiltRulesAndTimelines(es, supertest); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -441,7 +441,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); await deleteRule(supertest, 'rule-1'); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT - 1, @@ -459,7 +459,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, @@ -480,7 +480,7 @@ export default ({ getService }: FtrProviderContext): void => { createRuleAssetSavedObject({ rule_id: 'rule-1', version: 3 }), ]); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ rules_custom_installed: 0, rules_installed: RULES_COUNT, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts index 9acef16bfbeb..cf7a2c961e06 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/get_prebuilt_timelines_status.ts @@ -23,7 +23,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should return the number of timeline templates available to install', async () => { - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ timelines_installed: 0, @@ -36,7 +36,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return the number of installed timeline templates after installing them', async () => { await installPrebuiltRulesAndTimelines(es, supertest); - const body = await getPrebuiltRulesAndTimelinesStatus(supertest); + const body = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(body).toMatchObject({ timelines_installed: expect.any(Number), timelines_not_installed: 0, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts index a75c8f87bd78..4a85fe68fcba 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/install_and_upgrade_prebuilt_rules.ts @@ -80,7 +80,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(1); // Call the install prebuilt rules again and check that the missing rule was installed @@ -102,7 +102,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_updated).toBe(1); // Call the install prebuilt rules again and check that the outdated rule was updated @@ -117,7 +117,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(0); expect(statusResponse.rules_not_updated).toBe(0); @@ -162,7 +162,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(1); // Call the install prebuilt rules again and check that the missing rule was installed @@ -183,7 +183,7 @@ export default ({ getService }: FtrProviderContext): void => { await createPrebuiltRuleAssetSavedObjects(es, ruleAssetSavedObjects); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(1); @@ -199,7 +199,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -258,7 +258,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRulesAndTimelines(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(0); // Call the install prebuilt rules again and check that no rules were installed @@ -276,7 +276,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_installed).toBe(1); // Call the install prebuilt rules endpoint again and check that the missing rule was installed @@ -296,7 +296,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_updated).toBe(1); // Call the install prebuilt rules again and check that the outdated rule was updated @@ -304,7 +304,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.rules_installed).toBe(0); expect(response.rules_updated).toBe(1); - const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(_statusResponse.rules_not_installed).toBe(0); expect(_statusResponse.rules_not_updated).toBe(0); }); @@ -323,7 +323,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that one prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(statusResponse.rules_not_updated).toBe(1); expect(statusResponse.rules_not_installed).toBe(0); @@ -332,7 +332,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.rules_installed).toBe(0); expect(response.rules_updated).toBe(1); - const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(supertest); + const _statusResponse = await getPrebuiltRulesAndTimelinesStatus(es, supertest); expect(_statusResponse.rules_not_updated).toBe(0); expect(_statusResponse.rules_not_installed).toBe(0); }); @@ -366,7 +366,7 @@ export default ({ getService }: FtrProviderContext): void => { await installPrebuiltRules(es, supertest); // Check that all prebuilt rules were installed - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); // Call the install prebuilt rules again and check that no rules were installed @@ -384,7 +384,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteRule(supertest, 'rule-1'); // Check that one prebuilt rule is missing - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(1); // Call the install prebuilt rules endpoint again and check that the missing rule was installed @@ -404,7 +404,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that the prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(1); // Call the upgrade prebuilt rules endpoint and check that the outdated rule was updated @@ -412,7 +412,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.summary.succeeded).toBe(1); expect(response.summary.total).toBe(1); - const status = await getPrebuiltRulesStatus(supertest); + const status = await getPrebuiltRulesStatus(es, supertest); expect(status.stats.num_prebuilt_rules_to_install).toBe(0); expect(status.stats.num_prebuilt_rules_to_upgrade).toBe(0); }); @@ -431,7 +431,7 @@ export default ({ getService }: FtrProviderContext): void => { ]); // Check that the prebuilt rule status shows that one rule is outdated - const statusResponse = await getPrebuiltRulesStatus(supertest); + const statusResponse = await getPrebuiltRulesStatus(es, supertest); expect(statusResponse.stats.num_prebuilt_rules_to_upgrade).toBe(1); expect(statusResponse.stats.num_prebuilt_rules_to_install).toBe(0); @@ -440,7 +440,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(response.summary.succeeded).toBe(1); expect(response.summary.total).toBe(1); - const status = await getPrebuiltRulesStatus(supertest); + const status = await getPrebuiltRulesStatus(es, supertest); expect(status.stats.num_prebuilt_rules_to_install).toBe(0); expect(status.stats.num_prebuilt_rules_to_upgrade).toBe(0); }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts index 981bfd712678..2711f28ce88e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts @@ -29,6 +29,7 @@ export default ({ getService }: FtrProviderContext): void => { const es = getService('es'); const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); let currentVersion: string; let previousVersion: string; @@ -96,7 +97,7 @@ export default ({ getService }: FtrProviderContext): void => { // PART 1: Install prebuilt rules from the previous minor version as the current version // Verify that status is empty before package installation - const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0); expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -105,14 +106,15 @@ export default ({ getService }: FtrProviderContext): void => { const installPreviousPackageResponse = await installPrebuiltRulesPackageByVersion( es, supertest, - previousVersion + previousVersion, + retry ); expect(installPreviousPackageResponse._meta.install_source).toBe('registry'); expect(installPreviousPackageResponse.items.length).toBeGreaterThan(0); // Verify that status is updated after the installation of package "N-1" - const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBeGreaterThan(0); expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0); @@ -157,12 +159,13 @@ export default ({ getService }: FtrProviderContext): void => { const installLatestPackageResponse = await installPrebuiltRulesPackageByVersion( es, supertest, - currentVersion + currentVersion, + retry ); expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0); // Verify status after intallation of the latest package - const statusAfterLatestPackageInstallation = await getPrebuiltRulesStatus(supertest); + const statusAfterLatestPackageInstallation = await getPrebuiltRulesStatus(es, supertest); expect( statusAfterLatestPackageInstallation.stats.num_prebuilt_rules_installed ).toBeGreaterThan(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts index 0b7c66f57cb2..afcdf130392f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts @@ -24,3 +24,4 @@ export * from './wait_for_index_to_populate'; export * from './get_stats'; export * from './get_detection_metrics_from_body'; export * from './get_stats_url'; +export * from './retry'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts index f888216cb6ee..16439adb00e3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts @@ -6,6 +6,7 @@ */ import type { Client } from '@elastic/elasticsearch'; +import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; /** * Refresh an index, making changes available to search. @@ -17,3 +18,28 @@ export const refreshIndex = async (es: Client, index?: string) => { index, }); }; + +/** + * Refresh an index, making changes available to search. + * Reusable utility which refreshes all saved object indices, to make them available for search, especially + * useful when needing to perform a search on an index that has just been written to. + * + * An example of this when installing the prebuilt detection rules SO of type 'security-rule': + * the savedObjectsClient does this with a call with explicit `refresh: false`. + * So, despite of the fact that the endpoint waits until the prebuilt rule will be + * successfully indexed, it doesn't wait until they become "visible" for subsequent read + * operations. + * + * Additionally, this method clears the cache for all saved object indices. This helps in cases in which + * saved object is read, then written to, and then read again, and the second read returns stale data. + * @param es The Elasticsearch client + */ +export const refreshSavedObjectIndices = async (es: Client) => { + // Refresh indices to prevent a race condition between a write and subsequent read operation. To + // fix it deterministically we have to refresh saved object indices and wait until it's done. + await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + + // Additionally, we need to clear the cache to ensure that the next read operation will + // not return stale data. + await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts new file mode 100644 index 000000000000..dafd16aaa9f5 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RetryService } from '@kbn/ftr-common-functional-services'; + +/** + * Retry wrapper for async supertests, with a maximum number of retries. + * You can pass in a function that executes a supertest test, and make assertions + * on the response. If the test fails, it will retry the test the number of retries + * that are passed in. + * + * Example usage: + * ```ts + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(`/api/fleet/epm/packages/security_detection_engine`) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); + * ``` + * @param test The function containing a test to run + * @param retryService The retry service to use + * @param retries The maximum number of retries + * @param timeout The timeout for each retry + * @param retryDelay The delay between each retry + * @returns The response from the test + */ +export const retry = async ({ + test, + retryService, + retries = 2, + timeout = 30000, + retryDelay = 200, +}: { + test: () => Promise; + retryService: RetryService; + retries?: number; + timeout?: number; + retryDelay?: number; +}): Promise => { + let retryAttempt = 0; + const response = await retryService.tryForTime( + timeout, + async () => { + if (retryAttempt > retries) { + // Log error message if we reached the maximum number of retries + // but don't throw an error, return it to break the retry loop. + return new Error('Reached maximum number of retries for test.'); + } + + retryAttempt = retryAttempt + 1; + + return test(); + }, + undefined, + retryDelay + ); + + // Now throw the error in order to fail the test. + if (response instanceof Error) { + throw response; + } + + return response; +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts index 2d03e597dc5a..7f683ca9994b 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts @@ -10,6 +10,8 @@ import { PREBUILT_RULES_STATUS_URL, } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; +import type { Client } from '@elastic/elasticsearch'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * (LEGACY) @@ -18,8 +20,11 @@ import type SuperTest from 'supertest'; * @param supertest The supertest deps */ export const getPrebuiltRulesAndTimelinesStatus = async ( + es: Client, supertest: SuperTest.SuperTest ): Promise => { + await refreshSavedObjectIndices(es); + const response = await supertest .get(PREBUILT_RULES_STATUS_URL) .set('kbn-xsrf', 'true') diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts index 0f785f8a7745..da044637fc77 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts @@ -10,6 +10,8 @@ import { GetPrebuiltRulesStatusResponseBody, } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type SuperTest from 'supertest'; +import type { Client } from '@elastic/elasticsearch'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * Helper to retrieve the prebuilt rules status @@ -17,8 +19,11 @@ import type SuperTest from 'supertest'; * @param supertest The supertest deps */ export const getPrebuiltRulesStatus = async ( + es: Client, supertest: SuperTest.SuperTest ): Promise => { + await refreshSavedObjectIndices(es); + const response = await supertest .get(GET_PREBUILT_RULES_STATUS_URL) .set('kbn-xsrf', 'true') diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 259369346cc8..988d73660d0e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -6,9 +6,15 @@ */ import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; +import { RetryService } from '@kbn/ftr-common-functional-services'; +import expect from 'expect'; +import { retry } from '../../retry'; +import { refreshSavedObjectIndices } from '../../refresh_index'; + +const MAX_RETRIES = 2; +const ATTEMPT_TIMEOUT = 120000; /** * Installs latest available non-prerelease prebuilt rules package `security_detection_engine`. @@ -21,37 +27,35 @@ import { epmRouteService } from '@kbn/fleet-plugin/common'; export const installPrebuiltRulesPackageViaFleetAPI = async ( es: Client, - supertest: SuperTest.SuperTest + supertest: SuperTest.SuperTest, + retryService: RetryService ): Promise => { - const fleetResponse = await supertest - .post(`/api/fleet/epm/packages/security_detection_engine`) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(200); + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(`/api/fleet/epm/packages/security_detection_engine`) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the Fleet package with prebuilt detection rules. - // Prebuilt rules are assets that Fleet indexes as saved objects of a certain type. - // Fleet does this via a savedObjectsClient.import() call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule assets will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // Now, the time left until the next refresh can be anything from 0 to the default value, and - // it depends on the time when savedObjectsClient.import() call happens relative to the time of - // the next refresh. Also, probably the refresh time can be delayed when ES is under load? - // Anyway, this can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + return testResponse.body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); - return fleetResponse.body as InstallPackageResponse; + await refreshSavedObjectIndices(es); + + return fleetResponse; }; /** * Installs prebuilt rules package `security_detection_engine`, passing in the version - * of the package as a parameter to the utl. + * of the package as a parameter to the url. * * @param es Elasticsearch client * @param supertest SuperTest instance @@ -62,17 +66,29 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( export const installPrebuiltRulesPackageByVersion = async ( es: Client, supertest: SuperTest.SuperTest, - version: string + version: string, + retryService: RetryService ): Promise => { - const fleetResponse = await supertest - .post(epmRouteService.getInstallPath('security_detection_engine', version)) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(200); + const fleetResponse = await retry({ + test: async () => { + const testResponse = await supertest + .post(epmRouteService.getInstallPath('security_detection_engine', version)) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); - return fleetResponse.body as InstallPackageResponse; + return fleetResponse as InstallPackageResponse; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts index 308fef271e98..499f97877bf1 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts @@ -12,7 +12,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * Installs available prebuilt rules in Kibana. Rules are @@ -47,17 +47,7 @@ export const installPrebuiltRules = async ( .send(payload) .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race conditions between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts index 776af6074e07..c83e8693f239 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts @@ -11,7 +11,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * (LEGACY) @@ -40,17 +40,7 @@ export const installPrebuiltRulesAndTimelines = async ( .send() .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index cc899ecc1dcc..592406e8c339 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -5,10 +5,21 @@ * 2.0. */ -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; -import { epmRouteService } from '@kbn/fleet-plugin/common'; +import { + BulkInstallPackageInfo, + BulkInstallPackagesResponse, + epmRouteService, +} from '@kbn/fleet-plugin/common'; import type { Client } from '@elastic/elasticsearch'; +import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; +import { RetryService } from '@kbn/ftr-common-functional-services'; +import expect from 'expect'; +import { retry } from '../../retry'; +import { refreshSavedObjectIndices } from '../../refresh_index'; + +const MAX_RETRIES = 2; +const ATTEMPT_TIMEOUT = 120000; /** * Installs the `security_detection_engine` package via fleet API. This will @@ -23,49 +34,71 @@ export const installPrebuiltRulesFleetPackage = async ({ supertest, version, overrideExistingPackage, + retryService, }: { es: Client; supertest: SuperTest.SuperTest; version?: string; overrideExistingPackage: boolean; -}): Promise => { + retryService: RetryService; +}): Promise => { if (version) { // Install a specific version - await supertest - .post(epmRouteService.getInstallPath('security_detection_engine', version)) - .set('kbn-xsrf', 'true') - .send({ - force: overrideExistingPackage, - }) - .expect(200); + const response = await retry({ + test: async () => { + const testResponse = await supertest + .post(epmRouteService.getInstallPath('security_detection_engine', version)) + .set('kbn-xsrf', 'true') + .send({ + force: overrideExistingPackage, + }) + .expect(200); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + + return testResponse.body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); + + await refreshSavedObjectIndices(es); + + return response; } else { // Install the latest version - await supertest - .post(epmRouteService.getBulkInstallPath()) - .query({ prerelease: true }) - .set('kbn-xsrf', 'true') - .send({ - packages: ['security_detection_engine'], - force: overrideExistingPackage, - }) - .expect(200); - } + const response = await retry({ + test: async () => { + const testResponse = await supertest + .post(epmRouteService.getBulkInstallPath()) + .query({ prerelease: true }) + .set('kbn-xsrf', 'true') + .send({ + packages: ['security_detection_engine'], + force: overrideExistingPackage, + }) + .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we installed the Fleet package with prebuilt detection rules. - // Prebuilt rules are assets that Fleet indexes as saved objects of a certain type. - // Fleet does this via a savedObjectsClient.import() call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule assets will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // Now, the time left until the next refresh can be anything from 0 to the default value, and - // it depends on the time when savedObjectsClient.import() call happens relative to the time of - // the next refresh. Also, probably the refresh time can be delayed when ES is under load? - // Anyway, this can cause race condition between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + const body = testResponse.body as BulkInstallPackagesResponse; + + // First and only item in the response should be the security_detection_engine package + expect(body.items[0]).toBeDefined(); + expect((body.items[0] as BulkInstallPackageInfo).result.assets).toBeDefined(); + // Endpoint call should have installed at least 1 security-rule asset + expect((body.items[0] as BulkInstallPackageInfo).result.assets?.length).toBeGreaterThan(0); + + return body; + }, + retryService, + retries: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + }); + + await refreshSavedObjectIndices(es); + + return response; + } }; /** diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts index caadba2619a7..c22aa9106a27 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts @@ -12,7 +12,7 @@ import { } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules'; import type { Client } from '@elastic/elasticsearch'; import type SuperTest from 'supertest'; -import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server'; +import { refreshSavedObjectIndices } from '../../refresh_index'; /** * Upgrades available prebuilt rules in Kibana. @@ -43,18 +43,7 @@ export const upgradePrebuiltRules = async ( .send(payload) .expect(200); - // Before we proceed, we need to refresh saved object indices. - // At the previous step we upgraded the prebuilt rules, which, under the hoods, installs new versions - // of the prebuilt detection rules SO of type 'security-rule'. - // The savedObjectsClient does this with a call with explicit `refresh: false`. - // So, despite of the fact that the endpoint waits until the prebuilt rule will be - // successfully indexed, it doesn't wait until they become "visible" for subsequent read - // operations. - // And this is usually what we do next in integration tests: we read these SOs with utility - // function such as getPrebuiltRulesAndTimelinesStatus(). - // This can cause race conditions between a write and subsequent read operation, and to - // fix it deterministically we have to refresh saved object indices and wait until it's done. - await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES }); + await refreshSavedObjectIndices(es); return response.body; }; diff --git a/x-pack/test/security_solution_api_integration/tsconfig.json b/x-pack/test/security_solution_api_integration/tsconfig.json index 6b1ca69f6aed..35e8e940f47f 100644 --- a/x-pack/test/security_solution_api_integration/tsconfig.json +++ b/x-pack/test/security_solution_api_integration/tsconfig.json @@ -37,5 +37,6 @@ "@kbn/fleet-plugin", "@kbn/repo-info", "@kbn/securitysolution-es-utils", + "@kbn/ftr-common-functional-services", ] }