From ecfd52fbe60dc28caeb1057ae3a72c799fb9165b Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 6 Feb 2024 16:51:23 +0100 Subject: [PATCH 1/4] Add try catch to test --- .../update_prebuilt_rules_package.ts | 3 +- .../detections_response/utils/retry.ts | 10 +++++-- .../install_fleet_package_by_url.ts | 30 ++++++++++++------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts index 11577bec1b5c..d4c0b546fb42 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts @@ -107,7 +107,8 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, previousVersion, - retry + retry, + log ); expect(installPreviousPackageResponse._meta.install_source).toBe('registry'); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts index dafd16aaa9f5..42f83af795ae 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -6,7 +6,7 @@ */ import { RetryService } from '@kbn/ftr-common-functional-services'; - +import type { ToolingLog } from '@kbn/tooling-log'; /** * Retry wrapper for async supertests, with a maximum number of retries. * You can pass in a function that executes a supertest test, and make assertions @@ -47,12 +47,14 @@ export const retry = async ({ retries = 2, timeout = 30000, retryDelay = 200, + log, }: { test: () => Promise; retryService: RetryService; retries?: number; timeout?: number; retryDelay?: number; + log?: ToolingLog; }): Promise => { let retryAttempt = 0; const response = await retryService.tryForTime( @@ -61,7 +63,11 @@ export const retry = async ({ if (retryAttempt > retries) { // Log error message if we reached the maximum number of retries // but don't throw an error, return it to break the retry loop. - return new Error('Reached maximum number of retries for test.'); + const errorMessage = `Reached maximum number of retries for test: ${ + retryAttempt - 1 + }/${retries}`; + log?.error(errorMessage); + return new Error(JSON.stringify({ retries: errorMessage })); } retryAttempt = retryAttempt + 1; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 988d73660d0e..48db5595b10c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -9,6 +9,7 @@ import type SuperTest from 'supertest'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; import { RetryService } from '@kbn/ftr-common-functional-services'; +import type { ToolingLog } from '@kbn/tooling-log'; import expect from 'expect'; import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../../refresh_index'; @@ -67,25 +68,32 @@ export const installPrebuiltRulesPackageByVersion = async ( es: Client, supertest: SuperTest.SuperTest, version: string, - retryService: RetryService + retryService: RetryService, + log: ToolingLog ): Promise => { const fleetResponse = await retry({ test: async () => { - const testResponse = await supertest - .post(epmRouteService.getInstallPath('security_detection_engine', version)) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(200); - expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); - expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + try { + const testResponse = await supertest + .post(epmRouteService.getInstallPath('security_detection_engine', version)) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(500); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); - return testResponse.body; + return testResponse.body; + } catch (error) { + log.error(`Retrying installPrebuiltRulesPackageByVersion: ${error}`); + throw error; + } }, retryService, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); From 73552368a4394cdc86943d2baa421e9b51eb4ba0 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 6 Feb 2024 17:37:11 +0100 Subject: [PATCH 2/4] Refactor --- .../install_latest_bundled_prebuilt_rules.ts | 3 +- .../prerelease_packages.ts | 3 +- .../update_prebuilt_rules_package.ts | 3 +- .../detections_response/utils/retry.ts | 15 +++++++-- .../install_fleet_package_by_url.ts | 31 +++++++++---------- .../install_prebuilt_rules_fleet_package.ts | 7 +++++ 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts index 32eb1d3dbdf0..dcb2561b1f3e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts @@ -61,7 +61,8 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, '99.0.0', - retry + retry, + log ); // As opposed to "registry" diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts index dcafdf8eaf1a..3ed663f7ecc6 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts @@ -49,7 +49,8 @@ export default ({ getService }: FtrProviderContext): void => { const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI( es, supertest, - retry + retry, + log ); expect(fleetPackageInstallationResponse.items.length).toBe(1); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts index d4c0b546fb42..ffba2bd01d98 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts @@ -161,7 +161,8 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, currentVersion, - retry + retry, + log ); expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts index 42f83af795ae..3007448ed895 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -44,17 +44,19 @@ import type { ToolingLog } from '@kbn/tooling-log'; export const retry = async ({ test, retryService, + utilityName, retries = 2, timeout = 30000, retryDelay = 200, log, }: { test: () => Promise; + utilityName: string; retryService: RetryService; retries?: number; timeout?: number; retryDelay?: number; - log?: ToolingLog; + log: ToolingLog; }): Promise => { let retryAttempt = 0; const response = await retryService.tryForTime( @@ -67,12 +69,19 @@ export const retry = async ({ retryAttempt - 1 }/${retries}`; log?.error(errorMessage); - return new Error(JSON.stringify({ retries: errorMessage })); + return new Error(JSON.stringify(errorMessage)); } retryAttempt = retryAttempt + 1; - return test(); + // Catch the error thrown by the test and log it, then throw it again + // to cause `tryForTime` to retry. + try { + return await test(); + } catch (error) { + log.error(`Retrying ${utilityName}: ${error}`); + throw error; + } }, undefined, retryDelay diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 48db5595b10c..331b260cb9a9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -29,7 +29,8 @@ const ATTEMPT_TIMEOUT = 120000; export const installPrebuiltRulesPackageViaFleetAPI = async ( es: Client, supertest: SuperTest.SuperTest, - retryService: RetryService + retryService: RetryService, + log: ToolingLog ): Promise => { const fleetResponse = await retry({ test: async () => { @@ -45,9 +46,11 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( return testResponse.body; }, + utilityName: installPrebuiltRulesPackageViaFleetAPI.name, retryService, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); @@ -73,23 +76,19 @@ export const installPrebuiltRulesPackageByVersion = async ( ): Promise => { const fleetResponse = await retry({ test: async () => { - try { - const testResponse = await supertest - .post(epmRouteService.getInstallPath('security_detection_engine', version)) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(500); - expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); - expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); + const testResponse = await supertest + .post(epmRouteService.getInstallPath('security_detection_engine', version)) + .set('kbn-xsrf', 'xxxx') + .set('elastic-api-version', '2023-10-31') + .type('application/json') + .send({ force: true }) + .expect(500); + expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); + expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); - return testResponse.body; - } catch (error) { - log.error(`Retrying installPrebuiltRulesPackageByVersion: ${error}`); - throw error; - } + return testResponse.body; }, + utilityName: installPrebuiltRulesPackageByVersion.name, retryService, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index 592406e8c339..770d966f50a5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -15,6 +15,7 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; import { RetryService } from '@kbn/ftr-common-functional-services'; import expect from 'expect'; +import { ToolingLog } from '@kbn/tooling-log'; import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../../refresh_index'; @@ -35,12 +36,14 @@ export const installPrebuiltRulesFleetPackage = async ({ version, overrideExistingPackage, retryService, + log, }: { es: Client; supertest: SuperTest.SuperTest; version?: string; overrideExistingPackage: boolean; retryService: RetryService; + log: ToolingLog; }): Promise => { if (version) { // Install a specific version @@ -59,8 +62,10 @@ export const installPrebuiltRulesFleetPackage = async ({ return testResponse.body; }, retryService, + utilityName: installPrebuiltRulesFleetPackage.name, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); @@ -91,8 +96,10 @@ export const installPrebuiltRulesFleetPackage = async ({ return body; }, retryService, + utilityName: installPrebuiltRulesFleetPackage.name, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); From f86277bb0f695a7d867ea0a6a42ac4e1454bd6a0 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Tue, 6 Feb 2024 17:49:05 +0100 Subject: [PATCH 3/4] Fix test --- .../utils/rules/prebuilt_rules/install_fleet_package_by_url.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 331b260cb9a9..2839795ab197 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -82,7 +82,7 @@ export const installPrebuiltRulesPackageByVersion = async ( .set('elastic-api-version', '2023-10-31') .type('application/json') .send({ force: true }) - .expect(500); + .expect(200); expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); From c9d5f8911392b88dacc4376e05c745afad645e52 Mon Sep 17 00:00:00 2001 From: jpdjere Date: Wed, 7 Feb 2024 11:07:15 +0100 Subject: [PATCH 4/4] Pass missing param --- .../management/trial_license_complete_tier/fleet_integration.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts index 1233af5c33f6..1a8394a3b514 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts @@ -49,6 +49,7 @@ export default ({ getService }: FtrProviderContext): void => { supertest, overrideExistingPackage: true, retryService: retry, + log, }); // Verify that status is updated after package installation