From c5a102b02b66f57525efc06e97c71fddfc1c7888 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:00:07 -0500 Subject: [PATCH] [8.12] [Security Solution] Improve logging for FTR test `retry` function (#176316) (#176497) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.12`: - [[Security Solution] Improve logging for FTR test `retry` function (#176316)](https://github.com/elastic/kibana/pull/176316) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Juan Pablo Djeredjian --- .../install_latest_bundled_prebuilt_rules.ts | 3 ++- .../prerelease_packages.ts | 3 ++- .../management/fleet_integration.ts | 1 + .../update_prebuilt_rules_package.ts | 6 ++++-- .../detections_response/utils/retry.ts | 21 ++++++++++++++++--- .../install_fleet_package_by_url.ts | 11 ++++++++-- .../install_prebuilt_rules_fleet_package.ts | 7 +++++++ 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts index 33f79b1bff8d..32dbcfe8724a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts @@ -61,7 +61,8 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, '99.0.0', - retry + retry, + log ); // As opposed to "registry" diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts index 83faf92bf7c8..74e9255269bf 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts @@ -49,7 +49,8 @@ export default ({ getService }: FtrProviderContext): void => { const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI( es, supertest, - retry + retry, + log ); expect(fleetPackageInstallationResponse.items.length).toBe(1); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts index a9653d759320..1599bfead346 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts @@ -49,6 +49,7 @@ export default ({ getService }: FtrProviderContext): void => { supertest, overrideExistingPackage: true, retryService: retry, + log, }); // Verify that status is updated after package installation diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts index 688816569c18..008bb44a8a33 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/update_prebuilt_rules_package.ts @@ -107,7 +107,8 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, previousVersion, - retry + retry, + log ); expect(installPreviousPackageResponse._meta.install_source).toBe('registry'); @@ -160,7 +161,8 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, currentVersion, - retry + retry, + log ); expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts index dafd16aaa9f5..3007448ed895 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts @@ -6,7 +6,7 @@ */ import { RetryService } from '@kbn/ftr-common-functional-services'; - +import type { ToolingLog } from '@kbn/tooling-log'; /** * Retry wrapper for async supertests, with a maximum number of retries. * You can pass in a function that executes a supertest test, and make assertions @@ -44,15 +44,19 @@ import { RetryService } from '@kbn/ftr-common-functional-services'; export const retry = async ({ test, retryService, + utilityName, retries = 2, timeout = 30000, retryDelay = 200, + log, }: { test: () => Promise; + utilityName: string; retryService: RetryService; retries?: number; timeout?: number; retryDelay?: number; + log: ToolingLog; }): Promise => { let retryAttempt = 0; const response = await retryService.tryForTime( @@ -61,12 +65,23 @@ export const retry = async ({ if (retryAttempt > retries) { // Log error message if we reached the maximum number of retries // but don't throw an error, return it to break the retry loop. - return new Error('Reached maximum number of retries for test.'); + const errorMessage = `Reached maximum number of retries for test: ${ + retryAttempt - 1 + }/${retries}`; + log?.error(errorMessage); + return new Error(JSON.stringify(errorMessage)); } retryAttempt = retryAttempt + 1; - return test(); + // Catch the error thrown by the test and log it, then throw it again + // to cause `tryForTime` to retry. + try { + return await test(); + } catch (error) { + log.error(`Retrying ${utilityName}: ${error}`); + throw error; + } }, undefined, retryDelay diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 988d73660d0e..2839795ab197 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -9,6 +9,7 @@ import type SuperTest from 'supertest'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; import { RetryService } from '@kbn/ftr-common-functional-services'; +import type { ToolingLog } from '@kbn/tooling-log'; import expect from 'expect'; import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../../refresh_index'; @@ -28,7 +29,8 @@ const ATTEMPT_TIMEOUT = 120000; export const installPrebuiltRulesPackageViaFleetAPI = async ( es: Client, supertest: SuperTest.SuperTest, - retryService: RetryService + retryService: RetryService, + log: ToolingLog ): Promise => { const fleetResponse = await retry({ test: async () => { @@ -44,9 +46,11 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( return testResponse.body; }, + utilityName: installPrebuiltRulesPackageViaFleetAPI.name, retryService, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); @@ -67,7 +71,8 @@ export const installPrebuiltRulesPackageByVersion = async ( es: Client, supertest: SuperTest.SuperTest, version: string, - retryService: RetryService + retryService: RetryService, + log: ToolingLog ): Promise => { const fleetResponse = await retry({ test: async () => { @@ -83,9 +88,11 @@ export const installPrebuiltRulesPackageByVersion = async ( return testResponse.body; }, + utilityName: installPrebuiltRulesPackageByVersion.name, retryService, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index 592406e8c339..770d966f50a5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -15,6 +15,7 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; import { RetryService } from '@kbn/ftr-common-functional-services'; import expect from 'expect'; +import { ToolingLog } from '@kbn/tooling-log'; import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../../refresh_index'; @@ -35,12 +36,14 @@ export const installPrebuiltRulesFleetPackage = async ({ version, overrideExistingPackage, retryService, + log, }: { es: Client; supertest: SuperTest.SuperTest; version?: string; overrideExistingPackage: boolean; retryService: RetryService; + log: ToolingLog; }): Promise => { if (version) { // Install a specific version @@ -59,8 +62,10 @@ export const installPrebuiltRulesFleetPackage = async ({ return testResponse.body; }, retryService, + utilityName: installPrebuiltRulesFleetPackage.name, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es); @@ -91,8 +96,10 @@ export const installPrebuiltRulesFleetPackage = async ({ return body; }, retryService, + utilityName: installPrebuiltRulesFleetPackage.name, retries: MAX_RETRIES, timeout: ATTEMPT_TIMEOUT, + log, }); await refreshSavedObjectIndices(es);