diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts index 383e37511ef8..acae8340b9d7 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts @@ -10,17 +10,31 @@ import { FtrService } from '../ftr_provider_context'; import { retryForSuccess } from './retry_for_success'; import { retryForTruthy } from './retry_for_truthy'; +interface TryWithRetriesOptions { + retryCount: number; + retryDelay?: number; + timeout?: number; +} + export class RetryService extends FtrService { private readonly config = this.ctx.getService('config'); private readonly log = this.ctx.getService('log'); + /** + * Use to retry block within {timeout} period and return block result. + * @param timeout retrying timeout + * @param block retriable action + * @param onFailureBlock optional action to run before the new retriable action attempt + * @param retryDelay optional delay before the new attempt + * @returns result from retriable action + */ public async tryForTime( timeout: number, block: () => Promise, onFailureBlock?: () => Promise, retryDelay?: number ) { - return await retryForSuccess(this.log, { + return await retryForSuccess(this.log, { timeout, methodName: 'retry.tryForTime', block, @@ -43,6 +57,13 @@ export class RetryService extends FtrService { }); } + /** + * Use to wait for block condition to be true + * @param description description for retriable action + * @param timeout retrying timeout + * @param block retriable action + * @param onFailureBlock optional action to run before the new retriable action attempt + */ public async waitForWithTimeout( description: string, timeout: number, @@ -71,4 +92,31 @@ export class RetryService extends FtrService { onFailureBlock, }); } + + /** + * Use to retry block {options.retryCount} times within {options.timeout} period and return block result + * @param description description for retriable action + * @param block retriable action + * @param options options.retryCount for how many attempts to retry + * @param onFailureBlock optional action to run before the new retriable action attempt + * @returns result from retriable action + */ + public async tryWithRetries( + description: string, + block: () => Promise, + options: TryWithRetriesOptions, + onFailureBlock?: () => Promise + ): Promise { + const { retryCount, timeout = this.config.get('timeouts.try'), retryDelay = 200 } = options; + + return await retryForSuccess(this.log, { + description, + timeout, + methodName: 'retry.tryWithRetries', + block, + onFailureBlock, + retryDelay, + retryCount, + }); + } } diff --git a/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts index f59d76028b9f..caa98a36b3f0 100644 --- a/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts @@ -13,9 +13,9 @@ const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const returnTrue = () => true; -const defaultOnFailure = (methodName: string) => (lastError: Error | undefined) => { +const defaultOnFailure = (methodName: string) => (lastError: Error | undefined, reason: string) => { throw new Error( - `${methodName} timeout${lastError ? `: ${lastError.stack || lastError.message}` : ''}` + `${methodName} ${reason}\n${lastError ? `${lastError.stack || lastError.message}` : ''}` ); }; @@ -44,32 +44,51 @@ interface Options { onFailureBlock?: () => Promise; onFailure?: ReturnType; accept?: (v: T) => boolean; + description?: string; retryDelay?: number; + retryCount?: number; } export async function retryForSuccess(log: ToolingLog, options: Options) { const { + description, timeout, methodName, block, onFailureBlock, + onFailure = defaultOnFailure(methodName), accept = returnTrue, retryDelay = 502, + retryCount, } = options; - const { onFailure = defaultOnFailure(methodName) } = options; const start = Date.now(); const criticalWebDriverErrors = ['NoSuchSessionError', 'NoSuchWindowError']; let lastError; + let attemptCounter = 0; + const addText = (str: string | undefined) => (str ? ` waiting for '${str}'` : ''); while (true) { + // Aborting if no retry attempts are left (opt-in) + if (retryCount && ++attemptCounter > retryCount) { + onFailure( + lastError, + // optionally extend error message with description + `reached the limit of attempts${addText(description)}: ${ + attemptCounter - 1 + } out of ${retryCount}` + ); + } + // Aborting if timeout is reached if (Date.now() - start > timeout) { - await onFailure(lastError); - throw new Error('expected onFailure() option to throw an error'); - } else if (lastError && criticalWebDriverErrors.includes(lastError.name)) { - // Aborting retry since WebDriver session is invalid or browser window is closed + onFailure(lastError, `reached timeout ${timeout} ms${addText(description)}`); + } + // Aborting if WebDriver session is invalid or browser window is closed + if (lastError && criticalWebDriverErrors.includes(lastError.name)) { throw new Error('WebDriver session is invalid, retry was aborted'); - } else if (lastError && onFailureBlock) { + } + // Run opt-in onFailureBlock before the next attempt + if (lastError && onFailureBlock) { const before = await runAttempt(onFailureBlock); if ('error' in before) { log.debug(`--- onRetryBlock error: ${before.error.message}`); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts index cba12ecf3376..8e90b423885a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts @@ -61,8 +61,7 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, '99.0.0', - retry, - log + retry ); // As opposed to "registry" diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts index fa3ffe093b5b..b0e7269a386e 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts @@ -49,8 +49,7 @@ export default ({ getService }: FtrProviderContext): void => { const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI( es, supertest, - retry, - log + retry ); expect(fleetPackageInstallationResponse.items.length).toBe(1); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts index 19dcd4ba1aa3..37c38e25397a 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts @@ -49,7 +49,6 @@ export default ({ getService }: FtrProviderContext): void => { supertest, overrideExistingPackage: true, retryService: retry, - log, }); // Verify that status is updated after package installation diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts index 21535741597d..98264cada697 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts @@ -107,8 +107,7 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, previousVersion, - retry, - log + retry ); expect(installPreviousPackageResponse._meta.install_source).toBe('registry'); @@ -161,8 +160,7 @@ export default ({ getService }: FtrProviderContext): void => { es, supertest, currentVersion, - retry, - log + retry ); expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts index d86075bb41f6..2ce85256b0fb 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/index.ts @@ -22,5 +22,4 @@ export * from './wait_for_index_to_populate'; export * from './get_stats'; export * from './get_detection_metrics_from_body'; export * from './get_stats_url'; -export * from './retry'; export * from './combine_to_ndjson'; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts deleted file mode 100644 index 3007448ed895..000000000000 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { RetryService } from '@kbn/ftr-common-functional-services'; -import type { ToolingLog } from '@kbn/tooling-log'; -/** - * Retry wrapper for async supertests, with a maximum number of retries. - * You can pass in a function that executes a supertest test, and make assertions - * on the response. If the test fails, it will retry the test the number of retries - * that are passed in. - * - * Example usage: - * ```ts - const fleetResponse = await retry({ - test: async () => { - const testResponse = await supertest - .post(`/api/fleet/epm/packages/security_detection_engine`) - .set('kbn-xsrf', 'xxxx') - .set('elastic-api-version', '2023-10-31') - .type('application/json') - .send({ force: true }) - .expect(200); - expect((testResponse.body as InstallPackageResponse).items).toBeDefined(); - expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0); - - return testResponse.body; - }, - retryService, - retries: MAX_RETRIES, - timeout: ATTEMPT_TIMEOUT, - }); - * ``` - * @param test The function containing a test to run - * @param retryService The retry service to use - * @param retries The maximum number of retries - * @param timeout The timeout for each retry - * @param retryDelay The delay between each retry - * @returns The response from the test - */ -export const retry = async ({ - test, - retryService, - utilityName, - retries = 2, - timeout = 30000, - retryDelay = 200, - log, -}: { - test: () => Promise; - utilityName: string; - retryService: RetryService; - retries?: number; - timeout?: number; - retryDelay?: number; - log: ToolingLog; -}): Promise => { - let retryAttempt = 0; - const response = await retryService.tryForTime( - timeout, - async () => { - if (retryAttempt > retries) { - // Log error message if we reached the maximum number of retries - // but don't throw an error, return it to break the retry loop. - const errorMessage = `Reached maximum number of retries for test: ${ - retryAttempt - 1 - }/${retries}`; - log?.error(errorMessage); - return new Error(JSON.stringify(errorMessage)); - } - - retryAttempt = retryAttempt + 1; - - // Catch the error thrown by the test and log it, then throw it again - // to cause `tryForTime` to retry. - try { - return await test(); - } catch (error) { - log.error(`Retrying ${utilityName}: ${error}`); - throw error; - } - }, - undefined, - retryDelay - ); - - // Now throw the error in order to fail the test. - if (response instanceof Error) { - throw response; - } - - return response; -}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts index 2839795ab197..863e4d79fb00 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts @@ -9,9 +9,7 @@ import type SuperTest from 'supertest'; import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import { epmRouteService } from '@kbn/fleet-plugin/common'; import { RetryService } from '@kbn/ftr-common-functional-services'; -import type { ToolingLog } from '@kbn/tooling-log'; import expect from 'expect'; -import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../../refresh_index'; const MAX_RETRIES = 2; @@ -29,11 +27,11 @@ const ATTEMPT_TIMEOUT = 120000; export const installPrebuiltRulesPackageViaFleetAPI = async ( es: Client, supertest: SuperTest.SuperTest, - retryService: RetryService, - log: ToolingLog + retryService: RetryService ): Promise => { - const fleetResponse = await retry({ - test: async () => { + const fleetResponse = await retryService.tryWithRetries( + installPrebuiltRulesPackageViaFleetAPI.name, + async () => { const testResponse = await supertest .post(`/api/fleet/epm/packages/security_detection_engine`) .set('kbn-xsrf', 'xxxx') @@ -46,12 +44,11 @@ export const installPrebuiltRulesPackageViaFleetAPI = async ( return testResponse.body; }, - utilityName: installPrebuiltRulesPackageViaFleetAPI.name, - retryService, - retries: MAX_RETRIES, - timeout: ATTEMPT_TIMEOUT, - log, - }); + { + retryCount: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + } + ); await refreshSavedObjectIndices(es); @@ -71,11 +68,11 @@ export const installPrebuiltRulesPackageByVersion = async ( es: Client, supertest: SuperTest.SuperTest, version: string, - retryService: RetryService, - log: ToolingLog + retryService: RetryService ): Promise => { - const fleetResponse = await retry({ - test: async () => { + const fleetResponse = await retryService.tryWithRetries( + installPrebuiltRulesPackageByVersion.name, + async () => { const testResponse = await supertest .post(epmRouteService.getInstallPath('security_detection_engine', version)) .set('kbn-xsrf', 'xxxx') @@ -88,12 +85,11 @@ export const installPrebuiltRulesPackageByVersion = async ( return testResponse.body; }, - utilityName: installPrebuiltRulesPackageByVersion.name, - retryService, - retries: MAX_RETRIES, - timeout: ATTEMPT_TIMEOUT, - log, - }); + { + retryCount: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + } + ); await refreshSavedObjectIndices(es); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts index 770d966f50a5..cbe609501a5f 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts @@ -15,8 +15,6 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types'; import type SuperTest from 'supertest'; import { RetryService } from '@kbn/ftr-common-functional-services'; import expect from 'expect'; -import { ToolingLog } from '@kbn/tooling-log'; -import { retry } from '../../retry'; import { refreshSavedObjectIndices } from '../../refresh_index'; const MAX_RETRIES = 2; @@ -36,19 +34,18 @@ export const installPrebuiltRulesFleetPackage = async ({ version, overrideExistingPackage, retryService, - log, }: { es: Client; supertest: SuperTest.SuperTest; version?: string; overrideExistingPackage: boolean; retryService: RetryService; - log: ToolingLog; }): Promise => { if (version) { // Install a specific version - const response = await retry({ - test: async () => { + const response = await retryService.tryWithRetries( + installPrebuiltRulesFleetPackage.name, + async () => { const testResponse = await supertest .post(epmRouteService.getInstallPath('security_detection_engine', version)) .set('kbn-xsrf', 'true') @@ -61,20 +58,20 @@ export const installPrebuiltRulesFleetPackage = async ({ return testResponse.body; }, - retryService, - utilityName: installPrebuiltRulesFleetPackage.name, - retries: MAX_RETRIES, - timeout: ATTEMPT_TIMEOUT, - log, - }); + { + retryCount: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + } + ); await refreshSavedObjectIndices(es); return response; } else { // Install the latest version - const response = await retry({ - test: async () => { + const response = await retryService.tryWithRetries( + installPrebuiltRulesFleetPackage.name, + async () => { const testResponse = await supertest .post(epmRouteService.getBulkInstallPath()) .query({ prerelease: true }) @@ -95,12 +92,11 @@ export const installPrebuiltRulesFleetPackage = async ({ return body; }, - retryService, - utilityName: installPrebuiltRulesFleetPackage.name, - retries: MAX_RETRIES, - timeout: ATTEMPT_TIMEOUT, - log, - }); + { + retryCount: MAX_RETRIES, + timeout: ATTEMPT_TIMEOUT, + } + ); await refreshSavedObjectIndices(es);