Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Security Solution] Improve logging for FTR test retry function #176316

Merged
merged 5 commits into from
Feb 8, 2024
Merged

[Security Solution] Improve logging for FTR test retry function #176316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ecfd52f
Add try catch to test
jpdjere Feb 6, 2024
7355236
Refactor
jpdjere Feb 6, 2024
f86277b
Fix test
jpdjere Feb 6, 2024
c9d5f89
Pass missing param
jpdjere Feb 7, 2024
cd59bd8
Merge branch 'main' into flaky-tests-2024-02-06
jpdjere Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ...ebuilt_rules_package/trial_license_complete_tier/install_latest_bundled_prebuilt_rules.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ export default ({ getService }: FtrProviderContext): void => {
es,
supertest,
'99.0.0',
retry
retry,
log
);

// As opposed to "registry"
Expand Down
3 changes: 2 additions & 1 deletion ...t_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/prerelease_packages.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ export default ({ getService }: FtrProviderContext): void => {
const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI(
es,
supertest,
retry
retry,
log
);

expect(fleetPackageInstallationResponse.items.length).toBe(1);
Expand Down
1 change: 1 addition & 0 deletions ...les_management/prebuilt_rules/management/trial_license_complete_tier/fleet_integration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ export default ({ getService }: FtrProviderContext): void => {
supertest,
overrideExistingPackage: true,
retryService: retry,
log,
});

// Verify that status is updated after package installation
Expand Down
6 changes: 4 additions & 2 deletions ...pdate_prebuilt_rules_package/trial_license_complete_tier/update_prebuilt_rules_package.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ export default ({ getService }: FtrProviderContext): void => {
es,
supertest,
previousVersion,
retry
retry,
log
);

expect(installPreviousPackageResponse._meta.install_source).toBe('registry');
Expand Down Expand Up @@ -160,7 +161,8 @@ export default ({ getService }: FtrProviderContext): void => {
es,
supertest,
currentVersion,
retry
retry,
log
);
expect(installLatestPackageResponse.items.length).toBeGreaterThanOrEqual(0);

Expand Down
21 changes: 18 additions & 3 deletions x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*/

import { RetryService } from '@kbn/ftr-common-functional-services';

import type { ToolingLog } from '@kbn/tooling-log';
/**
* Retry wrapper for async supertests, with a maximum number of retries.
* You can pass in a function that executes a supertest test, and make assertions
Expand Down Expand Up @@ -44,15 +44,19 @@ import { RetryService } from '@kbn/ftr-common-functional-services';
export const retry = async <T>({
test,
retryService,
utilityName,
retries = 2,
timeout = 30000,
retryDelay = 200,
log,
}: {
test: () => Promise<T>;
utilityName: string;
retryService: RetryService;
retries?: number;
timeout?: number;
retryDelay?: number;
log: ToolingLog;
}): Promise<T> => {
let retryAttempt = 0;
const response = await retryService.tryForTime(
Expand All @@ -61,12 +65,23 @@ export const retry = async <T>({
if (retryAttempt > retries) {
// Log error message if we reached the maximum number of retries
// but don't throw an error, return it to break the retry loop.
return new Error('Reached maximum number of retries for test.');
const errorMessage = `Reached maximum number of retries for test: ${
retryAttempt - 1
}/${retries}`;
log?.error(errorMessage);
return new Error(JSON.stringify(errorMessage));
}

retryAttempt = retryAttempt + 1;

return test();
// Catch the error thrown by the test and log it, then throw it again
// to cause `tryForTime` to retry.
try {
return await test();
} catch (error) {
log.error(`Retrying ${utilityName}: ${error}`);
throw error;
}
},
undefined,
retryDelay
Expand Down
11 changes: 9 additions & 2 deletions ...est_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type SuperTest from 'supertest';
import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types';
import { epmRouteService } from '@kbn/fleet-plugin/common';
import { RetryService } from '@kbn/ftr-common-functional-services';
import type { ToolingLog } from '@kbn/tooling-log';
import expect from 'expect';
import { retry } from '../../retry';
import { refreshSavedObjectIndices } from '../../refresh_index';
Expand All @@ -28,7 +29,8 @@ const ATTEMPT_TIMEOUT = 120000;
export const installPrebuiltRulesPackageViaFleetAPI = async (
es: Client,
supertest: SuperTest.SuperTest<SuperTest.Test>,
retryService: RetryService
retryService: RetryService,
log: ToolingLog
): Promise<InstallPackageResponse> => {
const fleetResponse = await retry<InstallPackageResponse>({
test: async () => {
Expand All @@ -44,9 +46,11 @@ export const installPrebuiltRulesPackageViaFleetAPI = async (

return testResponse.body;
},
utilityName: installPrebuiltRulesPackageViaFleetAPI.name,
retryService,
retries: MAX_RETRIES,
timeout: ATTEMPT_TIMEOUT,
log,
});

await refreshSavedObjectIndices(es);
Expand All @@ -67,7 +71,8 @@ export const installPrebuiltRulesPackageByVersion = async (
es: Client,
supertest: SuperTest.SuperTest<SuperTest.Test>,
version: string,
retryService: RetryService
retryService: RetryService,
log: ToolingLog
): Promise<InstallPackageResponse> => {
const fleetResponse = await retry<InstallPackageResponse>({
test: async () => {
Expand All @@ -83,9 +88,11 @@ export const installPrebuiltRulesPackageByVersion = async (

return testResponse.body;
},
utilityName: installPrebuiltRulesPackageByVersion.name,
retryService,
retries: MAX_RETRIES,
timeout: ATTEMPT_TIMEOUT,
log,
});

await refreshSavedObjectIndices(es);
Expand Down
7 changes: 7 additions & 0 deletions ...es/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types';
import type SuperTest from 'supertest';
import { RetryService } from '@kbn/ftr-common-functional-services';
import expect from 'expect';
import { ToolingLog } from '@kbn/tooling-log';
import { retry } from '../../retry';
import { refreshSavedObjectIndices } from '../../refresh_index';

Expand All @@ -35,12 +36,14 @@ export const installPrebuiltRulesFleetPackage = async ({
version,
overrideExistingPackage,
retryService,
log,
}: {
es: Client;
supertest: SuperTest.SuperTest<SuperTest.Test>;
version?: string;
overrideExistingPackage: boolean;
retryService: RetryService;
log: ToolingLog;
}): Promise<InstallPackageResponse | BulkInstallPackagesResponse> => {
if (version) {
// Install a specific version
Expand All @@ -59,8 +62,10 @@ export const installPrebuiltRulesFleetPackage = async ({
return testResponse.body;
},
retryService,
utilityName: installPrebuiltRulesFleetPackage.name,
retries: MAX_RETRIES,
timeout: ATTEMPT_TIMEOUT,
log,
});

await refreshSavedObjectIndices(es);
Expand Down Expand Up @@ -91,8 +96,10 @@ export const installPrebuiltRulesFleetPackage = async ({
return body;
},
retryService,
utilityName: installPrebuiltRulesFleetPackage.name,
retries: MAX_RETRIES,
timeout: ATTEMPT_TIMEOUT,
log,
});

await refreshSavedObjectIndices(es);
Expand Down