[Security Solution] Unskip flaky tests in Prebuilt Rules FTR Integration tests

packages/kbn-ftr-common-functional-services/services/retry/retry.ts

@@ -17,22 +17,29 @@ export class RetryService extends FtrService {
   public async tryForTime<T>(
     timeout: number,
     block: () => Promise<T>,
-    onFailureBlock?: () => Promise<T>
+    onFailureBlock?: () => Promise<T>,
+    retryDelay?: number
   ) {
     return await retryForSuccess(this.log, {
       timeout,
       methodName: 'retry.tryForTime',
       block,
       onFailureBlock,
+      retryDelay,
     });
   }
 
-  public async try<T>(block: () => Promise<T>, onFailureBlock?: () => Promise<T>) {
+  public async try<T>(
+    block: () => Promise<T>,
+    onFailureBlock?: () => Promise<T>,
+    retryDelay?: number
+  ) {
     return await retryForSuccess(this.log, {
       timeout: this.config.get('timeouts.try'),
       methodName: 'retry.try',
       block,
       onFailureBlock,
+      retryDelay,
     });
   }
 

packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts

@@ -44,14 +44,21 @@ interface Options<T> {
   onFailureBlock?: () => Promise<T>;
   onFailure?: ReturnType<typeof defaultOnFailure>;
   accept?: (v: T) => boolean;
+  retryDelay?: number;
 }
 
 export async function retryForSuccess<T>(log: ToolingLog, options: Options<T>) {
-  const { timeout, methodName, block, onFailureBlock, accept = returnTrue } = options;
+  const {
+    timeout,
+    methodName,
+    block,
+    onFailureBlock,
+    accept = returnTrue,
+    retryDelay = 502,
+  } = options;
   const { onFailure = defaultOnFailure(methodName) } = options;
 
   const start = Date.now();
-  const retryDelay = 502;
   const criticalWebDriverErrors = ['NoSuchSessionError', 'NoSuchWindowError'];
   let lastError;
 

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/update_actions.ts

@@ -218,7 +218,7 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await updateRule(supertest, ruleToUpdate);
 
-        const status = await getPrebuiltRulesAndTimelinesStatus(supertest);
+        const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest);
         expect(status.rules_not_installed).toBe(0);
       });
 

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/exceptions/workflows/role_based_rule_exceptions_workflows.ts

@@ -526,7 +526,7 @@ export default ({ getService }: FtrProviderContext) => {
             })
             .expect(200);
 
-          const status = await getPrebuiltRulesAndTimelinesStatus(supertest);
+          const status = await getPrebuiltRulesAndTimelinesStatus(es, supertest);
           expect(status.rules_not_installed).toEqual(0);
         });
       });

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/install_latest_bundled_prebuilt_rules.ts

@@ -22,14 +22,14 @@ export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
+  const retry = getService('retry');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/171380
   /* This test simulates an air-gapped environment in which the user doesn't have access to EPR.
   /* We first download the package from the registry as done during build time, and then
   /* attempt to install it from the local file system. The API response from EPM provides
   /* us with the information of whether the package was installed from the registry or
   /* from a package that was bundled with Kibana */
-  describe.skip('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => {
+  describe('@ess @serverless @skipInQA install_bundled_prebuilt_rules', () => {
     beforeEach(async () => {
       await deleteAllRules(supertest, log);
       await deleteAllPrebuiltRuleAssets(es);
@@ -52,15 +52,16 @@ export default ({ getService }: FtrProviderContext): void => {
 
     it('should install prebuilt rules from the package that comes bundled with Kibana', async () => {
       // Verify that status is empty before package installation
-      const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest);
+      const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);
 
       const bundledInstallResponse = await installPrebuiltRulesPackageByVersion(
         es,
         supertest,
-        '99.0.0'
+        '99.0.0',
+        retry
       );
 
       // As opposed to "registry"
@@ -71,7 +72,7 @@ export default ({ getService }: FtrProviderContext): void => {
       await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });
 
       // Verify that status is updated after package installation
-      const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest);
+      const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBeGreaterThan(0);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/prerelease_packages.ts

@@ -22,6 +22,7 @@ export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
+  const retry = getService('retry');
 
   /* This test makes use of the mock packages created in the '/fleet_bundled_packages' folder,
   /* in order to assert that, in production environments, the latest stable version of the package
@@ -39,15 +40,16 @@ export default ({ getService }: FtrProviderContext): void => {
 
     it('should install latest stable version and ignore prerelease packages', async () => {
       // Verify that status is empty before package installation
-      const statusBeforePackageInstallation = await getPrebuiltRulesStatus(supertest);
+      const statusBeforePackageInstallation = await getPrebuiltRulesStatus(es, supertest);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_installed).toBe(0);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_install).toBe(0);
       expect(statusBeforePackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);
 
       // Install package without specifying version to check if latest stable version is installed
       const fleetPackageInstallationResponse = await installPrebuiltRulesPackageViaFleetAPI(
         es,
-        supertest
+        supertest,
+        retry
       );
 
       expect(fleetPackageInstallationResponse.items.length).toBe(1);
@@ -59,7 +61,7 @@ export default ({ getService }: FtrProviderContext): void => {
       expect(prebuiltRulesFleetPackage.status).toBe(200);
 
       // Get status of our prebuilt rules (nothing should be instaled yet)
-      const statusAfterPackageInstallation = await getPrebuiltRulesStatus(supertest);
+      const statusAfterPackageInstallation = await getPrebuiltRulesStatus(es, supertest);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_installed).toBe(0);
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_install).toBe(1); // 1 rule in package 99.0.0
       expect(statusAfterPackageInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);
@@ -68,7 +70,7 @@ export default ({ getService }: FtrProviderContext): void => {
       await installPrebuiltRules(es, supertest);
 
       // Verify that status is updated after package installation
-      const statusAfterRulesInstallation = await getPrebuiltRulesStatus(supertest);
+      const statusAfterRulesInstallation = await getPrebuiltRulesStatus(es, supertest);
       expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_installed).toBe(1); // 1 rule in package 99.0.0
       expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_to_install).toBe(0);
       expect(statusAfterRulesInstallation.stats.num_prebuilt_rules_to_upgrade).toBe(0);

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/install_large_prebuilt_rules_package.ts

@@ -31,7 +31,10 @@ export default ({ getService }: FtrProviderContext): void => {
 
     it('should install a package containing 15000 prebuilt rules without crashing', async () => {
       // Verify that status is empty before package installation
-      const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
+      const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(
+        es,
+        supertest
+      );
       expect(statusBeforePackageInstallation.rules_installed).toBe(0);
       expect(statusBeforePackageInstallation.rules_not_installed).toBe(0);
       expect(statusBeforePackageInstallation.rules_not_updated).toBe(0);
@@ -40,7 +43,10 @@ export default ({ getService }: FtrProviderContext): void => {
       await installPrebuiltRulesAndTimelines(es, supertest);
 
       // Verify that status is updated after package installation
-      const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
+      const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(
+        es,
+        supertest
+      );
       expect(statusAfterPackageInstallation.rules_installed).toBe(750);
       expect(statusAfterPackageInstallation.rules_not_installed).toBe(0);
       expect(statusAfterPackageInstallation.rules_not_updated).toBe(0);

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/fleet_integration.ts

@@ -20,6 +20,7 @@ export default ({ getService }: FtrProviderContext): void => {
   const es = getService('es');
   const supertest = getService('supertest');
   const log = getService('log');
+  const retry = getService('retry');
 
   describe('@ess @serverless @skipInQA install_prebuilt_rules_from_real_package', () => {
     beforeEach(async () => {
@@ -33,10 +34,12 @@ export default ({ getService }: FtrProviderContext): void => {
      * Unlike other tests that use mocks, this test uses actual rules from the
      * package storage and checks that they are installed.
      */
-    // TODO: Fix and unskip https://github.com/elastic/kibana/issues/172107
-    it.skip('should install prebuilt rules from the package storage', async () => {
+    it('should install prebuilt rules from the package storage', async () => {
       // Verify that status is empty before package installation
-      const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
+      const statusBeforePackageInstallation = await getPrebuiltRulesAndTimelinesStatus(
+        es,
+        supertest
+      );
       expect(statusBeforePackageInstallation.rules_installed).toBe(0);
       expect(statusBeforePackageInstallation.rules_not_installed).toBe(0);
       expect(statusBeforePackageInstallation.rules_not_updated).toBe(0);
@@ -45,10 +48,14 @@ export default ({ getService }: FtrProviderContext): void => {
         es,
         supertest,
         overrideExistingPackage: true,
+        retryService: retry,
       });
 
       // Verify that status is updated after package installation
-      const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
+      const statusAfterPackageInstallation = await getPrebuiltRulesAndTimelinesStatus(
+        es,
+        supertest
+      );
       expect(statusAfterPackageInstallation.rules_installed).toBe(0);
       expect(statusAfterPackageInstallation.rules_not_installed).toBeGreaterThan(0);
       expect(statusAfterPackageInstallation.rules_not_updated).toBe(0);
@@ -59,7 +66,7 @@ export default ({ getService }: FtrProviderContext): void => {
       expect(response.rules_updated).toBe(0);
 
       // Verify that status is updated after rules installation
-      const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(supertest);
+      const statusAfterRuleInstallation = await getPrebuiltRulesAndTimelinesStatus(es, supertest);
       expect(statusAfterRuleInstallation.rules_installed).toBe(response.rules_installed);
       expect(statusAfterRuleInstallation.rules_not_installed).toBe(0);
       expect(statusAfterRuleInstallation.rules_not_updated).toBe(0);