Added docs to retry function

x-pack/test/detection_engine_api_integration/utils/index.ts

@@ -14,7 +14,6 @@ export * from './create_signals_index';
 export * from './delete_all_rules';
 export * from './delete_all_alerts';
 export * from './delete_all_timelines';
-export * from './es_indices';
 export * from './get_complex_rule';
 export * from './get_complex_rule_output';
 export * from './get_simple_rule';
@@ -29,7 +28,6 @@ export * from './update_rule';
 export * from './wait_for';
 export * from './wait_for_rule_status';
 export * from './prebuilt_rules/create_prebuilt_rule_saved_objects';
-export * from './prebuilt_rules/install_prebuilt_rules_and_timelines';
 export * from './get_simple_rule_update';
 export * from './get_simple_ml_rule_update';
 export * from './create_non_security_rule';

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/refresh_index.ts

@@ -6,6 +6,7 @@
  */
 
 import type { Client } from '@elastic/elasticsearch';
+import { ALL_SAVED_OBJECT_INDICES } from '@kbn/core-saved-objects-server';
 
 /**
  * Refresh an index, making changes available to search.
@@ -17,3 +18,28 @@ export const refreshIndex = async (es: Client, index?: string) => {
     index,
   });
 };
+
+/**
+ * Refresh an index, making changes available to search.
+ * Reusable utility which refreshes all saved object indices, to make them available for search, especially
+ * useful when needing to perform a search on an index that has just been written to.
+ *
+ * An example of this when installing the prebuilt detection rules SO of type 'security-rule':
+ * the savedObjectsClient does this with a call with explicit `refresh: false`.
+ * So, despite of the fact that the endpoint waits until the prebuilt rule will be
+ * successfully indexed, it doesn't wait until they become "visible" for subsequent read
+ * operations.
+ *
+ * Additionally, this method clears the cache for all saved object indices. This helps in cases in which
+ * saved object is read, then written to, and then read again, and the second read returns stale data.
+ * @param es The Elasticsearch client
+ */
+export const refreshSavedObjectIndices = async (es: Client) => {
+  // Refresh indices to prevent a race condition between a write and subsequent read operation. To
+  // fix it deterministically we have to refresh saved object indices and wait until it's done.
+  await es.indices.refresh({ index: ALL_SAVED_OBJECT_INDICES });
+
+  // Additionally, we need to clear the cache to ensure that the next read operation will
+  // not return stale data.
+  await es.indices.clearCache({ index: ALL_SAVED_OBJECT_INDICES });
+};

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/retry.ts

@@ -7,20 +7,53 @@
 
 import { RetryService } from '@kbn/ftr-common-functional-services';
 
-/*
- * Retry wrapper for async supertests, with a maximum number of retries
+/**
+ * Retry wrapper for async supertests, with a maximum number of retries.
+ * You can pass in a function that executes a supertest test, and make assertions
+ * on the response. If the test fails, it will retry the test the number of retries
+ * that are passed in.
+ * 
+ * Example usage:
+ * ```ts
+  const fleetResponse = await retry<InstallPackageResponse>({
+    test: async () => {
+      const testResponse = await supertest
+        .post(`/api/fleet/epm/packages/security_detection_engine`)
+        .set('kbn-xsrf', 'xxxx')
+        .set('elastic-api-version', '2023-10-31')
+        .type('application/json')
+        .send({ force: true })
+        .expect(200);
+      expect((testResponse.body as InstallPackageResponse).items).toBeDefined();
+      expect((testResponse.body as InstallPackageResponse).items.length).toBeGreaterThan(0);
+
+      return testResponse.body;
+    },
+    retryService,
+    retries: MAX_RETRIES,
+    timeout: ATTEMPT_TIMEOUT,
+  });
+ * ```
+ * @param test The function containing a test to run
+ * @param retryService The retry service to use
+ * @param retries The maximum number of retries
+ * @param timeout The timeout for each retry
+ * @param retryDelay The delay between each retry
+ * @returns The response from the test
  */
 export const retry = async <T>({
   test,
   retryService,
-  retries,
-  timeout,
+  retries = 2,
+  timeout = 30000,
+  retryDelay = 200,
 }: {
   test: () => Promise<T>;
   retryService: RetryService;
-  retries: number;
-  timeout: number;
-}): Promise<T | Error> => {
+  retries?: number;
+  timeout?: number;
+  retryDelay?: number;
+}): Promise<T> => {
   let retryAttempt = 0;
   const response = await retryService.tryForTime(
     timeout,
@@ -36,7 +69,7 @@ export const retry = async <T>({
       return test();
     },
     undefined,
-    200
+    retryDelay
   );
 
   // Now throw the error in order to fail the test.

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_and_timelines_status.ts

@@ -11,7 +11,7 @@ import {
 } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules';
 import type SuperTest from 'supertest';
 import type { Client } from '@elastic/elasticsearch';
-import { refreshSavedObjectIndices } from '../es_indices';
+import { refreshSavedObjectIndices } from '../../refresh_index';
 
 /**
  * (LEGACY)

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/get_prebuilt_rules_status.ts

@@ -11,7 +11,7 @@ import {
 } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules';
 import type SuperTest from 'supertest';
 import type { Client } from '@elastic/elasticsearch';
-import { refreshSavedObjectIndices } from '../es_indices';
+import { refreshSavedObjectIndices } from '../../refresh_index';
 
 /**
  * Helper to retrieve the prebuilt rules status

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_fleet_package_by_url.ts

@@ -10,7 +10,11 @@ import { InstallPackageResponse } from '@kbn/fleet-plugin/common/types';
 import { epmRouteService } from '@kbn/fleet-plugin/common';
 import { RetryService } from '@kbn/ftr-common-functional-services';
 import expect from 'expect';
-import { refreshSavedObjectIndices, retry } from '../..';
+import { retry } from '../../retry';
+import { refreshSavedObjectIndices } from '../../refresh_index';
+
+const MAX_RETRIES = 2;
+const ATTEMPT_TIMEOUT = 120000;
 
 /**
  * Installs latest available non-prerelease prebuilt rules package `security_detection_engine`.
@@ -41,13 +45,13 @@ export const installPrebuiltRulesPackageViaFleetAPI = async (
       return testResponse.body;
     },
     retryService,
-    retries: 2,
-    timeout: 120000,
+    retries: MAX_RETRIES,
+    timeout: ATTEMPT_TIMEOUT,
   });
 
   await refreshSavedObjectIndices(es);
 
-  return fleetResponse as InstallPackageResponse;
+  return fleetResponse;
 };
 /**
  * Installs prebuilt rules package `security_detection_engine`, passing in the version
@@ -80,8 +84,8 @@ export const installPrebuiltRulesPackageByVersion = async (
       return testResponse.body;
     },
     retryService,
-    retries: 2,
-    timeout: 120000,
+    retries: MAX_RETRIES,
+    timeout: ATTEMPT_TIMEOUT,
   });
 
   await refreshSavedObjectIndices(es);

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules.ts

@@ -12,7 +12,7 @@ import {
 } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules';
 import type { Client } from '@elastic/elasticsearch';
 import type SuperTest from 'supertest';
-import { refreshSavedObjectIndices } from '../es_indices';
+import { refreshSavedObjectIndices } from '../../refresh_index';
 
 /**
  * Installs available prebuilt rules in Kibana. Rules are

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_and_timelines.ts

@@ -11,7 +11,7 @@ import {
 } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules';
 import type { Client } from '@elastic/elasticsearch';
 import type SuperTest from 'supertest';
-import { refreshSavedObjectIndices } from '../es_indices';
+import { refreshSavedObjectIndices } from '../../refresh_index';
 
 /**
  * (LEGACY)

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/install_prebuilt_rules_fleet_package.ts

@@ -16,7 +16,10 @@ import type SuperTest from 'supertest';
 import { RetryService } from '@kbn/ftr-common-functional-services';
 import expect from 'expect';
 import { retry } from '../../retry';
-import { refreshSavedObjectIndices } from '../..';
+import { refreshSavedObjectIndices } from '../../refresh_index';
+
+const MAX_RETRIES = 2;
+const ATTEMPT_TIMEOUT = 120000;
 
 /**
  * Installs the `security_detection_engine` package via fleet API. This will
@@ -56,13 +59,13 @@ export const installPrebuiltRulesFleetPackage = async ({
         return testResponse.body;
       },
       retryService,
-      retries: 2,
-      timeout: 120000,
+      retries: MAX_RETRIES,
+      timeout: ATTEMPT_TIMEOUT,
     });
 
     await refreshSavedObjectIndices(es);
 
-    return response as InstallPackageResponse;
+    return response;
   } else {
     // Install the latest version
     const response = await retry<BulkInstallPackagesResponse>({
@@ -88,13 +91,13 @@ export const installPrebuiltRulesFleetPackage = async ({
         return body;
       },
       retryService,
-      retries: 2,
-      timeout: 120000,
+      retries: MAX_RETRIES,
+      timeout: ATTEMPT_TIMEOUT,
     });
 
     await refreshSavedObjectIndices(es);
 
-    return response as BulkInstallPackagesResponse;
+    return response;
   }
 };
 

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/prebuilt_rules/upgrade_prebuilt_rules.ts

@@ -12,7 +12,7 @@ import {
 } from '@kbn/security-solution-plugin/common/api/detection_engine/prebuilt_rules';
 import type { Client } from '@elastic/elasticsearch';
 import type SuperTest from 'supertest';
-import { refreshSavedObjectIndices } from '../es_indices';
+import { refreshSavedObjectIndices } from '../../refresh_index';
 
 /**
  * Upgrades available prebuilt rules in Kibana.