[EDR Workflows][E2E] Recreate agent on createEndpointHost task fail

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -162,10 +162,11 @@ for (const testSuite of testSuites) {
           `Group configuration was not found in groups.json for the following cypress suite: {${suiteName}}.`
         );
       }
+      const agentQueue = suiteName.includes('defend_workflows') ? 'n2-4-virt' : 'n2-4-spot';
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: { queue: 'n2-4-spot' },
+        agents: { queue: agentQueue },
         depends_on: 'build',
         parallelism: testSuite.count,
         concurrency,

x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/automated_response_actions.cy.ts

@@ -20,8 +20,7 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host';
 import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data';
 import { enableAllPolicyProtections } from '../../tasks/endpoint_policy';
 
-// FLAKY: https://github.com/elastic/kibana/issues/168340
-describe.skip(
+describe(
   'Automated Response Actions',
   {
     tags: [
@@ -76,8 +75,7 @@ describe.skip(
       disableExpandableFlyoutAdvancedSettings();
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/168427
-    describe.skip('From alerts', () => {
+    describe('From alerts', () => {
       let ruleId: string;
       let ruleName: string;
 

x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint_list/endpoints.cy.ts

@@ -32,8 +32,7 @@ import { createEndpointHost } from '../../tasks/create_endpoint_host';
 import { deleteAllLoadedEndpointData } from '../../tasks/delete_all_endpoint_data';
 import { enableAllPolicyProtections } from '../../tasks/endpoint_policy';
 
-// FLAKY: https://github.com/elastic/kibana/issues/168284
-describe.skip('Endpoints page', { tags: ['@ess', '@serverless'] }, () => {
+describe('Endpoints page', { tags: ['@ess', '@serverless'] }, () => {
   let indexedPolicy: IndexedFleetEndpointPolicyResponse;
   let policy: PolicyData;
   let createdHost: CreateAndEnrollEndpointHostResponse;

x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts

@@ -332,16 +332,39 @@ export const dataLoadersForRealEndpoints = (
       options: Omit<CreateAndEnrollEndpointHostOptions, 'log' | 'kbnClient'>
     ): Promise<CreateAndEnrollEndpointHostResponse> => {
       const { kbnClient, log } = await stackServicesPromise;
-      return createAndEnrollEndpointHost({
-        useClosestVersionMatch: true,
-        ...options,
-        log,
-        kbnClient,
-      }).then((newHost) => {
-        return waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000).then(() => {
+
+      let retryAttempt = 0;
+      const attemptCreateEndpointHost = async (): Promise<CreateAndEnrollEndpointHostResponse> => {
+        try {
+          log.info(`Creating endpoint host, attempt ${retryAttempt}`);
+          const newHost = await createAndEnrollEndpointHost({
+            useClosestVersionMatch: true,
+            ...options,
+            log,
+            kbnClient,
+          });
+          await waitForEndpointToStreamData(kbnClient, newHost.agentId, 360000);
           return newHost;
-        });
-      });
+        } catch (err) {
+          log.info(`Caught error when setting up the agent: ${err}`);
+          if (retryAttempt === 0 && err.agentId) {
+            retryAttempt++;
+            await destroyEndpointHost(kbnClient, {
+              hostname: err.hostname || '', // No hostname in CI env for vagrant
+              agentId: err.agentId,
+            });
+            log.info(`Deleted endpoint host ${err.agentId} and retrying`);
+            return attemptCreateEndpointHost();
+          } else {
+            log.info(
+              `${retryAttempt} attempts of creating endpoint host failed, reason for the last failure was ${err}`
+            );
+            throw err;
+          }
+        }
+      };
+
+      return attemptCreateEndpointHost();
     },
 
     destroyEndpointHost: async (

x-pack/plugins/security_solution/public/management/cypress/tasks/create_endpoint_host.ts

@@ -17,6 +17,6 @@ export const createEndpointHost = (
     {
       agentPolicyId,
     },
-    { timeout: timeout ?? 900000 } // 15 minutes, since setup can take 10 minutes and more. Task will time out if is not resolved within this time.
+    { timeout: timeout ?? 30 * 60 * 1000 }
   );
 };

x-pack/plugins/security_solution/scripts/endpoint/common/endpoint_host_services.ts

@@ -234,7 +234,7 @@ const createMultipassVm = async ({
   };
 };
 
-const deleteMultipassVm = async (vmName: string): Promise<void> => {
+export const deleteMultipassVm = async (vmName: string): Promise<void> => {
   if (process.env.CI) {
     await execa.command(`vagrant destroy -f`, {
       env: {
@@ -339,7 +339,10 @@ const enrollHostWithFleet = async ({
     ]);
   }
   log.info(`Waiting for Agent to check-in with Fleet`);
-  const agent = await waitForHostToEnroll(kbnClient, vmName, 240000);
+
+  const agent = await waitForHostToEnroll(kbnClient, vmName, 8 * 60 * 1000);
+
+  log.info(`Agent enrolled with Fleet, status: `, agent.status);
 
   return {
     agentId: agent.id,

x-pack/plugins/security_solution/scripts/endpoint/common/fleet_services.ts

@@ -147,15 +147,19 @@ export const waitForHostToEnroll = async (
     return elapsedTime > timeoutMs;
   };
   let found: Agent | undefined;
+  let agentId: string | undefined;
 
   while (!found && !hasTimedOut()) {
     found = await retryOnError(
       async () =>
         fetchFleetAgents(kbnClient, {
           perPage: 1,
-          kuery: `(local_metadata.host.hostname.keyword : "${hostname}") and (status:online)`,
+          kuery: `(local_metadata.host.hostname.keyword : "${hostname}")`,
           showInactive: false,
-        }).then((response) => response.items[0]),
+        }).then((response) => {
+          agentId = response.items[0]?.id;
+          return response.items.filter((agent) => agent.status === 'online')[0];
+        }),
       RETRYABLE_TRANSIENT_ERRORS
     );
 
@@ -166,7 +170,14 @@ export const waitForHostToEnroll = async (
   }
 
   if (!found) {
-    throw new Error(`Timed out waiting for host [${hostname}] to show up in Fleet`);
+    throw Object.assign(
+      new Error(
+        `Timed out waiting for host [${hostname}] to show up in Fleet in ${
+          timeoutMs / 60 / 1000
+        } seconds`
+      ),
+      { agentId, hostname }
+    );
   }
 
   return found;