[Osquery] Switch Osquery Cypress to run in parallel (#158692)

## Summary Adjust Osquery Cypress tests to latest and greatest CI parallelization --------- Co-authored-by: konrad.szwarc <[email protected]>
elastic · Jun 5, 2023 · 0e3e902 · 0e3e902
1 parent d20fea8
commit 0e3e902
Show file tree

Hide file tree

Showing 11 changed files with 63 additions and 105 deletions.
diff --git a/.buildkite/pipelines/pull_request/osquery_cypress.yml b/.buildkite/pipelines/pull_request/osquery_cypress.yml
@@ -4,7 +4,8 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 50
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/scripts/steps/functional/osquery_cypress.sh b/.buildkite/scripts/steps/functional/osquery_cypress.sh
@@ -7,11 +7,12 @@ source .buildkite/scripts/common/util.sh
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js
 
+Xvfb :99 -screen 0 1600x1200x24 &
+
+export DISPLAY=:99
+
 export JOB=kibana-osquery-cypress
 
 echo "--- Osquery Cypress tests"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --config x-pack/test/osquery_cypress/cli_config.ts
-
+yarn --cwd x-pack/plugins/osquery cypress:run
diff --git a/x-pack/plugins/osquery/cypress.config.ts b/x-pack/plugins/osquery/cypress.config.ts
@@ -32,6 +32,7 @@ export default defineCypressConfig({
   },
 
   e2e: {
+    specPattern: './cypress/e2e/**/*.cy.ts',
     baseUrl: 'http://localhost:5601',
     experimentalRunAllSpecs: true,
     experimentalMemoryManagement: true,

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts.cy.ts
@@ -105,7 +105,7 @@ describe('Alert Event Details', () => {
     const packData = packFixture();
     const multiQueryPackData = multiQueryPackFixture();
 
-    before(() => {
+    beforeEach(() => {
       loadPack(packData).then((data) => {
         packId = data.saved_object_id;
         packName = data.name;
@@ -119,7 +119,7 @@ describe('Alert Event Details', () => {
         ruleName = data.name;
       });
     });
-    after(() => {
+    afterEach(() => {
       cleanupPack(packId);
       cleanupPack(multiQueryPackId);
       cleanupRule(ruleId);

diff --git a/x-pack/plugins/osquery/package.json b/x-pack/plugins/osquery/package.json
@@ -5,10 +5,8 @@
   "private": true,
   "license": "Elastic License 2.0",
   "scripts": {
-    "cypress:open": "../../../node_modules/.bin/cypress open --config-file ./cypress.config.ts",
-    "cypress:open-as-ci": "node ../../../scripts/functional_tests --config ../../test/osquery_cypress/visual_config.ts",
-    "cypress:run": "../../../node_modules/.bin/cypress run --config-file ./cypress.config.ts",
-    "cypress:run-as-ci": "node ../../../scripts/functional_tests --config ../../test/osquery_cypress/cli_config.ts",
+    "cypress:open": "node ../security_solution/scripts/start_cypress_parallel open --config-file ../osquery/cypress.config.ts --ftr-config-file ../../../../../../x-pack/test/osquery_cypress/cli_config",
+    "cypress:run": "node ../security_solution/scripts/start_cypress_parallel run --config-file ../osquery/cypress.config.ts --ftr-config-file ../../../../../../x-pack/test/osquery_cypress/cli_config --concurrency 1",
     "nyc": "../../../node_modules/.bin/nyc report --reporter=text-summary"
   }
 }
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts b/x-pack/plugins/security_solution/scripts/endpoint/endpoint_agent_runner/fleet_server.ts
@@ -207,7 +207,7 @@ export const startFleetServerWithDocker = async ({
   log.indent(4);
 
   const esURL = new URL(elasticUrl);
-  const containerName = `dev-fleet-server.${esURL.hostname}`;
+  const containerName = `dev-fleet-server.${fleetServerPort}`;
   let esUrlWithRealIp: string = elasticUrl;
 
   if (isElasticOnLocalhost) {

diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts
@@ -63,11 +63,9 @@ export const cli = () => {
 
       const cypressConfigFile = await import(require.resolve(`../../${argv.configFile}`));
       const spec: string | undefined = argv?.spec as string;
-      const files = retrieveIntegrations(
-        spec ? (spec.includes(',') ? spec.split(',') : [spec]) : cypressConfigFile?.e2e?.specPattern
-      );
+      const files = retrieveIntegrations(spec ? [spec] : cypressConfigFile?.e2e?.specPattern);
 
-      if (!files.length) {
+      if (!files?.length) {
         throw new Error('No files found');
       }
 
@@ -219,9 +217,9 @@ export const cli = () => {
                   kibana: {
                     port: kibanaPort,
                   },
-                  // fleetserver: {
-                  //   port: fleetServerPort,
-                  // },
+                  fleetserver: {
+                    port: fleetServerPort,
+                  },
                 },
                 kbnTestServer: {
                   serverArgs: [
@@ -290,6 +288,7 @@ export const cli = () => {
 
             const options = {
               installDir: process.env.KIBANA_INSTALL_DIR,
+              ci: process.env.CI,
             };
 
             const shutdownEs = await pRetry(
@@ -308,9 +307,10 @@ export const cli = () => {
               procs,
               config,
               installDir: options?.installDir,
-              extraKbnOpts: options?.installDir
-                ? []
-                : ['--dev', '--no-dev-config', '--no-dev-credentials'],
+              extraKbnOpts:
+                options?.installDir || options?.ci || !isOpen
+                  ? []
+                  : ['--dev', '--no-dev-config', '--no-dev-credentials'],
               onEarlyExit,
             });
 
@@ -322,7 +322,9 @@ export const cli = () => {
               EsVersion.getDefault()
             );
 
-            const customEnv = await functionalTestRunner.run(abortCtrl.signal);
+            const customEnv = await pRetry(() => functionalTestRunner.run(abortCtrl.signal), {
+              retries: 1,
+            });
 
             if (isOpen) {
               await cypress.open({
@@ -363,7 +365,13 @@ export const cli = () => {
           });
           return result;
         },
-        { concurrency: !isOpen ? 3 : 1 }
+        {
+          concurrency: (argv.concurrency as number | undefined)
+            ? (argv.concurrency as number)
+            : !isOpen
+            ? 3
+            : 1,
+        }
       ).then((results) => {
         renderSummaryTable(results as CypressCommandLine.CypressRunResult[]);
         const hasFailedTests = _.some(

diff --git a/x-pack/test/osquery_cypress/agent.ts b/x-pack/test/osquery_cypress/agent.ts
@@ -19,11 +19,13 @@ import { addIntegrationToAgentPolicy } from './utils';
 export class AgentManager extends Manager {
   private log: ToolingLog;
   private kbnClient: KbnClient;
+  private fleetServerPort: string;
   private agentContainerId?: string;
 
-  constructor(kbnClient: KbnClient, log: ToolingLog) {
+  constructor(kbnClient: KbnClient, fleetServerPort: string, log: ToolingLog) {
     super();
     this.log = log;
+    this.fleetServerPort = fleetServerPort;
     this.kbnClient = kbnClient;
   }
 
@@ -71,7 +73,7 @@ export class AgentManager extends Manager {
       '--env',
       'FLEET_ENROLL=1',
       '--env',
-      `FLEET_URL=https://host.docker.internal:8220`,
+      `FLEET_URL=https://host.docker.internal:${this.fleetServerPort}`,
       '--env',
       `FLEET_ENROLLMENT_TOKEN=${policy.api_key}`,
       '--env',

diff --git a/x-pack/test/osquery_cypress/cli_config.ts b/x-pack/test/osquery_cypress/cli_config.ts
@@ -7,13 +7,13 @@
 
 import { FtrConfigProviderContext } from '@kbn/test';
 
-import { OsqueryCypressCliTestRunner } from './runner';
+import { startOsqueryCypress } from './runner';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const osqueryCypressConfig = await readConfigFile(require.resolve('./config.ts'));
   return {
     ...osqueryCypressConfig.getAll(),
 
-    testRunner: OsqueryCypressCliTestRunner,
+    testRunner: startOsqueryCypress,
   };
 }
diff --git a/x-pack/test/osquery_cypress/runner.ts b/x-pack/test/osquery_cypress/runner.ts
@@ -5,22 +5,16 @@
  * 2.0.
  */
 
-import { resolve } from 'path';
 import Url from 'url';
 
-import { withProcRunner } from '@kbn/dev-proc-runner';
-
 import { startRuntimeServices } from '@kbn/security-solution-plugin/scripts/endpoint/endpoint_agent_runner/runtime';
 import { FtrProviderContext } from './ftr_provider_context';
 
 import { AgentManager } from './agent';
 import { FleetManager } from './fleet_server';
 import { getLatestAvailableAgentVersion } from '../defend_workflows_cypress/utils';
 
-async function withFleetAgent(
-  { getService }: FtrProviderContext,
-  runner: (runnerEnv: Record<string, string>) => Promise<void>
-) {
+async function setupFleetAgent({ getService }: FtrProviderContext) {
   const log = getService('log');
   const config = getService('config');
   const kbnClient = getService('kibanaServer');
@@ -46,62 +40,34 @@ async function withFleetAgent(
   });
 
   const fleetManager = new FleetManager(kbnClient, log);
-  const agentManager = new AgentManager(kbnClient, log);
+  const agentManager = new AgentManager(kbnClient, config.get('servers.fleetserver.port'), log);
 
   await fleetManager.setup();
   await agentManager.setup();
-  try {
-    await runner({});
-  } finally {
-    agentManager.cleanup();
-    fleetManager.cleanup();
-  }
 }
 
-export async function OsqueryCypressCliTestRunner(context: FtrProviderContext) {
-  await startOsqueryCypress(context, 'run');
-}
+export async function startOsqueryCypress(context: FtrProviderContext) {
+  const config = context.getService('config');
 
-export async function OsqueryCypressVisualTestRunner(context: FtrProviderContext) {
-  await startOsqueryCypress(context, 'open');
-}
+  await setupFleetAgent(context);
 
-function startOsqueryCypress(context: FtrProviderContext, cypressCommand: string) {
-  const log = context.getService('log');
-  const config = context.getService('config');
-  return withFleetAgent(context, (runnerEnv) =>
-    withProcRunner(log, async (procs) => {
-      await procs.run('cypress', {
-        cmd: 'yarn',
-        args: [`cypress:${cypressCommand}`],
-        cwd: resolve(__dirname, '../../plugins/osquery'),
-        env: {
-          FORCE_COLOR: '1',
-          // eslint-disable-next-line @typescript-eslint/naming-convention
-          CYPRESS_baseUrl: Url.format({
-            protocol: config.get('servers.kibana.protocol'),
-            hostname: config.get('servers.kibana.hostname'),
-            port: config.get('servers.kibana.port'),
-          }),
-          // eslint-disable-next-line @typescript-eslint/naming-convention
-          CYPRESS_protocol: config.get('servers.kibana.protocol'),
-          // eslint-disable-next-line @typescript-eslint/naming-convention
-          CYPRESS_hostname: config.get('servers.kibana.hostname'),
-          // eslint-disable-next-line @typescript-eslint/naming-convention
-          CYPRESS_configport: config.get('servers.kibana.port'),
-          CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
-          CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.kibana.username'),
-          CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.kibana.password'),
-          CYPRESS_KIBANA_URL: Url.format({
-            protocol: config.get('servers.kibana.protocol'),
-            hostname: config.get('servers.kibana.hostname'),
-            port: config.get('servers.kibana.port'),
-          }),
-          ...runnerEnv,
-          ...process.env,
-        },
-        wait: true,
-      });
-    })
-  );
+  return {
+    FORCE_COLOR: '1',
+    baseUrl: Url.format({
+      protocol: config.get('servers.kibana.protocol'),
+      hostname: config.get('servers.kibana.hostname'),
+      port: config.get('servers.kibana.port'),
+    }),
+    protocol: config.get('servers.kibana.protocol'),
+    hostname: config.get('servers.kibana.hostname'),
+    configport: config.get('servers.kibana.port'),
+    ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
+    ELASTICSEARCH_USERNAME: config.get('servers.kibana.username'),
+    ELASTICSEARCH_PASSWORD: config.get('servers.kibana.password'),
+    KIBANA_URL: Url.format({
+      protocol: config.get('servers.kibana.protocol'),
+      hostname: config.get('servers.kibana.hostname'),
+      port: config.get('servers.kibana.port'),
+    }),
+  };
 }
diff --git a/x-pack/test/osquery_cypress/visual_config.ts b/x-pack/test/osquery_cypress/visual_config.ts