[TIP] Run e2e pipeline on CI (#144776)

## Summary

Last week, our team has dicovered that e2e tests are not executed on CI,
this PR is an attempt to fix that.

Threat intel pipeline should be run whenever the `threat_intelligence`
plugin source or related tests config has changed.

.buildkite/ftr_configs.yml

@@ -33,6 +33,8 @@ disabled:
   - x-pack/test/security_solution_cypress/upgrade_config.ts
   - x-pack/test/security_solution_cypress/visual_config.ts
   - x-pack/test/threat_intelligence_cypress/visual_config.ts
+  - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts
+  - x-pack/test/threat_intelligence_cypress/config.ts
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts

.buildkite/pipelines/pull_request/threat_intelligence.yml

@@ -0,0 +1,12 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
+    label: 'Threat Intelligence Tests'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -68,6 +68,17 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/threat_intelligence/,
+        /^x-pack\/test\/threat_intelligence_cypress/,
+        /^x-pack\/plugins\/security_solution\/public\/threat_intelligence/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/threat_intelligence.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^src\/plugins\/data/,

.buildkite/scripts/steps/functional/threat_intelligence.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+
+export JOB=kibana-threat-intelligence-chrome
+export CLI_NUMBER=${CLI_NUMBER:-$((BUILDKITE_PARALLEL_JOB+1))}
+export CLI_COUNT=${CLI_COUNT:-$BUILDKITE_PARALLEL_JOB_COUNT}
+
+echo "--- Threat Intelligence tests (Chrome)"
+
+node scripts/functional_tests \
+  --debug --bail \
+  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
+  --config x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts

x-pack/plugins/threat_intelligence/README.md

@@ -64,7 +64,9 @@ node scripts/generate_indicators.js
 
 see the file in order to adjust the amount of indicators generated. The default is one million.
 
-## Data for E2E tests
+## E2E
+
+### Data fixtures and loading process
 
 Use es_archives to export data for e2e testing purposes, like so:
 
@@ -76,6 +78,20 @@ These can be loaded at will with `x-pack/plugins/threat_intelligence/cypress/tas
 
 You can use this approach to load separate data dumps for every test case, to cover all critical scenarios.
 
+### Running locally
+
+`cd` into plugin root and execute `yarn cypress:open-as-ci`
+
+### CI Execution
+
+The entry point for PR testing is `.buildkite/pipelines/pull_request/threat_intelligence.yml` file, see that for details on
+how the test suite is executed & extra options regarding parallelism, retrying etc.
+
+E2E tests for this plugin will only be executed if any of the files changed within the PR matches dependency list here:
+`.buildkite/scripts/pipelines/pull_request/pipeline.ts`
+
+It is also possible to run all tests by attaching a PR flag: `ci:all-cypress-suites`.
+
 ## FAQ
 
 ### How is the Threat Intelligence code loaded in Kibana?

x-pack/plugins/threat_intelligence/cypress/e2e/query_bar.cy.ts

@@ -43,7 +43,7 @@ describe('Indicators', () => {
   });
 
   describe('Indicators query bar interaction', () => {
-    before(() => {
+    beforeEach(() => {
       cy.visit(THREAT_INTELLIGENCE);
 
       selectRange();
@@ -67,16 +67,30 @@ describe('Indicators', () => {
 
     it('should add filter to kql and filter in and out values when clicking in an indicators table cell', () => {
       cy.get(INDICATOR_TYPE_CELL).its('length').should('be.gte', 0);
-      cy.get(INDICATOR_TYPE_CELL).first().trigger('mouseover');
-      cy.get(INDICATORS_TABLE_CELL_FILTER_IN_BUTTON).should('exist').click();
+
+      cy.get(INDICATOR_TYPE_CELL)
+        .first()
+        .should('be.visible')
+        .trigger('mouseover')
+        .within((_cell) => {
+          cy.get(INDICATORS_TABLE_CELL_FILTER_IN_BUTTON).should('exist').click({
+            force: true,
+          });
+        });
+
       cy.get(KQL_FILTER).should('exist');
       cy.get(INDICATOR_TYPE_CELL).its('length').should('be.gte', 0);
     });
 
     it('should add negated filter and filter out and out values when clicking in an indicators table cell', () => {
       cy.get(INDICATOR_TYPE_CELL).its('length').should('be.gte', 0);
-      cy.get(INDICATOR_TYPE_CELL).first().trigger('mouseover');
-      cy.get(INDICATORS_TABLE_CELL_FILTER_OUT_BUTTON).should('exist').click();
+      cy.get(INDICATOR_TYPE_CELL)
+        .first()
+        .trigger('mouseover')
+        .within((_cell) => {
+          cy.get(INDICATORS_TABLE_CELL_FILTER_OUT_BUTTON).should('exist').click({ force: true });
+        });
+
       cy.get(KQL_FILTER).should('exist');
       cy.get(INDICATOR_TYPE_CELL).its('length').should('be.gte', 0);
     });

x-pack/plugins/threat_intelligence/cypress/e2e/timeline.cy.ts

@@ -9,18 +9,17 @@ import {
   BARCHART_POPOVER_BUTTON,
   BARCHART_TIMELINE_BUTTON,
   FLYOUT_CLOSE_BUTTON,
-  FLYOUT_OVERVIEW_TAB_BLOCKS_ITEM,
   FLYOUT_OVERVIEW_TAB_BLOCKS_TIMELINE_BUTTON,
   FLYOUT_OVERVIEW_TAB_TABLE_ROW_TIMELINE_BUTTON,
-  FLYOUT_TABLE_TAB_ROW_TIMELINE_BUTTON,
-  FLYOUT_TABS,
   INDICATOR_FLYOUT_INVESTIGATE_IN_TIMELINE_BUTTON,
   INDICATOR_TYPE_CELL,
   INDICATORS_TABLE_CELL_TIMELINE_BUTTON,
   INDICATORS_TABLE_INVESTIGATE_IN_TIMELINE_BUTTON_ICON,
   TIMELINE_DRAGGABLE_ITEM,
   TOGGLE_FLYOUT_BUTTON,
   UNTITLED_TIMELINE_BUTTON,
+  FLYOUT_TABLE_MORE_ACTIONS_BUTTON,
+  FLYOUT_BLOCK_MORE_ACTIONS_BUTTON,
 } from '../screens/indicators';
 import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver';
 import { login } from '../tasks/login';
@@ -41,7 +40,7 @@ describe('Indicators', () => {
   });
 
   describe('Indicators timeline interactions', () => {
-    before(() => {
+    beforeEach(() => {
       cy.visit(THREAT_INTELLIGENCE);
 
       selectRange();
@@ -56,13 +55,14 @@ describe('Indicators', () => {
 
     it('should add entry in timeline when clicking in an indicator table cell', () => {
       cy.get(INDICATOR_TYPE_CELL).first().trigger('mouseover');
-      cy.get(INDICATORS_TABLE_CELL_TIMELINE_BUTTON).should('exist').first().click();
+      cy.get(INDICATORS_TABLE_CELL_TIMELINE_BUTTON).should('exist').first().click({ force: true });
       cy.get(UNTITLED_TIMELINE_BUTTON).should('exist').first().click();
       cy.get(TIMELINE_DRAGGABLE_ITEM).should('exist');
     });
 
     it('should add entry in timeline when clicking in an indicator flyout overview tab table row', () => {
       cy.get(TOGGLE_FLYOUT_BUTTON).first().click({ force: true });
+      cy.get(FLYOUT_TABLE_MORE_ACTIONS_BUTTON).first().click({ force: true });
       cy.get(FLYOUT_OVERVIEW_TAB_TABLE_ROW_TIMELINE_BUTTON).should('exist').first().click();
       cy.get(FLYOUT_CLOSE_BUTTON).should('exist').click();
       cy.get(UNTITLED_TIMELINE_BUTTON).should('exist').first().click();
@@ -71,21 +71,8 @@ describe('Indicators', () => {
 
     it('should add entry in timeline when clicking in an indicator flyout overview block', () => {
       cy.get(TOGGLE_FLYOUT_BUTTON).first().click({ force: true });
-      cy.get(FLYOUT_OVERVIEW_TAB_BLOCKS_ITEM).first().trigger('mouseover');
-      cy.get(FLYOUT_OVERVIEW_TAB_BLOCKS_TIMELINE_BUTTON)
-        .should('exist')
-        .first()
-        .click({ force: true });
-      cy.get(FLYOUT_CLOSE_BUTTON).should('exist').click();
-      cy.get(UNTITLED_TIMELINE_BUTTON).should('exist').first().click();
-      cy.get(TIMELINE_DRAGGABLE_ITEM).should('exist');
-    });
-
-    it('should add entry in timeline when clicking in an indicator flyout table tab', () => {
-      cy.get(TOGGLE_FLYOUT_BUTTON).first().click({ force: true });
-      cy.get(FLYOUT_TABS).should('exist');
-      cy.get(`${FLYOUT_TABS} button:nth-child(2)`).click();
-      cy.get(FLYOUT_TABLE_TAB_ROW_TIMELINE_BUTTON).should('exist').first().click();
+      cy.get(FLYOUT_BLOCK_MORE_ACTIONS_BUTTON).first().click({ force: true });
+      cy.get(FLYOUT_OVERVIEW_TAB_BLOCKS_TIMELINE_BUTTON).should('exist').first().click();
       cy.get(FLYOUT_CLOSE_BUTTON).should('exist').click();
       cy.get(UNTITLED_TIMELINE_BUTTON).should('exist').first().click();
       cy.get(TIMELINE_DRAGGABLE_ITEM).should('exist');
@@ -100,7 +87,6 @@ describe('Indicators', () => {
     it('should investigate in timeline when clicking in an indicator flyout', () => {
       cy.get(TOGGLE_FLYOUT_BUTTON).first().click({ force: true });
       cy.get(INDICATOR_FLYOUT_INVESTIGATE_IN_TIMELINE_BUTTON).should('exist').first().click();
-      cy.get(FLYOUT_CLOSE_BUTTON).should('exist').click();
       cy.get(UNTITLED_TIMELINE_BUTTON).should('exist').first().click();
       cy.get(TIMELINE_DRAGGABLE_ITEM).should('exist');
     });

x-pack/plugins/threat_intelligence/cypress/reporter_config.json

@@ -0,0 +1,10 @@
+{
+  "reporterEnabled": "mochawesome, mocha-junit-reporter",
+  "reporterOptions": {
+    "html": false,
+    "json": true,
+    "mochaFile": "../../../target/kibana-threat-intelligence/cypress/results/TEST-threat-intelligence-cypress-[hash].xml",
+    "overwrite": false,
+    "reportDir": "../../../target/kibana-threat-intelligence/cypress/results"
+  }
+}

x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts

@@ -39,16 +39,17 @@ export const INDICATORS_TABLE_LAST_SEEN_COLUMN_HEADER = `[data-test-subj="dataGr
 
 export const TABLE_CONTROLS = '[data-test-sub="dataGridControls"]';
 
-export const INDICATOR_TYPE_CELL = '[data-gridcell-column-id="threat.indicator.type"]';
+export const INDICATOR_TYPE_CELL =
+  '[role="gridcell"][data-gridcell-column-id="threat.indicator.type"]';
 
 export const INDICATORS_TABLE_CELL_TIMELINE_BUTTON =
-  '[data-test-subj="tiIndicatorsTableCellTimelineButton"]';
+  '[data-test-subj="tiIndicatorsTableCellTimelineButton"] button';
 
 export const INDICATORS_TABLE_CELL_FILTER_IN_BUTTON =
-  '[data-test-subj="tiIndicatorsTableCellFilterInButton"]';
+  '[data-test-subj="tiIndicatorsTableCellFilterInButton"] button';
 
 export const INDICATORS_TABLE_CELL_FILTER_OUT_BUTTON =
-  '[data-test-subj="tiIndicatorsTableCellFilterOutButton"]';
+  '[data-test-subj="tiIndicatorsTableCellFilterOutButton"] button';
 
 export const INDICATORS_TABLE_INVESTIGATE_IN_TIMELINE_BUTTON_ICON =
   '[data-test-subj="tiIndicatorTableInvestigateInTimelineButtonIcon"]';
@@ -91,6 +92,12 @@ export const FLYOUT_OVERVIEW_TAB_BLOCKS_FILTER_IN_BUTTON =
 export const FLYOUT_OVERVIEW_TAB_BLOCKS_FILTER_OUT_BUTTON =
   '[data-test-subj="tiFlyoutOverviewHighLevelBlocksFilterOutButton"]';
 
+export const FLYOUT_TABLE_MORE_ACTIONS_BUTTON =
+  '[data-test-subj="tiFlyoutOverviewTableRowPopoverButton"] button';
+
+export const FLYOUT_BLOCK_MORE_ACTIONS_BUTTON =
+  '[data-test-subj="tiFlyoutOverviewHighLevelBlocksPopoverButton"] button';
+
 export const FLYOUT_TABLE_TAB_ROW_TIMELINE_BUTTON =
   '[data-test-subj="tiFlyoutTableTabRowTimelineButton"]';
 

x-pack/plugins/threat_intelligence/cypress/support/e2e.js

@@ -5,8 +5,6 @@
  * 2.0.
  */
 
-/* eslint-disable no-undef */
-
 // ***********************************************************
 // This example support/index.js is processed and
 // loaded automatically before your test files.
@@ -22,14 +20,10 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
-// Import commands.js using ES2015 syntax:
-import './commands';
-
 // Alternatively you can use CommonJS syntax:
 // require('./commands')
 
-Cypress.on('uncaught:exception', (err) => {
-  if (err.message.includes('ResizeObserver')) {
-    return false;
-  }
+// eslint-disable-next-line no-undef
+Cypress.on('uncaught:exception', () => {
+  return false;
 });

x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { FtrProviderContext } from './ftr_provider_context';
+
+import { ThreatIntelligenceCypressCliTestRunnerCI } from './runner';
+
+const cliNumber = parseInt(process.env.CLI_NUMBER ?? '1', 10);
+const cliCount = parseInt(process.env.CLI_COUNT ?? '1', 10);
+
+// eslint-disable-next-line import/no-default-export
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const securitySolutionCypressConfig = await readConfigFile(require.resolve('./config.ts'));
+  return {
+    ...securitySolutionCypressConfig.getAll(),
+
+    testRunner: (context: FtrProviderContext) =>
+      ThreatIntelligenceCypressCliTestRunnerCI(context, cliCount, cliNumber),
+  };
+}