[EDR Workflows] Verify flaky osquery tests (#169749)

elastic · Nov 1, 2023 · a15a948 · a15a948
1 parent 531b3ec
commit a15a948
Show file tree

Hide file tree

Showing 34 changed files with 239 additions and 152 deletions.
diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh
@@ -15,6 +15,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-osquery/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'

diff --git a/x-pack/plugins/osquery/cypress/cypress.config.ts b/x-pack/plugins/osquery/cypress/cypress.config.ts
@@ -11,6 +11,7 @@ import path from 'path';
 import { safeLoad as loadYaml } from 'js-yaml';
 import { readFileSync } from 'fs';
 
+import { getFailedSpecVideos } from './support/filter_videos';
 import type { YamlRoleDefinitions } from '../../../test_serverless/shared/lib';
 import { setupUserDataLoader } from '../../../test_serverless/functional/test_suites/security/cypress/support/setup_data_loader_tasks';
 const ROLES_YAML_FILE_PATH = path.join(
@@ -36,8 +37,9 @@ export default defineCypressConfig({
 
   screenshotsFolder: '../../../target/kibana-osquery/cypress/screenshots',
   trashAssetsBeforeRuns: false,
-  video: false,
+  video: true,
   videosFolder: '../../../target/kibana-osquery/cypress/videos',
+  videoCompression: 15,
   viewportHeight: 900,
   viewportWidth: 1440,
   experimentalStudio: true,
@@ -59,6 +61,7 @@ export default defineCypressConfig({
     numTestsKeptInMemory: 3,
     setupNodeEvents(on, config) {
       setupUserDataLoader(on, config, { roleDefinitions, additionalRoleName: 'viewer' });
+      on('after:spec', getFailedSpecVideos);
 
       return config;
     },

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/add_integration.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/add_integration.cy.ts
@@ -66,8 +66,7 @@ describe('ALL - Add Integration', { tags: ['@ess', '@serverless'] }, () => {
     }
   );
 
-  // FLAKY: https://github.com/elastic/kibana/issues/169702
-  describe.skip('Add and upgrade integration', { tags: ['@ess', '@serverless'] }, () => {
+  describe('Add and upgrade integration', { tags: ['@ess', '@serverless'] }, () => {
     const oldVersion = '0.7.4';
     const [integrationName, policyName] = generateRandomStringName(2);
     let policyId: string;

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_automated_action_results.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_automated_action_results.cy.ts
@@ -5,15 +5,19 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import { cleanupRule, loadRule } from '../../tasks/api_fixtures';
 import { checkActionItemsInResults, loadRuleAlerts } from '../../tasks/live_query';
 
 const UUID_REGEX = '[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}';
 
-// FLAKY: https://github.com/elastic/kibana/issues/169727
-describe.skip('Alert Flyout Automated Action Results', () => {
+describe('Alert Flyout Automated Action Results', () => {
   let ruleId: string;
 
+  before(() => {
+    initializeDataViews();
+  });
+
   beforeEach(() => {
     loadRule(true).then((data) => {
       ruleId = data.id;

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_cases.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_cases.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import { OSQUERY_FLYOUT_BODY_EDITOR } from '../../screens/live_query';
 import {
   cleanupCase,
@@ -29,6 +30,9 @@ describe('Alert Event Details - Cases', { tags: ['@ess', '@serverless'] }, () =>
   let packId: string;
   let packName: string;
   const packData = packFixture();
+  before(() => {
+    initializeDataViews();
+  });
 
   beforeEach(() => {
     loadPack(packData).then((data) => {

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_linked_apps.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_linked_apps.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import { cleanupRule, loadRule } from '../../tasks/api_fixtures';
 import { RESPONSE_ACTIONS_ITEM_0, RESPONSE_ACTIONS_ITEM_1 } from '../../tasks/response_actions';
 import {
@@ -25,7 +26,9 @@ describe(
   () => {
     let ruleId: string;
     let ruleName: string;
-
+    before(() => {
+      initializeDataViews();
+    });
     beforeEach(() => {
       loadRule().then((data) => {
         ruleId = data.id;

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_multiple_agents.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_multiple_agents.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import { cleanupRule, loadRule } from '../../tasks/api_fixtures';
 import {
   inputQuery,
@@ -14,8 +15,7 @@ import {
 } from '../../tasks/live_query';
 import { OSQUERY_FLYOUT_BODY_EDITOR } from '../../screens/live_query';
 
-// FLAKY: https://github.com/elastic/kibana/issues/170157
-describe.skip(
+describe(
   'Alert Event Details - dynamic params',
   {
     tags: ['@ess', '@serverless'],
@@ -25,6 +25,7 @@ describe.skip(
     let ruleName: string;
 
     before(() => {
+      initializeDataViews();
       loadRule(true).then((data) => {
         ruleId = data.id;
         ruleName = data.name;

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/alerts_response_actions_form.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import {
   cleanupPack,
   cleanupRule,
@@ -31,7 +32,9 @@ describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serve
   let packName: string;
   const packData = packFixture();
   const multiQueryPackData = multiQueryPackFixture();
-
+  before(() => {
+    initializeDataViews();
+  });
   beforeEach(() => {
     loadPack(packData).then((data) => {
       packId = data.saved_object_id;
@@ -55,6 +58,7 @@ describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serve
   it('adds response actions with osquery with proper validation and form values', () => {
     cy.visit('/app/security/rules');
     clickRuleName(ruleName);
+    cy.getBySel('globalLoadingIndicator').should('not.exist');
     cy.getBySel('editRuleSettingsLink').click();
     cy.getBySel('globalLoadingIndicator').should('not.exist');
     closeDateTabIfVisible();
@@ -81,20 +85,22 @@ describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serve
 
     cy.getBySel(OSQUERY_RESPONSE_ACTION_ADD_BUTTON).click();
 
-    cy.getBySel(RESPONSE_ACTIONS_ITEM_2).within(() => {
-      cy.contains('Query is a required field');
-      inputQuery('select * from uptime');
-      cy.contains('Query is a required field').should('not.exist');
-      cy.contains('Advanced').click();
-      typeInECSFieldInput('{downArrow}{enter}');
-      cy.getBySel('osqueryColumnValueSelect').type('days{downArrow}{enter}');
-      cy.wait(1000); // wait for the validation to trigger - cypress is way faster than users ;)
-    });
+    cy.getBySel(RESPONSE_ACTIONS_ITEM_2)
+      .within(() => {
+        cy.contains('Query is a required field');
+        inputQuery('select * from uptime');
+        cy.contains('Query is a required field').should('not.exist');
+        cy.contains('Advanced').click();
+        typeInECSFieldInput('{downArrow}{enter}');
+        cy.getBySel('osqueryColumnValueSelect').type('days{downArrow}{enter}');
+      })
+      .clickOutside();
 
     cy.getBySel('ruleEditSubmitButton').click();
     cy.contains(`${ruleName} was saved`).should('exist');
     closeToastIfVisible();
 
+    cy.getBySel('globalLoadingIndicator').should('not.exist');
     cy.getBySel('editRuleSettingsLink').click();
     cy.getBySel('globalLoadingIndicator').should('not.exist');
     cy.getBySel('edit-rule-actions-tab').click();
@@ -114,18 +120,21 @@ describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serve
       cy.contains('select * from uptime1');
       cy.getBySel('remove-response-action').click();
     });
-    cy.getBySel(RESPONSE_ACTIONS_ITEM_0).within(() => {
-      cy.contains('Search for a pack to run');
-      cy.contains('Pack is a required field');
-      cy.getBySel('comboBoxInput').type(`${packName}{downArrow}{enter}`);
-    });
+    cy.getBySel(RESPONSE_ACTIONS_ITEM_0)
+      .within(() => {
+        cy.contains('Search for a pack to run');
+        cy.contains('Pack is a required field');
+        cy.getBySel('comboBoxInput').type(`${packName}{downArrow}{enter}`);
+      })
+      .clickOutside();
     cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => {
       cy.contains('select * from uptime');
       cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}');
       cy.contains('Days of uptime');
     });
 
     cy.intercept('PUT', '/api/detection_engine/rules').as('saveRuleSingleQuery');
+
     cy.getBySel('ruleEditSubmitButton').click();
     cy.wait('@saveRuleSingleQuery').should(({ request }) => {
       const oneQuery = [
@@ -141,8 +150,10 @@ describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serve
     cy.contains(`${ruleName} was saved`).should('exist');
     closeToastIfVisible();
 
+    cy.getBySel('globalLoadingIndicator').should('not.exist');
     cy.getBySel('editRuleSettingsLink').click();
     cy.getBySel('globalLoadingIndicator').should('not.exist');
+
     cy.getBySel('edit-rule-actions-tab').click();
     cy.getBySel(RESPONSE_ACTIONS_ITEM_0)
       .within(() => {
@@ -153,12 +164,15 @@ describe('Alert Event Details - Response Actions Form', { tags: ['@ess', '@serve
       })
       .clickOutside();
 
-    cy.getBySel(RESPONSE_ACTIONS_ITEM_1).within(() => {
-      cy.contains('select * from uptime');
-      cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}');
-      cy.contains('Days of uptime');
-    });
+    cy.getBySel(RESPONSE_ACTIONS_ITEM_1)
+      .within(() => {
+        cy.contains('select * from uptime');
+        cy.contains('Custom key/value pairs. e.g. {"application":"foo-bar","env":"production"}');
+        cy.contains('Days of uptime');
+      })
+      .clickOutside();
     cy.intercept('PUT', '/api/detection_engine/rules').as('saveRuleMultiQuery');
+
     cy.contains('Save changes').click();
     cy.wait('@saveRuleMultiQuery').should(({ request }) => {
       const threeQueries = [

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import {
   addLiveQueryToCase,
   checkActionItemsInResults,
@@ -14,11 +15,11 @@ import { navigateTo } from '../../tasks/navigation';
 import { loadLiveQuery, loadCase, cleanupCase } from '../../tasks/api_fixtures';
 import { ServerlessRoleName } from '../../support/roles';
 
-// FLAKY: https://github.com/elastic/kibana/issues/169747
-describe.skip('Add to Cases', () => {
+describe('Add to Cases', () => {
   let liveQueryId: string;
   let liveQueryQuery: string;
   before(() => {
+    initializeDataViews();
     loadLiveQuery({
       agent_all: true,
       query: "SELECT * FROM os_version where name='Ubuntu';",
@@ -32,7 +33,7 @@ describe.skip('Add to Cases', () => {
   describe('observability', { tags: ['@ess'] }, () => {
     let caseId: string;
     let caseTitle: string;
-    before(() => {
+    beforeEach(() => {
       loadCase('observability').then((caseInfo) => {
         caseId = caseInfo.id;
         caseTitle = caseInfo.title;
@@ -41,7 +42,7 @@ describe.skip('Add to Cases', () => {
       navigateTo('/app/osquery');
     });
 
-    after(() => {
+    afterEach(() => {
       cleanupCase(caseId);
     });
 
@@ -64,7 +65,7 @@ describe.skip('Add to Cases', () => {
     let caseId: string;
     let caseTitle: string;
 
-    before(() => {
+    beforeEach(() => {
       loadCase('securitySolution').then((caseInfo) => {
         caseId = caseInfo.id;
         caseTitle = caseInfo.title;
@@ -73,7 +74,7 @@ describe.skip('Add to Cases', () => {
       navigateTo('/app/osquery');
     });
 
-    after(() => {
+    afterEach(() => {
       cleanupCase(caseId);
     });
 

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/custom_space.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/custom_space.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import { navigateTo } from '../../tasks/navigation';
 import {
   checkActionItemsInResults,
@@ -28,6 +29,7 @@ describe('ALL - Custom space', () => {
       let spaceId: string;
 
       before(() => {
+        initializeDataViews();
         cy.wrap(
           new Promise<string>((resolve) => {
             if (testSpace.name !== 'default') {

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/ecs_mappings.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/ecs_mappings.cy.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { initializeDataViews } from '../../tasks/login';
 import { getAdvancedButton } from '../../screens/integrations';
 import { navigateTo } from '../../tasks/navigation';
 import {
@@ -19,6 +20,10 @@ import {
 import { ServerlessRoleName } from '../../support/roles';
 
 describe('EcsMapping', { tags: ['@ess', '@serverless'] }, () => {
+  before(() => {
+    initializeDataViews();
+  });
+
   beforeEach(() => {
     cy.login(ServerlessRoleName.SOC_MANAGER);
   });

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/live_query.cy.ts
@@ -18,8 +18,7 @@ import { LIVE_QUERY_EDITOR } from '../../screens/live_query';
 import { getAdvancedButton } from '../../screens/integrations';
 import { ServerlessRoleName } from '../../support/roles';
 
-// FLAKY: https://github.com/elastic/kibana/issues/169725
-describe.skip('ALL - Live Query', { tags: ['@ess', '@serverless'] }, () => {
+describe('ALL - Live Query', { tags: ['@ess', '@serverless'] }, () => {
   beforeEach(() => {
     cy.login(ServerlessRoleName.SOC_MANAGER);
     navigateTo('/app/osquery');
@@ -87,6 +86,6 @@ describe.skip('ALL - Live Query', { tags: ['@ess', '@serverless'] }, () => {
 
     inputQuery('{selectall}{backspace}{selectall}{backspace}');
     // not sure if this is how it used to work when I implemented the functionality, but let's leave it like this for now
-    cy.get(LIVE_QUERY_EDITOR).invoke('height').should('be.gt', 200).and('be.lt', 380);
+    cy.get(LIVE_QUERY_EDITOR).invoke('height').should('be.gt', 200).and('be.lt', 400);
   });
 });
diff --git a/x-pack/plugins/osquery/cypress/e2e/all/metrics.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/metrics.cy.ts
@@ -11,9 +11,7 @@ import { loadSavedQuery, cleanupSavedQuery } from '../../tasks/api_fixtures';
 import { triggerLoadData } from '../../tasks/inventory';
 import { ServerlessRoleName } from '../../support/roles';
 
-// FLAKY: https://github.com/elastic/kibana/issues/169574
-// FLAKY: https://github.com/elastic/kibana/issues/169575
-describe.skip('ALL - Inventory', { tags: ['@ess'] }, () => {
+describe('ALL - Inventory', { tags: ['@ess'] }, () => {
   let savedQueryName: string;
   let savedQueryId: string;
 

diff --git a/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/packs_create_edit.cy.ts
@@ -15,7 +15,7 @@ import {
   findFormFieldByRowsLabelAndType,
   inputQuery,
 } from '../../tasks/live_query';
-import { activatePack, deactivatePack, preparePack } from '../../tasks/packs';
+import { changePackActiveStatus, preparePack } from '../../tasks/packs';
 import {
   closeModalIfVisible,
   closeToastIfVisible,
@@ -513,8 +513,8 @@ describe('Packs - Create and Edit', { tags: ['@ess', '@serverless'] }, () => {
 
     it('', () => {
       cy.contains('Packs').click();
-      deactivatePack(packName);
-      activatePack(packName);
+      changePackActiveStatus(packName);
+      changePackActiveStatus(packName);
     });
   });