Re-enable and fix APM E2E tests (elastic#114831)

* Re-enable previously disabled APM E2E tests. * Round to the nearest second in `getComparisonTypes` to avoid cases where a millisecond difference can change which results get shown. * Simplify error count alert tests to test the "happy path" (elastic#79284 exists in order to expand to more tests for rule editing and creation.) * Wait for alert list API request to complete before clicking "Create rule" button when running the test to create a rule from the Stack Management UI. I ran the e2e tests 100 times locally with no failures so I'm confident the flakiness has been addressed. Fixes elastic#114419. Fixes elastic#109205.
kibanamachine · Oct 15, 2021 · 41ce557 · 41ce557
1 parent b25dd4e
commit 41ce557
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 66 deletions.
diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.js b/.buildkite/scripts/pipelines/pull_request/pipeline.js
@@ -55,24 +55,23 @@ const uploadPipeline = (pipelineContent) => {
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/base.yml', false));
 
     if (
-      await doAnyChangesMatch([
+      (await doAnyChangesMatch([
         /^x-pack\/plugins\/security_solution/,
         /^x-pack\/test\/security_solution_cypress/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/sections\/action_connector_form/,
         /^x-pack\/plugins\/triggers_actions_ui\/public\/application\/context\/actions_connectors_context\.tsx/,
-      ]) || process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+      ])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution.yml'));
     }
 
-    // Disabled for now, these are failing/disabled in Jenkins currently as well
-    // if (
-    //   await doAnyChangesMatch([
-    //     /^x-pack\/plugins\/apm/,
-    //   ]) || process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
-    // ) {
-    //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
-    // }
+    if (
+      (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
+      process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
+    }
 
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 

diff --git a/vars/tasks.groovy b/vars/tasks.groovy
@@ -146,14 +146,13 @@ def functionalXpack(Map params = [:]) {
       }
     }
 
-    //temporarily disable apm e2e test since it's breaking.
-    // whenChanged([
-    //   'x-pack/plugins/apm/',
-    // ]) {
-    //   if (githubPr.isPr()) {
-    //     task(kibanaPipeline.functionalTestProcess('xpack-APMCypress', './test/scripts/jenkins_apm_cypress.sh'))
-    //   }
-    // }
+    whenChanged([
+      'x-pack/plugins/apm/',
+    ]) {
+      if (githubPr.isPr()) {
+        task(kibanaPipeline.functionalTestProcess('xpack-APMCypress', './test/scripts/jenkins_apm_cypress.sh'))
+      }
+    }
 
     whenChanged([
       'x-pack/plugins/uptime/',

diff --git a/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/rules/error_count.spec.ts b/x-pack/plugins/apm/ftr_e2e/cypress/integration/power_user/rules/error_count.spec.ts
@@ -5,81 +5,88 @@
  * 2.0.
  */
 
+function deleteAllRules() {
+  cy.request({
+    log: false,
+    method: 'GET',
+    url: '/api/alerting/rules/_find',
+  }).then(({ body }) => {
+    if (body.data.length > 0) {
+      cy.log(`Deleting rules`);
+    }
+
+    body.data.map(({ id }: { id: string }) => {
+      cy.request({
+        headers: { 'kbn-xsrf': 'true' },
+        log: false,
+        method: 'DELETE',
+        url: `/api/alerting/rule/${id}`,
+      });
+    });
+  });
+}
+
 describe('Rules', () => {
   describe('Error count', () => {
     const ruleName = 'Error count threshold';
     const comboBoxInputSelector =
       '.euiPopover__panel-isOpen [data-test-subj=comboBoxSearchInput]';
     const confirmModalButtonSelector =
       '.euiModal button[data-test-subj=confirmModalConfirmButton]';
-    const deleteButtonSelector =
-      '[data-test-subj=deleteActionHoverButton]:first';
-    const editButtonSelector = '[data-test-subj=editActionHoverButton]:first';
 
     describe('when created from APM', () => {
       describe('when created from Service Inventory', () => {
         before(() => {
           cy.loginAsPowerUser();
+          deleteAllRules();
         });
 
-        it('creates and updates a rule', () => {
+        after(() => {
+          deleteAllRules();
+        });
+
+        it('creates a rule', () => {
           // Create a rule in APM
           cy.visit('/app/apm/services');
           cy.contains('Alerts and rules').click();
           cy.contains('Error count').click();
           cy.contains('Create threshold rule').click();
 
-          // Change the environment to "testing"
-          cy.contains('Environment All').click();
-          cy.get(comboBoxInputSelector).type('testing{enter}');
-
           // Save, with no actions
           cy.contains('button:not(:disabled)', 'Save').click();
           cy.get(confirmModalButtonSelector).click();
 
           cy.contains(`Created rule "${ruleName}`);
-
-          // Go to Stack Management
-          cy.contains('Alerts and rules').click();
-          cy.contains('Manage rules').click();
-
-          // Edit the rule, changing the environment to "All"
-          cy.get(editButtonSelector).click();
-          cy.contains('Environment testing').click();
-          cy.get(comboBoxInputSelector).type('All{enter}');
-          cy.contains('button:not(:disabled)', 'Save').click();
-
-          cy.contains(`Updated '${ruleName}'`);
-
-          // Wait for the table to be ready for next edit click
-          cy.get('.euiBasicTable').not('.euiBasicTable-loading');
-
-          // Ensure the rule now shows "All" for the environment
-          cy.get(editButtonSelector).click();
-          cy.contains('Environment All');
-          cy.contains('button', 'Cancel').click();
-
-          // Delete the rule
-          cy.get(deleteButtonSelector).click();
-          cy.get(confirmModalButtonSelector).click();
-
-          // Ensure the table is empty
-          cy.contains('Create your first rule');
         });
       });
     });
 
     describe('when created from Stack management', () => {
       before(() => {
         cy.loginAsPowerUser();
+        deleteAllRules();
+        cy.intercept(
+          'GET',
+          '/api/alerting/rules/_find?page=1&per_page=10&default_search_operator=AND&sort_field=name&sort_order=asc'
+        ).as('list rules API call');
+      });
+
+      after(() => {
+        deleteAllRules();
       });
 
       it('creates a rule', () => {
         // Go to stack management
         cy.visit('/app/management/insightsAndAlerting/triggersActions/rules');
 
+        // Wait for this call to finish so the create rule button does not disappear.
+        // The timeout is set high because at this point we're also waiting for the
+        // full page load.
+        cy.wait('@list rules API call', { timeout: 30000 });
+
         // Create a rule
         cy.contains('button', 'Create rule').click();
+
         cy.get('[name=name]').type(ruleName);
         cy.contains('.euiFlyout button', ruleName).click();
 
@@ -92,16 +99,6 @@ describe('Rules', () => {
         cy.get(confirmModalButtonSelector).click();
 
         cy.contains(`Created rule "${ruleName}`);
-
-        // Wait for the table to be ready for next delete click
-        cy.get('.euiBasicTable').not('.euiBasicTable-loading');
-
-        // Delete the rule
-        cy.get(deleteButtonSelector).click();
-        cy.get(confirmModalButtonSelector).click();
-
-        // Ensure the table is empty
-        cy.contains('Create your first rule');
       });
     });
   });

diff --git a/x-pack/plugins/apm/ftr_e2e/cypress/support/commands.ts b/x-pack/plugins/apm/ftr_e2e/cypress/support/commands.ts
@@ -21,6 +21,7 @@ Cypress.Commands.add(
     cy.log(`Logging in as ${username}`);
     const kibanaUrl = Cypress.env('KIBANA_URL');
     cy.request({
+      log: false,
       method: 'POST',
       url: `${kibanaUrl}/internal/security/login`,
       body: {

diff --git a/x-pack/plugins/apm/public/components/shared/time_comparison/get_comparison_types.ts b/x-pack/plugins/apm/public/components/shared/time_comparison/get_comparison_types.ts
@@ -16,14 +16,14 @@ export function getComparisonTypes({
   start?: string;
   end?: string;
 }) {
-  const momentStart = moment(start);
-  const momentEnd = moment(end);
+  const momentStart = moment(start).startOf('second');
+  const momentEnd = moment(end).startOf('second');
 
   const dateDiff = getDateDifference({
     start: momentStart,
     end: momentEnd,
-    unitOfTime: 'days',
     precise: true,
+    unitOfTime: 'days',
   });
 
   // Less than or equals to one day

diff --git a/x-pack/plugins/apm/public/components/shared/time_comparison/index.test.tsx b/x-pack/plugins/apm/public/components/shared/time_comparison/index.test.tsx
@@ -96,6 +96,19 @@ describe('TimeComparison', () => {
         TimeRangeComparisonType.WeekBefore.valueOf(),
       ]);
     });
+
+    it('shows week and day before when 24 hours is selected but milliseconds are different', () => {
+      expect(
+        getComparisonTypes({
+          start: '2021-10-15T00:52:59.554Z',
+          end: '2021-10-14T00:52:59.553Z',
+        })
+      ).toEqual([
+        TimeRangeComparisonType.DayBefore.valueOf(),
+        TimeRangeComparisonType.WeekBefore.valueOf(),
+      ]);
+    });
+
     it('shows week before when 25 hours is selected', () => {
       expect(
         getComparisonTypes({