[Rollups] Fix functional tests (#186103)

Fixes #183975 Fixes #183976 Fixes #183925 Fixes #183926 Fixes #183927 Fixes #104569 Fixes #56816 Fixes #168267 ## Summary This PR fixes the failing Rollup functional tests. The reason for the failures is that, since 8.15, Es doesn't allow creating a new rollup job if there is no existing rollup usage in the cluster, where rollup usage is considered rollup jobs or rollup indices. To fix the tests, we create a mock rollup index which essentially is an index that has a `_meta` mappings property with rollup information. Flaky test runner build x200: https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6316
elastic · Jun 13, 2024 · b44e778 · b44e778
1 parent 350b34b
commit b44e778
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 16 deletions.
diff --git a/x-pack/test/functional/apps/rollup_job/hybrid_index_pattern.js b/x-pack/test/functional/apps/rollup_job/hybrid_index_pattern.js
@@ -8,6 +8,7 @@
 import datemath from '@kbn/datemath';
 import expect from '@kbn/expect';
 import mockRolledUpData, { mockIndices } from './hybrid_index_helper';
+import { MOCK_ROLLUP_INDEX_NAME, createMockRollupIndex } from './test_helpers';
 
 export default function ({ getService, getPageObjects }) {
   const es = getService('es');
@@ -17,9 +18,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'settings']);
   const esDeleteAllIndices = getService('esDeleteAllIndices');
 
-  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/183975
-  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/183976
-  describe.skip('hybrid index pattern', function () {
+  describe('hybrid index pattern', function () {
     //Since rollups can only be created once with the same name (even if you delete it),
     //we add the Date.now() to avoid name collision if you run the tests locally back to back.
     const rollupJobName = `hybrid-index-pattern-test-rollup-job-${Date.now()}`;
@@ -43,6 +42,10 @@ export default function ({ getService, getPageObjects }) {
       await kibanaServer.uiSettings.replace({
         defaultIndex: 'rollup',
       });
+
+      // From 8.15, Es only allows creating a new rollup job when there is existing rollup usage in the cluster
+      // We will simulate rollup usage by creating a mock-up rollup index
+      await createMockRollupIndex(es);
     });
 
     it('create hybrid index pattern', async () => {
@@ -107,7 +110,7 @@ export default function ({ getService, getPageObjects }) {
       // ensure all fields are available
       await PageObjects.settings.clickIndexPatternByName(rollupIndexPatternName);
       const fields = await PageObjects.settings.getFieldNames();
-      expect(fields).to.eql(['@timestamp', '_id', '_index', '_score', '_source']);
+      expect(fields).to.eql(['@timestamp', '_id', '_ignored', '_index', '_score', '_source']);
     });
 
     after(async () => {
@@ -118,6 +121,7 @@ export default function ({ getService, getPageObjects }) {
         rollupTargetIndexName,
         `${regularIndexPrefix}*`,
         `${rollupSourceIndexPrefix}*`,
+        MOCK_ROLLUP_INDEX_NAME,
       ]);
       await kibanaServer.savedObjects.cleanStandardList();
     });

diff --git a/x-pack/test/functional/apps/rollup_job/rollup_jobs.js b/x-pack/test/functional/apps/rollup_job/rollup_jobs.js
@@ -8,7 +8,7 @@
 import datemath from '@kbn/datemath';
 import expect from '@kbn/expect';
 import { mockIndices } from './hybrid_index_helper';
-// import { FtrProviderContext } from '../../ftr_provider_context';
+import { MOCK_ROLLUP_INDEX_NAME, createMockRollupIndex } from './test_helpers';
 
 export default function ({ getService, getPageObjects }) {
   const config = getService('config');
@@ -23,12 +23,7 @@ export default function ({ getService, getPageObjects }) {
     remoteEs = getService('remoteEs');
   }
 
-  // FLAKY: https://github.com/elastic/kibana/issues/183925
-  // FLAKY: https://github.com/elastic/kibana/issues/183926
-  // FLAKY: https://github.com/elastic/kibana/issues/183927
-  // FLAKY: https://github.com/elastic/kibana/issues/183928
-  // FLAKY: https://github.com/elastic/kibana/issues/104569
-  describe.skip('rollup job', function () {
+  describe('rollup job', function () {
     // Since rollups can only be created once with the same name (even if you delete it),
     // we add the Date.now() to avoid name collision.
     const rollupJobName = 'rollup-to-be-' + Date.now();
@@ -52,6 +47,10 @@ export default function ({ getService, getPageObjects }) {
       // await security.testUser.setRoles(['manage_rollups_role', 'global_ccr_role']);
       await security.testUser.setRoles(['superuser']);
       await PageObjects.common.navigateToApp('rollupJob');
+
+      // From 8.15, Es only allows creating a new rollup job when there is existing rollup usage in the cluster
+      // We will simulate rollup usage by creating a mock-up rollup index
+      await createMockRollupIndex(es);
     });
 
     it('create new rollup job', async () => {
@@ -88,7 +87,7 @@ export default function ({ getService, getPageObjects }) {
       });
 
       // Delete all data indices that were created.
-      await esDeleteAllIndices([targetIndexName], false);
+      await esDeleteAllIndices([targetIndexName, MOCK_ROLLUP_INDEX_NAME], false);
       if (isRunningCcs) {
         await esDeleteAllIndices([indexPatternToUse], true);
       } else {

diff --git a/x-pack/test/functional/apps/rollup_job/test_helpers.ts b/x-pack/test/functional/apps/rollup_job/test_helpers.ts
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import Client from '@elastic/elasticsearch/lib/client';
+
+export const MOCK_ROLLUP_INDEX_NAME = 'mock-rollup-index';
+
+export const createMockRollupIndex = async (es: Client) => {
+  await es.indices.create({
+    index: MOCK_ROLLUP_INDEX_NAME,
+    mappings: {
+      _meta: {
+        _rollup: {
+          logs_job: {
+            id: 'mockRollupJob',
+            index_pattern: MOCK_ROLLUP_INDEX_NAME,
+            rollup_index: 'rollup_index',
+            cron: '0 0 0 ? * 7',
+            page_size: 1000,
+            groups: {
+              date_histogram: {
+                interval: '24h',
+                delay: '1d',
+                time_zone: 'UTC',
+                field: 'testCreatedField',
+              },
+              terms: {
+                fields: ['testTotalField', 'testTagField'],
+              },
+              histogram: {
+                interval: '7',
+                fields: ['testTotalField'],
+              },
+            },
+          },
+        },
+        'rollup-version': '',
+      },
+    },
+  });
+};
diff --git a/x-pack/test/functional/apps/rollup_job/tsvb.js b/x-pack/test/functional/apps/rollup_job/tsvb.js
@@ -7,6 +7,7 @@
 
 import expect from '@kbn/expect';
 import mockRolledUpData from './hybrid_index_helper';
+import { MOCK_ROLLUP_INDEX_NAME, createMockRollupIndex } from './test_helpers';
 
 export default function ({ getService, getPageObjects }) {
   const es = getService('es');
@@ -24,9 +25,7 @@ export default function ({ getService, getPageObjects }) {
   const fromTime = 'Oct 15, 2019 @ 00:00:01.000';
   const toTime = 'Oct 15, 2019 @ 19:31:44.000';
 
-  // FLAKY: https://github.com/elastic/kibana/issues/56816
-  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/168267
-  describe.skip('tsvb integration', function () {
+  describe('tsvb integration', function () {
     //Since rollups can only be created once with the same name (even if you delete it),
     //we add the Date.now() to avoid name collision if you run the tests locally back to back.
     const rollupJobName = `tsvb-test-rollup-job-${Date.now()}`;
@@ -49,6 +48,10 @@ export default function ({ getService, getPageObjects }) {
         'metrics:allowStringIndices': true,
         'timepicker:timeDefaults': `{ "from": "${fromTime}", "to": "${toTime}"}`,
       });
+
+      // From 8.15, Es only allows creating a new rollup job when there is existing rollup usage in the cluster
+      // We will simulate rollup usage by creating a mock-up rollup index
+      await createMockRollupIndex(es);
     });
 
     it('create rollup tsvb', async () => {
@@ -108,7 +111,11 @@ export default function ({ getService, getPageObjects }) {
         method: 'DELETE',
       });
 
-      await esDeleteAllIndices([rollupTargetIndexName, rollupSourceIndexName]);
+      await esDeleteAllIndices([
+        rollupTargetIndexName,
+        rollupSourceIndexName,
+        MOCK_ROLLUP_INDEX_NAME,
+      ]);
       await kibanaServer.importExport.unload(
         'x-pack/test/functional/fixtures/kbn_archiver/rollup/rollup.json'
       );