[8.7] [Kibana migrator test kit] Flush logger after running migrations (

#152818) (#152890) # Backport This will backport the following commits from `main` to `8.7`: - [[Kibana migrator test kit] Flush logger after running migrations (#152818)](#152818)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)
elastic · Mar 8, 2023 · e70452f · e70452f
1 parent db3b8c3
commit e70452f
Show file tree

Hide file tree

Showing 4 changed files with 443 additions and 214 deletions.
diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/cleanup.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/cleanup.test.ts
@@ -10,122 +10,47 @@ import Path from 'path';
 import Fs from 'fs';
 import Util from 'util';
 import JSON5 from 'json5';
+import { type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
+import { SavedObjectsType } from '@kbn/core-saved-objects-server';
+import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { getMigrationDocLink, delay } from '../test_utils';
 import {
-  createTestServers,
-  createRootWithCorePlugins,
-  type TestElasticsearchUtils,
-} from '@kbn/core-test-helpers-kbn-server';
-import { Root } from '@kbn/core-root-server-internal';
-import { getMigrationDocLink } from '../test_utils';
+  clearLog,
+  currentVersion,
+  defaultKibanaIndex,
+  getKibanaMigratorTestKit,
+  nextMinor,
+  startElasticsearch,
+} from '../kibana_migrator_test_kit';
 
 const migrationDocLink = getMigrationDocLink().resolveMigrationFailures;
 const logFilePath = Path.join(__dirname, 'cleanup.log');
 
-const asyncUnlink = Util.promisify(Fs.unlink);
 const asyncReadFile = Util.promisify(Fs.readFile);
 
-async function removeLogFile() {
-  // ignore errors if it doesn't exist
-  await asyncUnlink(logFilePath).catch(() => void 0);
-}
-
-function createRoot() {
-  return createRootWithCorePlugins(
-    {
-      migrations: {
-        skip: false,
-      },
-      logging: {
-        appenders: {
-          file: {
-            type: 'file',
-            fileName: logFilePath,
-            layout: {
-              type: 'json',
-            },
-          },
-        },
-        loggers: [
-          {
-            name: 'root',
-            appenders: ['file'],
-            level: 'debug', // DEBUG logs are required to retrieve the PIT _id from the action response logs
-          },
-        ],
-      },
-    },
-    {
-      oss: true,
-    }
-  );
-}
-
 describe('migration v2', () => {
-  let esServer: TestElasticsearchUtils;
-  let root: Root;
+  let esServer: TestElasticsearchUtils['es'];
+  let esClient: ElasticsearchClient;
 
   beforeAll(async () => {
-    await removeLogFile();
+    esServer = await startElasticsearch();
   });
 
-  afterAll(async () => {
-    if (root) {
-      await root.shutdown();
-    }
-    if (esServer) {
-      await esServer.stop();
-    }
-
-    await new Promise((resolve) => setTimeout(resolve, 10000));
+  beforeEach(async () => {
+    esClient = await setupBaseline();
+    await clearLog(logFilePath);
   });
 
   it('clean ups if migration fails', async () => {
-    const { startES } = createTestServers({
-      adjustTimeout: (t: number) => jest.setTimeout(t),
-      settings: {
-        es: {
-          license: 'basic',
-          // original SO:
-          // {
-          //   _index: '.kibana_7.13.0_001',
-          //     _type: '_doc',
-          //   _id: 'index-pattern:test_index*',
-          //   _version: 1,
-          //   result: 'created',
-          //   _shards: { total: 2, successful: 1, failed: 0 },
-          //   _seq_no: 0,
-          //     _primary_term: 1
-          // }
-          dataArchive: Path.join(__dirname, '..', 'archives', '7.13.0_with_corrupted_so.zip'),
-        },
-      },
-    });
-
-    root = createRoot();
-
-    esServer = await startES();
-    await root.preboot();
-    const coreSetup = await root.setup();
+    const { runMigrations, client } = await setupNextMinor();
 
-    coreSetup.savedObjects.registerType({
-      name: 'foo',
-      hidden: false,
-      mappings: {
-        properties: {},
-      },
-      namespaceType: 'agnostic',
-      migrations: {
-        '7.14.0': (doc) => doc,
-      },
-    });
+    await expect(runMigrations()).rejects.toThrowErrorMatchingInlineSnapshot(`
+      "Unable to complete saved object migrations for the [${defaultKibanaIndex}] index: Migrations failed. Reason: 1 corrupt saved object documents were found: corrupt:2baf4de0-a6d4-11ed-ba5a-39196fc76e60
 
-    await expect(root.start()).rejects.toThrowErrorMatchingInlineSnapshot(`
-            "Unable to complete saved object migrations for the [.kibana] index: Migrations failed. Reason: 1 corrupt saved object documents were found: index-pattern:test_index*
-
-            To allow migrations to proceed, please delete or fix these documents.
-            Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
-            Please refer to ${migrationDocLink} for more information."
-          `);
+      To allow migrations to proceed, please delete or fix these documents.
+      Note that you can configure Kibana to automatically discard corrupt documents and transform errors for this migration.
+      Please refer to ${migrationDocLink} for more information."
+    `);
 
     const logFileContent = await asyncReadFile(logFilePath, 'utf-8');
     const records = logFileContent
@@ -134,15 +59,14 @@ describe('migration v2', () => {
       .map((str) => JSON5.parse(str));
 
     const logRecordWithPit = records.find(
-      (rec) => rec.message === '[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT RESPONSE'
+      (rec) => rec.message === `[${defaultKibanaIndex}] REINDEX_SOURCE_TO_TEMP_OPEN_PIT RESPONSE`
     );
 
     expect(logRecordWithPit).toBeTruthy();
 
     const pitId = logRecordWithPit.right.pitId;
     expect(pitId).toBeTruthy();
 
-    const client = esServer.es.getClient();
     await expect(
       client.search({
         body: {
@@ -152,4 +76,129 @@ describe('migration v2', () => {
       // throws an exception that cannot search with closed PIT
     ).rejects.toThrow(/search_phase_execution_exception/);
   });
+
+  afterEach(async () => {
+    await esClient?.indices.delete({ index: `${defaultKibanaIndex}_${currentVersion}_001` });
+  });
+
+  afterAll(async () => {
+    await esServer?.stop();
+    await delay(10);
+  });
 });
+
+const setupBaseline = async () => {
+  const typesCurrent: SavedObjectsType[] = [
+    {
+      name: 'complex',
+      hidden: false,
+      namespaceType: 'agnostic',
+      mappings: {
+        properties: {
+          name: { type: 'text' },
+          value: { type: 'integer' },
+        },
+      },
+      migrations: {},
+    },
+  ];
+
+  const savedObjects = [
+    {
+      id: 'complex:4baf4de0-a6d4-11ed-ba5a-39196fc76e60',
+      body: {
+        type: 'complex',
+        complex: {
+          name: 'foo',
+          value: 5,
+        },
+        references: [],
+        coreMigrationVersion: currentVersion,
+        updated_at: '2023-02-07T11:04:44.914Z',
+        created_at: '2023-02-07T11:04:44.914Z',
+      },
+    },
+    {
+      id: 'corrupt:2baf4de0-a6d4-11ed-ba5a-39196fc76e60', // incorrect id => corrupt object
+      body: {
+        type: 'complex',
+        complex: {
+          name: 'bar',
+          value: 3,
+        },
+        references: [],
+        coreMigrationVersion: currentVersion,
+        updated_at: '2023-02-07T11:04:44.914Z',
+        created_at: '2023-02-07T11:04:44.914Z',
+      },
+    },
+  ];
+
+  const { runMigrations, client } = await getKibanaMigratorTestKit({
+    types: typesCurrent,
+    logFilePath,
+  });
+
+  await runMigrations();
+
+  // inject corrupt saved objects directly using esClient
+  await Promise.all(
+    savedObjects.map((savedObject) => {
+      client.create({
+        index: defaultKibanaIndex,
+        refresh: 'wait_for',
+        ...savedObject,
+      });
+    })
+  );
+
+  return client;
+};
+
+const setupNextMinor = async () => {
+  const typesNextMinor: SavedObjectsType[] = [
+    {
+      name: 'complex',
+      hidden: false,
+      namespaceType: 'agnostic',
+      mappings: {
+        properties: {
+          name: { type: 'keyword' },
+          value: { type: 'long' },
+        },
+      },
+      migrations: {
+        [nextMinor]: (doc) => doc,
+      },
+    },
+  ];
+
+  return await getKibanaMigratorTestKit({
+    types: typesNextMinor,
+    kibanaVersion: nextMinor,
+    logFilePath,
+    settings: {
+      migrations: {
+        skip: false,
+      },
+      logging: {
+        appenders: {
+          file: {
+            type: 'file',
+            fileName: logFilePath,
+            layout: {
+              type: 'json',
+            },
+          },
+        },
+        loggers: [
+          {
+            name: 'root',
+            appenders: ['file'],
+            level: 'debug', // DEBUG logs are required to retrieve the PIT _id from the action response logs
+          },
+        ],
+      },
+    },
+  });
+};