Adds integration test for collecting document migration failures acro…

…ss all batches

src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts

@@ -0,0 +1,153 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+import Util from 'util';
+import * as kbnTestServer from '../../../../test_helpers/kbn_server';
+import { Root } from '../../../root';
+
+const logFilePath = Path.join(__dirname, 'migration_test_corrupt_docs_kibana.log');
+
+const asyncUnlink = Util.promisify(Fs.unlink);
+async function removeLogFile() {
+  // ignore errors if it doesn't exist
+  await asyncUnlink(logFilePath).catch(() => void 0);
+}
+
+describe('migration v2 with corrupt saved object documents', () => {
+  let esServer: kbnTestServer.TestElasticsearchUtils;
+  let root: Root;
+
+  beforeAll(async () => {
+    await removeLogFile();
+  });
+
+  afterAll(async () => {
+    if (root) {
+      await root.shutdown();
+    }
+    if (esServer) {
+      await esServer.stop();
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, 10000));
+  });
+
+  it('collects corrupt saved object documents accross batches', async () => {
+    const { startES } = kbnTestServer.createTestServers({
+      adjustTimeout: (t: number) => jest.setTimeout(t),
+      settings: {
+        es: {
+          license: 'basic',
+          // original uncorrupt SO:
+          // {
+          //  type: 'foo', // 'bar', 'baz'
+          //  foo: {}, // bar: {}, baz: {}
+          //  migrationVersion: {
+          //    foo: '7.13.0',
+          //  },
+          // },
+          // original corrupt SO example:
+          // {
+          //  id: 'bar:123'
+          //  type: 'foo',
+          //  foo: {},
+          //  migrationVersion: {
+          //    foo: '7.13.0',
+          //  },
+          // },
+          // contains migrated index with 8.0 aliases to skip migration, but run outdated doc search
+          dataArchive: Path.join(
+            __dirname,
+            'archives',
+            '8.0.0_migrated_with_corrupt_outdated_docs.zip'
+          ),
+        },
+      },
+    });
+
+    root = createRoot();
+
+    esServer = await startES();
+    const coreSetup = await root.setup();
+
+    coreSetup.savedObjects.registerType({
+      name: 'foo',
+      hidden: false,
+      mappings: { properties: {} },
+      namespaceType: 'agnostic',
+      migrations: {
+        '7.14.0': (doc) => doc,
+      },
+    });
+    coreSetup.savedObjects.registerType({
+      name: 'bar',
+      hidden: false,
+      mappings: { properties: {} },
+      namespaceType: 'agnostic',
+      migrations: {
+        '7.14.0': (doc) => doc,
+      },
+    });
+    coreSetup.savedObjects.registerType({
+      name: 'baz',
+      hidden: false,
+      mappings: { properties: {} },
+      namespaceType: 'agnostic',
+      migrations: {
+        '7.14.0': (doc) => doc,
+      },
+    });
+    try {
+      await root.start();
+    } catch (err) {
+      const corruptFooSOs = /foo:/g;
+      const corruptBarSOs = /bar:/g;
+      const corruptBazSOs = /baz:/g;
+      expect(
+        [...err.message.matchAll(corruptFooSOs)].concat(
+          [...err.message.matchAll(corruptBarSOs)],
+          [...err.message.matchAll(corruptBazSOs)]
+        ).length
+      ).toEqual(16);
+    }
+  });
+});
+
+function createRoot() {
+  return kbnTestServer.createRootWithCorePlugins(
+    {
+      migrations: {
+        skip: false,
+        enableV2: true,
+        batchSize: 5,
+      },
+      logging: {
+        appenders: {
+          file: {
+            type: 'file',
+            fileName: logFilePath,
+            layout: {
+              type: 'json',
+            },
+          },
+        },
+        loggers: [
+          {
+            name: 'root',
+            appenders: ['file'],
+          },
+        ],
+      },
+    },
+    {
+      oss: true,
+    }
+  );
+}

src/core/server/saved_objects/migrationsv2/model.test.ts

@@ -821,7 +821,7 @@ describe('migrations v2 model', () => {
         const newState = model(testState, res) as FatalState;
         expect(newState.controlState).toBe('FATAL');
         expect(newState.reason).toMatchInlineSnapshot(
-          `"Migrations failed. Reason: Corrupt saved object documents: a:b.  To allow migrations to proceed, please delete these documents."`
+          `"Migrations failed. Reason: Corrupt saved object documents: a:b. To allow migrations to proceed, please delete these documents."`
         );
       });
     });

src/core/server/saved_objects/migrationsv2/model.ts

@@ -106,12 +106,12 @@ function extractTransformFailuresReason(
 ): { corruptDocsReason: string; transformErrsReason: string } {
   const corruptDocumentIdReason =
     corruptDocumentIds.length > 0
-      ? `Corrupt saved object documents: ${corruptDocumentIds.join(',')}.`
-      : ' ';
+      ? ` Corrupt saved object documents: ${corruptDocumentIds.join(',')}`
+      : '';
   // we have both the saved object Id and the stack trace in each `transformErrors` item.
   const transformErrorsReason =
     transformErrors.length > 0
-      ? 'Transformation errors: ' +
+      ? ' Transformation errors: ' +
         transformErrors
           .map((errObj) => `${errObj.rawId}: ${errObj.err.message}\n ${errObj.err.stack ?? ''}`)
           .join('/n')
@@ -534,7 +534,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
           return {
             ...stateP,
             controlState: 'FATAL',
-            reason: `Migrations failed. Reason: ${corruptDocsReason} ${transformErrsReason} To allow migrations to proceed, please delete these documents.`,
+            reason: `Migrations failed. Reason:${corruptDocsReason}${transformErrsReason}. To allow migrations to proceed, please delete these documents.`,
           };
         } else {
           // we don't have any more outdated documents and we haven't encountered any document transformation issues.