Allow migrator to run a reindex script when converting an index to an alias

src/core/server/saved_objects/migrations/core/build_index_map.test.ts

@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createIndexMap } from './build_index_map';
+
+test('mappings without index pattern goes to default index', () => {
+  const result = createIndexMap(
+    '.kibana',
+    {
+      type1: {
+        isNamespaceAgnostic: false,
+      },
+    },
+    {
+      type1: {
+        properties: {
+          field1: {
+            type: 'string',
+          },
+        },
+      },
+    }
+  );
+  expect(result).toEqual({
+    '.kibana': {
+      typeMappings: {
+        type1: {
+          properties: {
+            field1: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test(`mappings with custom index pattern doesn't go to default index`, () => {
+  const result = createIndexMap(
+    '.kibana',
+    {
+      type1: {
+        isNamespaceAgnostic: false,
+        indexPattern: '.other_kibana',
+      },
+    },
+    {
+      type1: {
+        properties: {
+          field1: {
+            type: 'string',
+          },
+        },
+      },
+    }
+  );
+  expect(result).toEqual({
+    '.other_kibana': {
+      typeMappings: {
+        type1: {
+          properties: {
+            field1: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test('creating a script gets added to the index pattern', () => {
+  const result = createIndexMap(
+    '.kibana',
+    {
+      type1: {
+        isNamespaceAgnostic: false,
+        indexPattern: '.other_kibana',
+        convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+      },
+    },
+    {
+      type1: {
+        properties: {
+          field1: {
+            type: 'string',
+          },
+        },
+      },
+    }
+  );
+  expect(result).toEqual({
+    '.other_kibana': {
+      script: `ctx._id = ctx._source.type + ':' + ctx._id`,
+      typeMappings: {
+        type1: {
+          properties: {
+            field1: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test('throws when two scripts are defined for an index pattern', () => {
+  const defaultIndex = '.kibana';
+  const savedObjectSchemas = {
+    type1: {
+      isNamespaceAgnostic: false,
+      convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+    },
+    type2: {
+      isNamespaceAgnostic: false,
+      convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+    },
+  };
+  const indexMap = {
+    type1: {
+      properties: {
+        field1: {
+          type: 'string',
+        },
+      },
+    },
+    type2: {
+      properties: {
+        field1: {
+          type: 'string',
+        },
+      },
+    },
+  };
+  expect(() =>
+    createIndexMap(defaultIndex, savedObjectSchemas, indexMap)
+  ).toThrowErrorMatchingInlineSnapshot(
+    `"convertToAliasScript has been defined more than once for index pattern \\".kibana\\""`
+  );
+});

src/core/server/saved_objects/migrations/core/build_index_map.ts

@@ -20,6 +20,13 @@
 import { MappingProperties } from '../../mappings';
 import { SavedObjectsSchemaDefinition } from '../../schema';
 
+export interface IndexMap {
+  [index: string]: {
+    typeMappings: MappingProperties;
+    script?: string;
+  };
+}
+
 /*
  * This file contains logic to convert savedObjectSchemas into a dictonary of indexes and documents
  */
@@ -28,13 +35,22 @@ export function createIndexMap(
   savedObjectSchemas: SavedObjectsSchemaDefinition,
   indexMap: MappingProperties
 ) {
-  const map: { [index: string]: MappingProperties } = {};
+  const map: IndexMap = {};
   Object.keys(indexMap).forEach(type => {
-    const indexPattern = (savedObjectSchemas[type] || {}).indexPattern || defaultIndex;
+    const schema = savedObjectSchemas[type] || {};
+    const script = schema.convertToAliasScript;
+    const indexPattern = schema.indexPattern || defaultIndex;
     if (!map.hasOwnProperty(indexPattern as string)) {
-      map[indexPattern] = {};
+      map[indexPattern] = { typeMappings: {} };
+    }
+    map[indexPattern].typeMappings[type] = indexMap[type];
+    if (script && map[indexPattern].script) {
+      throw Error(
+        `convertToAliasScript has been defined more than once for index pattern "${indexPattern}"`
+      );
+    } else if (script) {
+      map[indexPattern].script = script;
     }
-    map[indexPattern][type] = indexMap[type];
   });
   return map;
 }

src/core/server/saved_objects/migrations/core/call_cluster.ts

@@ -90,6 +90,10 @@ export interface ReindexOpts {
   body: {
     dest: IndexOpts;
     source: IndexOpts & { size: number };
+    script?: {
+      source: string;
+      lang: 'painless';
+    };
   };
   refresh: boolean;
   waitForCompletion: boolean;

src/core/server/saved_objects/migrations/core/elastic_index.test.ts

@@ -231,6 +231,10 @@ describe('ElasticIndex', () => {
               body: {
                 dest: { index: '.ze-index' },
                 source: { index: '.muchacha' },
+                script: {
+                  source: `ctx._id = ctx._source.type + ':' + ctx._id`,
+                  lang: 'painless',
+                },
               },
               refresh: true,
               waitForCompletion: false,
@@ -267,7 +271,13 @@ describe('ElasticIndex', () => {
           properties: { foo: { type: 'keyword' } },
         },
       };
-      await Index.convertToAlias(callCluster as any, info, '.muchacha', 10);
+      await Index.convertToAlias(
+        callCluster as any,
+        info,
+        '.muchacha',
+        10,
+        `ctx._id = ctx._source.type + ':' + ctx._id`
+      );
 
       expect(callCluster.mock.calls.map(([path]) => path)).toEqual([
         'indices.create',

src/core/server/saved_objects/migrations/core/elastic_index.ts

@@ -228,14 +228,15 @@ export async function convertToAlias(
   callCluster: CallCluster,
   info: FullIndexInfo,
   alias: string,
-  batchSize: number
+  batchSize: number,
+  script?: string
 ) {
   await callCluster('indices.create', {
     body: { mappings: info.mappings, settings },
     index: info.indexName,
   });
 
-  await reindex(callCluster, alias, info.indexName, batchSize);
+  await reindex(callCluster, alias, info.indexName, batchSize, script);
 
   await claimAlias(callCluster, info.indexName, alias, [{ remove_index: { index: alias } }]);
 }
@@ -316,7 +317,13 @@ function assertResponseIncludeAllShards({ _shards }: { _shards: ShardsInfo }) {
 /**
  * Reindexes from source to dest, polling for the reindex completion.
  */
-async function reindex(callCluster: CallCluster, source: string, dest: string, batchSize: number) {
+async function reindex(
+  callCluster: CallCluster,
+  source: string,
+  dest: string,
+  batchSize: number,
+  script?: string
+) {
   // We poll instead of having the request wait for completion, as for large indices,
   // the request times out on the Elasticsearch side of things. We have a relatively tight
   // polling interval, as the request is fairly efficent, and we don't
@@ -326,6 +333,12 @@ async function reindex(callCluster: CallCluster, source: string, dest: string, b
     body: {
       dest: { index: dest },
       source: { index: source, size: batchSize },
+      script: script
+        ? {
+            source: script,
+            lang: 'painless',
+          }
+        : undefined,
     },
     refresh: true,
     waitForCompletion: false,

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -176,7 +176,7 @@ async function migrateSourceToDest(context: Context) {
   if (!source.aliases[alias]) {
     log.info(`Reindexing ${alias} to ${source.indexName}`);
 
-    await Index.convertToAlias(callCluster, source, alias, batchSize);
+    await Index.convertToAlias(callCluster, source, alias, batchSize, context.convertToAliasScript);
   }
 
   const read = Index.reader(callCluster, source.indexName, { batchSize, scrollDuration });

src/core/server/saved_objects/migrations/core/migration_context.ts

@@ -42,6 +42,7 @@ export interface MigrationOpts {
   mappingProperties: MappingProperties;
   documentMigrator: VersionedTransformer;
   serializer: SavedObjectsSerializer;
+  convertToAliasScript?: string;
 
   /**
    * If specified, templates matching the specified pattern will be removed
@@ -62,6 +63,7 @@ export interface Context {
   scrollDuration: string;
   serializer: SavedObjectsSerializer;
   obsoleteIndexTemplatePattern?: string;
+  convertToAliasScript?: string;
 }
 
 /**
@@ -87,6 +89,7 @@ export async function migrationContext(opts: MigrationOpts): Promise<Context> {
     scrollDuration: opts.scrollDuration,
     serializer: opts.serializer,
     obsoleteIndexTemplatePattern: opts.obsoleteIndexTemplatePattern,
+    convertToAliasScript: opts.convertToAliasScript,
   };
 }
 

src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts

@@ -106,11 +106,12 @@ export class KibanaMigrator {
         documentMigrator: this.documentMigrator,
         index,
         log: this.log,
-        mappingProperties: indexMap[index],
+        mappingProperties: indexMap[index].typeMappings,
         pollInterval: config.get('migrations.pollInterval'),
         scrollDuration: config.get('migrations.scrollDuration'),
         serializer: this.serializer,
         obsoleteIndexTemplatePattern: 'kibana_index_template*',
+        convertToAliasScript: indexMap[index].script,
       });
     });
 

src/core/server/saved_objects/schema/schema.ts

@@ -20,6 +20,7 @@ interface SavedObjectsSchemaTypeDefinition {
   isNamespaceAgnostic: boolean;
   hidden?: boolean;
   indexPattern?: string;
+  convertToAliasScript?: string;
 }
 
 export interface SavedObjectsSchemaDefinition {