diff --git a/src/core/server/saved_objects/migrations/core/build_index_map.test.ts b/src/core/server/saved_objects/migrations/core/build_index_map.test.ts new file mode 100644 index 0000000000000..c4a3319fa22c9 --- /dev/null +++ b/src/core/server/saved_objects/migrations/core/build_index_map.test.ts @@ -0,0 +1,158 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createIndexMap } from './build_index_map'; + +test('mappings without index pattern goes to default index', () => { + const result = createIndexMap( + '.kibana', + { + type1: { + isNamespaceAgnostic: false, + }, + }, + { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + } + ); + expect(result).toEqual({ + '.kibana': { + typeMappings: { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + }, + }, + }); +}); + +test(`mappings with custom index pattern doesn't go to default index`, () => { + const result = createIndexMap( + '.kibana', + { + type1: { + isNamespaceAgnostic: false, + indexPattern: '.other_kibana', + }, + }, + { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + } + ); + expect(result).toEqual({ + '.other_kibana': { + typeMappings: { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + }, + }, + }); +}); + +test('creating a script gets added to the index pattern', () => { + const result = createIndexMap( + '.kibana', + { + type1: { + isNamespaceAgnostic: false, + indexPattern: '.other_kibana', + convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`, + }, + }, + { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + } + ); + expect(result).toEqual({ + '.other_kibana': { + script: `ctx._id = ctx._source.type + ':' + ctx._id`, + typeMappings: { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + }, + }, + }); +}); + +test('throws when two scripts are defined for an index pattern', () => { + const defaultIndex = '.kibana'; + const savedObjectSchemas = { + type1: { + isNamespaceAgnostic: false, + convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`, + }, + type2: { + isNamespaceAgnostic: false, + convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`, + }, + }; + const indexMap = { + type1: { + properties: { + field1: { + type: 'string', + }, + }, + }, + type2: { + properties: { + field1: { + type: 'string', + }, + }, + }, + }; + expect(() => + createIndexMap(defaultIndex, savedObjectSchemas, indexMap) + ).toThrowErrorMatchingInlineSnapshot( + `"convertToAliasScript has been defined more than once for index pattern \\".kibana\\""` + ); +}); diff --git a/src/core/server/saved_objects/migrations/core/build_index_map.ts b/src/core/server/saved_objects/migrations/core/build_index_map.ts index 365c79692ba0d..e08bc9e972767 100644 --- a/src/core/server/saved_objects/migrations/core/build_index_map.ts +++ b/src/core/server/saved_objects/migrations/core/build_index_map.ts @@ -20,6 +20,13 @@ import { MappingProperties } from '../../mappings'; import { SavedObjectsSchemaDefinition } from '../../schema'; +export interface IndexMap { + [index: string]: { + typeMappings: MappingProperties; + script?: string; + }; +} + /* * This file contains logic to convert savedObjectSchemas into a dictonary of indexes and documents */ @@ -28,13 +35,22 @@ export function createIndexMap( savedObjectSchemas: SavedObjectsSchemaDefinition, indexMap: MappingProperties ) { - const map: { [index: string]: MappingProperties } = {}; + const map: IndexMap = {}; Object.keys(indexMap).forEach(type => { - const indexPattern = (savedObjectSchemas[type] || {}).indexPattern || defaultIndex; + const schema = savedObjectSchemas[type] || {}; + const script = schema.convertToAliasScript; + const indexPattern = schema.indexPattern || defaultIndex; if (!map.hasOwnProperty(indexPattern as string)) { - map[indexPattern] = {}; + map[indexPattern] = { typeMappings: {} }; + } + map[indexPattern].typeMappings[type] = indexMap[type]; + if (script && map[indexPattern].script) { + throw Error( + `convertToAliasScript has been defined more than once for index pattern "${indexPattern}"` + ); + } else if (script) { + map[indexPattern].script = script; } - map[indexPattern][type] = indexMap[type]; }); return map; } diff --git a/src/core/server/saved_objects/migrations/core/call_cluster.ts b/src/core/server/saved_objects/migrations/core/call_cluster.ts index f5b4f787a61d4..628f2785e6c64 100644 --- a/src/core/server/saved_objects/migrations/core/call_cluster.ts +++ b/src/core/server/saved_objects/migrations/core/call_cluster.ts @@ -90,6 +90,10 @@ export interface ReindexOpts { body: { dest: IndexOpts; source: IndexOpts & { size: number }; + script?: { + source: string; + lang: 'painless'; + }; }; refresh: boolean; waitForCompletion: boolean; diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts index 65df2fd580d84..393cbb7fbb2ae 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts @@ -231,6 +231,10 @@ describe('ElasticIndex', () => { body: { dest: { index: '.ze-index' }, source: { index: '.muchacha' }, + script: { + source: `ctx._id = ctx._source.type + ':' + ctx._id`, + lang: 'painless', + }, }, refresh: true, waitForCompletion: false, @@ -267,7 +271,13 @@ describe('ElasticIndex', () => { properties: { foo: { type: 'keyword' } }, }, }; - await Index.convertToAlias(callCluster as any, info, '.muchacha', 10); + await Index.convertToAlias( + callCluster as any, + info, + '.muchacha', + 10, + `ctx._id = ctx._source.type + ':' + ctx._id` + ); expect(callCluster.mock.calls.map(([path]) => path)).toEqual([ 'indices.create', diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index 9606a46edef95..da76905d1c65c 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -228,14 +228,15 @@ export async function convertToAlias( callCluster: CallCluster, info: FullIndexInfo, alias: string, - batchSize: number + batchSize: number, + script?: string ) { await callCluster('indices.create', { body: { mappings: info.mappings, settings }, index: info.indexName, }); - await reindex(callCluster, alias, info.indexName, batchSize); + await reindex(callCluster, alias, info.indexName, batchSize, script); await claimAlias(callCluster, info.indexName, alias, [{ remove_index: { index: alias } }]); } @@ -316,7 +317,13 @@ function assertResponseIncludeAllShards({ _shards }: { _shards: ShardsInfo }) { /** * Reindexes from source to dest, polling for the reindex completion. */ -async function reindex(callCluster: CallCluster, source: string, dest: string, batchSize: number) { +async function reindex( + callCluster: CallCluster, + source: string, + dest: string, + batchSize: number, + script?: string +) { // We poll instead of having the request wait for completion, as for large indices, // the request times out on the Elasticsearch side of things. We have a relatively tight // polling interval, as the request is fairly efficent, and we don't @@ -326,6 +333,12 @@ async function reindex(callCluster: CallCluster, source: string, dest: string, b body: { dest: { index: dest }, source: { index: source, size: batchSize }, + script: script + ? { + source: script, + lang: 'painless', + } + : undefined, }, refresh: true, waitForCompletion: false, diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.ts b/src/core/server/saved_objects/migrations/core/index_migrator.ts index 7fc2bcfb72602..c75fa68572c71 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.ts @@ -176,7 +176,7 @@ async function migrateSourceToDest(context: Context) { if (!source.aliases[alias]) { log.info(`Reindexing ${alias} to ${source.indexName}`); - await Index.convertToAlias(callCluster, source, alias, batchSize); + await Index.convertToAlias(callCluster, source, alias, batchSize, context.convertToAliasScript); } const read = Index.reader(callCluster, source.indexName, { batchSize, scrollDuration }); diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts index f3c4b271c3a72..a151a8d37a524 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.ts @@ -42,6 +42,7 @@ export interface MigrationOpts { mappingProperties: MappingProperties; documentMigrator: VersionedTransformer; serializer: SavedObjectsSerializer; + convertToAliasScript?: string; /** * If specified, templates matching the specified pattern will be removed @@ -62,6 +63,7 @@ export interface Context { scrollDuration: string; serializer: SavedObjectsSerializer; obsoleteIndexTemplatePattern?: string; + convertToAliasScript?: string; } /** @@ -87,6 +89,7 @@ export async function migrationContext(opts: MigrationOpts): Promise { scrollDuration: opts.scrollDuration, serializer: opts.serializer, obsoleteIndexTemplatePattern: opts.obsoleteIndexTemplatePattern, + convertToAliasScript: opts.convertToAliasScript, }; } diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index b2a03a7623bfe..f51c15f882a4e 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -106,11 +106,12 @@ export class KibanaMigrator { documentMigrator: this.documentMigrator, index, log: this.log, - mappingProperties: indexMap[index], + mappingProperties: indexMap[index].typeMappings, pollInterval: config.get('migrations.pollInterval'), scrollDuration: config.get('migrations.scrollDuration'), serializer: this.serializer, obsoleteIndexTemplatePattern: 'kibana_index_template*', + convertToAliasScript: indexMap[index].script, }); }); diff --git a/src/core/server/saved_objects/schema/schema.ts b/src/core/server/saved_objects/schema/schema.ts index 6756feeb15a0f..1f098d0b6e21d 100644 --- a/src/core/server/saved_objects/schema/schema.ts +++ b/src/core/server/saved_objects/schema/schema.ts @@ -20,6 +20,7 @@ interface SavedObjectsSchemaTypeDefinition { isNamespaceAgnostic: boolean; hidden?: boolean; indexPattern?: string; + convertToAliasScript?: string; } export interface SavedObjectsSchemaDefinition {