From dbd846005ed427f44a86e9dfe81d389049427ada Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Fri, 30 Apr 2021 12:43:03 +0200 Subject: [PATCH] remove dead code and enableV2 flag --- .../__snapshots__/elastic_index.test.ts.snap | 40 - .../migrations/core/call_cluster.ts | 176 ---- .../migrations/core/elastic_index.test.ts | 733 ----------------- .../migrations/core/elastic_index.ts | 363 --------- .../saved_objects/migrations/core/index.ts | 6 +- .../migrations/core/index_migrator.test.ts | 478 ----------- .../migrations/core/index_migrator.ts | 198 ----- .../migrations/core/migration_context.ts | 130 +-- .../core/migration_coordinator.test.ts | 75 -- .../migrations/core/migration_coordinator.ts | 124 --- .../core/migration_es_client.test.ts | 55 -- .../migrations/core/migration_es_client.ts | 79 -- ...ration_es_client.test.mock.ts => types.ts} | 19 +- .../migrations/kibana/kibana_migrator.mock.ts | 2 - .../migrations/kibana/kibana_migrator.test.ts | 197 ++--- .../migrations/kibana/kibana_migrator.ts | 65 +- .../integration_tests/cleanup.test.ts | 1 - .../integration_tests/migration.test.ts | 1 - .../migration_7.7.2_xpack_100k.test.ts | 1 - .../integration_tests/rewriting_id.test.ts | 1 - .../migrations_state_action_machine.test.ts | 1 - .../saved_objects/saved_objects_config.ts | 19 +- .../saved_objects/saved_objects_service.ts | 15 +- .../saved_objects/service/lib/repository.ts | 11 +- .../apis/saved_objects/index.ts | 1 - .../apis/saved_objects/migrations.ts | 760 ------------------ 26 files changed, 106 insertions(+), 3445 deletions(-) delete mode 100644 src/core/server/saved_objects/migrations/core/__snapshots__/elastic_index.test.ts.snap delete mode 100644 src/core/server/saved_objects/migrations/core/call_cluster.ts delete mode 100644 src/core/server/saved_objects/migrations/core/elastic_index.test.ts delete mode 100644 src/core/server/saved_objects/migrations/core/index_migrator.test.ts delete mode 100644 src/core/server/saved_objects/migrations/core/index_migrator.ts delete mode 100644 src/core/server/saved_objects/migrations/core/migration_coordinator.test.ts delete mode 100644 src/core/server/saved_objects/migrations/core/migration_coordinator.ts delete mode 100644 src/core/server/saved_objects/migrations/core/migration_es_client.test.ts delete mode 100644 src/core/server/saved_objects/migrations/core/migration_es_client.ts rename src/core/server/saved_objects/migrations/core/{migration_es_client.test.mock.ts => types.ts} (53%) delete mode 100644 test/api_integration/apis/saved_objects/migrations.ts diff --git a/src/core/server/saved_objects/migrations/core/__snapshots__/elastic_index.test.ts.snap b/src/core/server/saved_objects/migrations/core/__snapshots__/elastic_index.test.ts.snap deleted file mode 100644 index 6bd567be204d0..0000000000000 --- a/src/core/server/saved_objects/migrations/core/__snapshots__/elastic_index.test.ts.snap +++ /dev/null @@ -1,40 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ElasticIndex write writes documents in bulk to the index 1`] = ` -Array [ - Object { - "body": Array [ - Object { - "index": Object { - "_id": "niceguy:fredrogers", - "_index": ".myalias", - }, - }, - Object { - "niceguy": Object { - "aka": "Mr Rogers", - }, - "quotes": Array [ - "The greatest gift you ever give is your honest self.", - ], - "type": "niceguy", - }, - Object { - "index": Object { - "_id": "badguy:rickygervais", - "_index": ".myalias", - }, - }, - Object { - "badguy": Object { - "aka": "Dominic Badguy", - }, - "migrationVersion": Object { - "badguy": "2.3.4", - }, - "type": "badguy", - }, - ], - }, -] -`; diff --git a/src/core/server/saved_objects/migrations/core/call_cluster.ts b/src/core/server/saved_objects/migrations/core/call_cluster.ts deleted file mode 100644 index f37bbdd14a899..0000000000000 --- a/src/core/server/saved_objects/migrations/core/call_cluster.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/** - * This file is nothing more than type signatures for the subset of - * elasticsearch.js that migrations use. There is no actual logic / - * funcationality contained here. - */ - -import type { estypes } from '@elastic/elasticsearch'; -import { IndexMapping } from '../../mappings'; - -export interface CallCluster { - (path: 'bulk', opts: { body: object[] }): Promise; - (path: 'count', opts: CountOpts): Promise<{ count: number; _shards: estypes.ShardStatistics }>; - (path: 'clearScroll', opts: { scrollId: string }): Promise; - (path: 'indices.create', opts: IndexCreationOpts): Promise; - (path: 'indices.exists', opts: IndexOpts): Promise; - (path: 'indices.existsAlias', opts: { name: string }): Promise; - (path: 'indices.get', opts: IndexOpts & Ignorable): Promise; - (path: 'indices.getAlias', opts: { name: string } & Ignorable): Promise; - (path: 'indices.getMapping', opts: IndexOpts): Promise; - (path: 'indices.getSettings', opts: IndexOpts): Promise; - (path: 'indices.refresh', opts: IndexOpts): Promise; - (path: 'indices.updateAliases', opts: UpdateAliasesOpts): Promise; - (path: 'indices.deleteTemplate', opts: { name: string }): Promise; - (path: 'cat.templates', opts: { format: 'json'; name: string }): Promise>; - (path: 'reindex', opts: ReindexOpts): Promise; - (path: 'scroll', opts: ScrollOpts): Promise; - (path: 'search', opts: SearchOpts): Promise; - (path: 'tasks.get', opts: { taskId: string }): Promise<{ - completed: boolean; - error?: ErrorResponse; - }>; -} - -/////////////////////////////////////////////////////////////////// -// callCluster argument type definitions -/////////////////////////////////////////////////////////////////// - -export interface Ignorable { - ignore: number[]; -} - -export interface CountOpts { - body: { - query: object; - }; - index: string; -} - -export interface PutMappingOpts { - body: IndexMapping; - index: string; -} - -export interface IndexOpts { - index: string; -} - -export interface IndexCreationOpts { - index: string; - body?: { - mappings?: IndexMapping; - settings?: { - number_of_shards: number; - auto_expand_replicas: string; - }; - }; -} - -export interface ReindexOpts { - body: { - dest: IndexOpts; - source: IndexOpts & { size: number }; - script?: { - source: string; - lang: 'painless'; - }; - }; - refresh: boolean; - waitForCompletion: boolean; -} - -export type AliasAction = - | { remove_index: IndexOpts } - | { remove: { index: string; alias: string } } - | { add: { index: string; alias: string } }; - -export interface UpdateAliasesOpts { - body: { - actions: AliasAction[]; - }; -} - -export interface SearchOpts { - body: object; - index: string; - scroll?: string; -} - -export interface ScrollOpts { - scroll: string; - scrollId: string; -} - -/////////////////////////////////////////////////////////////////// -// callCluster result type definitions -/////////////////////////////////////////////////////////////////// - -export interface NotFound { - status: 404; -} - -export interface MappingResult { - [index: string]: { - mappings: IndexMapping; - }; -} - -export interface AliasResult { - [alias: string]: object; -} - -export interface IndexSettingsResult { - [indexName: string]: { - settings: { - index: { - number_of_shards: string; - auto_expand_replicas: string; - provided_name: string; - creation_date: string; - number_of_replicas: string; - uuid: string; - version: { created: '7000001' }; - }; - }; - }; -} - -export interface RawDoc { - _id: estypes.Id; - _source: any; - _type?: string; -} - -export interface SearchResults { - hits: { - hits: RawDoc[]; - }; - _scroll_id?: string; - _shards: estypes.ShardStatistics; -} - -export interface ErrorResponse { - type: string; - reason: string; -} - -export interface BulkResult { - items: Array<{ index: { error?: ErrorResponse } }>; -} - -export interface IndexInfo { - aliases: AliasResult; - mappings: IndexMapping; -} - -export interface IndicesInfo { - [index: string]: IndexInfo; -} diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts deleted file mode 100644 index 1d2ec6abc0dd1..0000000000000 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ /dev/null @@ -1,733 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { estypes } from '@elastic/elasticsearch'; -import _ from 'lodash'; -import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; -import * as Index from './elastic_index'; - -describe('ElasticIndex', () => { - let client: ReturnType; - - beforeEach(() => { - client = elasticsearchClientMock.createElasticsearchClient(); - }); - describe('fetchInfo', () => { - test('it handles 404', async () => { - client.indices.get.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - const info = await Index.fetchInfo(client, '.kibana-test'); - expect(info).toEqual({ - aliases: {}, - exists: false, - indexName: '.kibana-test', - mappings: { dynamic: 'strict', properties: {} }, - }); - - expect(client.indices.get).toHaveBeenCalledWith({ index: '.kibana-test' }, { ignore: [404] }); - }); - - test('decorates index info with exists and indexName', async () => { - client.indices.get.mockImplementation((params) => { - const index = params!.index as string; - return elasticsearchClientMock.createSuccessTransportRequestPromise({ - [index]: { - aliases: { foo: index }, - mappings: { dynamic: 'strict', properties: { a: 'b' } }, - settings: {}, - }, - } as estypes.GetIndexResponse); - }); - - const info = await Index.fetchInfo(client, '.baz'); - expect(info).toEqual({ - aliases: { foo: '.baz' }, - mappings: { dynamic: 'strict', properties: { a: 'b' } }, - exists: true, - indexName: '.baz', - settings: {}, - }); - }); - }); - - describe('createIndex', () => { - test('calls indices.create', async () => { - await Index.createIndex(client, '.abcd', { foo: 'bar' } as any); - - expect(client.indices.create).toHaveBeenCalledTimes(1); - expect(client.indices.create).toHaveBeenCalledWith({ - body: { - mappings: { foo: 'bar' }, - settings: { - auto_expand_replicas: '0-1', - number_of_shards: 1, - }, - }, - index: '.abcd', - }); - }); - }); - - describe('claimAlias', () => { - test('handles unaliased indices', async () => { - client.indices.getAlias.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - await Index.claimAlias(client, '.hola-42', '.hola'); - - expect(client.indices.getAlias).toHaveBeenCalledWith( - { - name: '.hola', - }, - { ignore: [404] } - ); - expect(client.indices.updateAliases).toHaveBeenCalledWith({ - body: { - actions: [{ add: { index: '.hola-42', alias: '.hola' } }], - }, - }); - expect(client.indices.refresh).toHaveBeenCalledWith({ - index: '.hola-42', - }); - }); - - test('removes existing alias', async () => { - client.indices.getAlias.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - '.my-fanci-index': { aliases: { '.muchacha': {} } }, - }) - ); - - await Index.claimAlias(client, '.ze-index', '.muchacha'); - - expect(client.indices.getAlias).toHaveBeenCalledTimes(1); - expect(client.indices.updateAliases).toHaveBeenCalledWith({ - body: { - actions: [ - { remove: { index: '.my-fanci-index', alias: '.muchacha' } }, - { add: { index: '.ze-index', alias: '.muchacha' } }, - ], - }, - }); - expect(client.indices.refresh).toHaveBeenCalledWith({ - index: '.ze-index', - }); - }); - - test('allows custom alias actions', async () => { - client.indices.getAlias.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - '.my-fanci-index': { aliases: { '.muchacha': {} } }, - }) - ); - - await Index.claimAlias(client, '.ze-index', '.muchacha', [ - { remove_index: { index: 'awww-snap!' } }, - ]); - - expect(client.indices.getAlias).toHaveBeenCalledTimes(1); - expect(client.indices.updateAliases).toHaveBeenCalledWith({ - body: { - actions: [ - { remove_index: { index: 'awww-snap!' } }, - { remove: { index: '.my-fanci-index', alias: '.muchacha' } }, - { add: { index: '.ze-index', alias: '.muchacha' } }, - ], - }, - }); - expect(client.indices.refresh).toHaveBeenCalledWith({ - index: '.ze-index', - }); - }); - }); - - describe('convertToAlias', () => { - test('it creates the destination index, then reindexes to it', async () => { - client.indices.getAlias.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - '.my-fanci-index': { aliases: { '.muchacha': {} } }, - }) - ); - client.reindex.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - task: 'abc', - } as estypes.ReindexResponse) - ); - client.tasks.get.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - completed: true, - } as estypes.GetTaskResponse) - ); - - const info = { - aliases: {}, - exists: true, - indexName: '.ze-index', - mappings: { - dynamic: 'strict' as const, - properties: { foo: { type: 'keyword' } }, - }, - }; - - await Index.convertToAlias( - client, - info, - '.muchacha', - 10, - `ctx._id = ctx._source.type + ':' + ctx._id` - ); - - expect(client.indices.create).toHaveBeenCalledWith({ - body: { - mappings: { - dynamic: 'strict', - properties: { foo: { type: 'keyword' } }, - }, - settings: { auto_expand_replicas: '0-1', number_of_shards: 1 }, - }, - index: '.ze-index', - }); - - expect(client.reindex).toHaveBeenCalledWith({ - body: { - dest: { index: '.ze-index' }, - source: { index: '.muchacha', size: 10 }, - script: { - source: `ctx._id = ctx._source.type + ':' + ctx._id`, - lang: 'painless', - }, - }, - refresh: true, - wait_for_completion: false, - }); - - expect(client.tasks.get).toHaveBeenCalledWith({ - task_id: 'abc', - }); - - expect(client.indices.updateAliases).toHaveBeenCalledWith({ - body: { - actions: [ - { remove_index: { index: '.muchacha' } }, - { remove: { alias: '.muchacha', index: '.my-fanci-index' } }, - { add: { index: '.ze-index', alias: '.muchacha' } }, - ], - }, - }); - - expect(client.indices.refresh).toHaveBeenCalledWith({ - index: '.ze-index', - }); - }); - - test('throws error if re-index task fails', async () => { - client.indices.getAlias.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - '.my-fanci-index': { aliases: { '.muchacha': {} } }, - }) - ); - client.reindex.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - task: 'abc', - } as estypes.ReindexResponse) - ); - client.tasks.get.mockResolvedValue( - // @ts-expect-error @elastic/elasticsearch GetTaskResponse requires a `task` property even on errors - elasticsearchClientMock.createSuccessTransportRequestPromise({ - completed: true, - error: { - type: 'search_phase_execution_exception', - reason: 'all shards failed', - failed_shards: [], - }, - } as estypes.GetTaskResponse) - ); - - const info = { - aliases: {}, - exists: true, - indexName: '.ze-index', - mappings: { - dynamic: 'strict', - properties: { foo: { type: 'keyword' } }, - }, - }; - - // @ts-expect-error - await expect(Index.convertToAlias(client, info, '.muchacha', 10)).rejects.toThrow( - /Re-index failed \[search_phase_execution_exception\] all shards failed/ - ); - - expect(client.indices.create).toHaveBeenCalledWith({ - body: { - mappings: { - dynamic: 'strict', - properties: { foo: { type: 'keyword' } }, - }, - settings: { auto_expand_replicas: '0-1', number_of_shards: 1 }, - }, - index: '.ze-index', - }); - - expect(client.reindex).toHaveBeenCalledWith({ - body: { - dest: { index: '.ze-index' }, - source: { index: '.muchacha', size: 10 }, - }, - refresh: true, - wait_for_completion: false, - }); - - expect(client.tasks.get).toHaveBeenCalledWith({ - task_id: 'abc', - }); - }); - }); - - describe('write', () => { - test('writes documents in bulk to the index', async () => { - client.bulk.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - items: [] as any[], - } as estypes.BulkResponse) - ); - - const index = '.myalias'; - const docs = [ - { - _id: 'niceguy:fredrogers', - _source: { - type: 'niceguy', - niceguy: { - aka: 'Mr Rogers', - }, - quotes: ['The greatest gift you ever give is your honest self.'], - }, - }, - { - _id: 'badguy:rickygervais', - _source: { - type: 'badguy', - badguy: { - aka: 'Dominic Badguy', - }, - migrationVersion: { badguy: '2.3.4' }, - }, - }, - ]; - - await Index.write(client, index, docs); - - expect(client.bulk).toHaveBeenCalled(); - expect(client.bulk.mock.calls[0]).toMatchSnapshot(); - }); - - test('fails if any document fails', async () => { - client.bulk.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - items: [{ index: { error: { type: 'shazm', reason: 'dern' } } }], - } as estypes.BulkResponse) - ); - - const index = '.myalias'; - const docs = [ - { - _id: 'niceguy:fredrogers', - _source: { - type: 'niceguy', - niceguy: { - aka: 'Mr Rogers', - }, - }, - }, - ]; - - await expect(Index.write(client as any, index, docs)).rejects.toThrow(/dern/); - expect(client.bulk).toHaveBeenCalledTimes(1); - }); - }); - - describe('reader', () => { - test('returns docs in batches', async () => { - const index = '.myalias'; - const batch1 = [ - { - _id: 'such:1', - _source: { type: 'such', such: { num: 1 } }, - }, - ]; - const batch2 = [ - { - _id: 'aaa:2', - _source: { type: 'aaa', aaa: { num: 2 } }, - }, - { - _id: 'bbb:3', - _source: { - bbb: { num: 3 }, - migrationVersion: { bbb: '3.2.5' }, - type: 'bbb', - }, - }, - ]; - - client.search = jest.fn().mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'x', - _shards: { success: 1, total: 1 }, - hits: { hits: _.cloneDeep(batch1) }, - }) - ); - client.scroll = jest - .fn() - .mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'y', - _shards: { success: 1, total: 1 }, - hits: { hits: _.cloneDeep(batch2) }, - }) - ) - .mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'z', - _shards: { success: 1, total: 1 }, - hits: { hits: [] }, - }) - ); - - const read = Index.reader(client, index, { batchSize: 100, scrollDuration: '5m' }); - - expect(await read()).toEqual(batch1); - expect(await read()).toEqual(batch2); - expect(await read()).toEqual([]); - - expect(client.search).toHaveBeenCalledWith({ - body: { - size: 100, - query: { - bool: { - must_not: [ - { - term: { - type: 'fleet-agent-events', - }, - }, - { - term: { - type: 'tsvb-validation-telemetry', - }, - }, - { - bool: { - must: [ - { - match: { - type: 'search-session', - }, - }, - { - match: { - 'search-session.persisted': false, - }, - }, - ], - }, - }, - ], - }, - }, - }, - index, - scroll: '5m', - }); - expect(client.scroll).toHaveBeenCalledWith({ - scroll: '5m', - scroll_id: 'x', - }); - expect(client.scroll).toHaveBeenCalledWith({ - scroll: '5m', - scroll_id: 'y', - }); - expect(client.clearScroll).toHaveBeenCalledWith({ - scroll_id: 'z', - }); - }); - - test('returns all root-level properties', async () => { - const index = '.myalias'; - const batch = [ - { - _id: 'such:1', - _source: { - acls: '3230a', - foos: { is: 'fun' }, - such: { num: 1 }, - type: 'such', - }, - }, - ]; - - client.search = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'x', - _shards: { success: 1, total: 1 }, - hits: { hits: _.cloneDeep(batch) }, - }) - ); - client.scroll = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'z', - _shards: { success: 1, total: 1 }, - hits: { hits: [] }, - }) - ); - - const read = Index.reader(client, index, { - batchSize: 100, - scrollDuration: '5m', - }); - - expect(await read()).toEqual(batch); - }); - - test('fails if not all shards were successful', async () => { - const index = '.myalias'; - - client.search = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _shards: { successful: 1, total: 2 }, - }) - ); - - const read = Index.reader(client, index, { - batchSize: 100, - scrollDuration: '5m', - }); - - await expect(read()).rejects.toThrow(/shards failed/); - }); - - test('handles shards not being returned', async () => { - const index = '.myalias'; - const batch = [ - { - _id: 'such:1', - _source: { - acls: '3230a', - foos: { is: 'fun' }, - such: { num: 1 }, - type: 'such', - }, - }, - ]; - - client.search = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'x', - hits: { hits: _.cloneDeep(batch) }, - }) - ); - client.scroll = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - _scroll_id: 'z', - hits: { hits: [] }, - }) - ); - - const read = Index.reader(client, index, { - batchSize: 100, - scrollDuration: '5m', - }); - - expect(await read()).toEqual(batch); - }); - }); - - describe('migrationsUpToDate', () => { - // A helper to reduce boilerplate in the hasMigration tests that follow. - async function testMigrationsUpToDate({ - index = '.myindex', - mappings, - count, - migrations, - kibanaVersion, - }: any) { - client.indices.get = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - [index]: { mappings }, - }) - ); - client.count = jest.fn().mockReturnValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - count, - _shards: { success: 1, total: 1 }, - }) - ); - - const hasMigrations = await Index.migrationsUpToDate( - client, - index, - migrations, - kibanaVersion - ); - return { hasMigrations }; - } - - test('is false if the index mappings do not contain migrationVersion', async () => { - const { hasMigrations } = await testMigrationsUpToDate({ - index: '.myalias', - mappings: { - properties: { - dashboard: { type: 'text' }, - }, - }, - count: 0, - migrations: { dashy: '2.3.4' }, - kibanaVersion: '7.10.0', - }); - - expect(hasMigrations).toBeFalsy(); - expect(client.indices.get).toHaveBeenCalledWith( - { - index: '.myalias', - }, - { - ignore: [404], - } - ); - }); - - test('is true if there are no migrations defined', async () => { - const { hasMigrations } = await testMigrationsUpToDate({ - index: '.myalias', - mappings: { - properties: { - migrationVersion: { - dynamic: 'true', - type: 'object', - }, - dashboard: { type: 'text' }, - }, - }, - count: 2, - migrations: {}, - kibanaVersion: '7.10.0', - }); - - expect(hasMigrations).toBeTruthy(); - expect(client.indices.get).toHaveBeenCalledTimes(1); - }); - - test('is true if there are no documents out of date', async () => { - const { hasMigrations } = await testMigrationsUpToDate({ - index: '.myalias', - mappings: { - properties: { - migrationVersion: { - dynamic: 'true', - type: 'object', - }, - dashboard: { type: 'text' }, - }, - }, - count: 0, - migrations: { dashy: '23.2.5' }, - }); - - expect(hasMigrations).toBeTruthy(); - expect(client.indices.get).toHaveBeenCalledTimes(1); - expect(client.count).toHaveBeenCalledTimes(1); - }); - - test('is false if there are documents out of date', async () => { - const { hasMigrations } = await testMigrationsUpToDate({ - index: '.myalias', - mappings: { - properties: { - migrationVersion: { - dynamic: 'true', - type: 'object', - }, - dashboard: { type: 'text' }, - }, - }, - count: 3, - migrations: { dashy: '23.2.5' }, - kibanaVersion: '7.10.0', - }); - - expect(hasMigrations).toBeFalsy(); - expect(client.indices.get).toHaveBeenCalledTimes(1); - expect(client.count).toHaveBeenCalledTimes(1); - }); - - test('counts docs that are out of date', async () => { - await testMigrationsUpToDate({ - index: '.myalias', - mappings: { - properties: { - migrationVersion: { - dynamic: 'true', - type: 'object', - }, - dashboard: { type: 'text' }, - }, - }, - count: 0, - migrations: { - dashy: '23.2.5', - bashy: '99.9.3', - flashy: '3.4.5', - }, - kibanaVersion: '7.10.0', - }); - - function shouldClause(type: string, version: string) { - return { - bool: { - must: [ - { exists: { field: type } }, - { - bool: { - must_not: { term: { [`migrationVersion.${type}`]: version } }, - }, - }, - ], - }, - }; - } - - expect(client.count).toHaveBeenCalledWith({ - body: { - query: { - bool: { - should: [ - shouldClause('dashy', '23.2.5'), - shouldClause('bashy', '99.9.3'), - shouldClause('flashy', '3.4.5'), - { - bool: { - must_not: { - term: { - coreMigrationVersion: '7.10.0', - }, - }, - }, - }, - ], - }, - }, - }, - index: '.myalias', - }); - }); - }); -}); diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index 44dd60097f1cd..04ec238475e41 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -6,27 +6,7 @@ * Side Public License, v 1. */ -/* - * This module contains various functions for querying and manipulating - * elasticsearch indices. - */ - -import _ from 'lodash'; import { estypes } from '@elastic/elasticsearch'; -import { MigrationEsClient } from './migration_es_client'; -import { IndexMapping } from '../../mappings'; -import { SavedObjectsMigrationVersion } from '../../types'; -import { AliasAction, RawDoc } from './call_cluster'; -import { SavedObjectsRawDocSource } from '../../serialization'; - -const settings = { number_of_shards: 1, auto_expand_replicas: '0-1' }; - -export interface FullIndexInfo { - aliases: { [name: string]: object }; - exists: boolean; - indexName: string; - mappings: IndexMapping; -} // When migrating from the outdated index we use a read query which excludes // saved objects which are no longer used. These saved objects will still be @@ -67,346 +47,3 @@ export const excludeUnusedTypesQuery: estypes.QueryContainer = { ], }, }; - -/** - * A slight enhancement to indices.get, that adds indexName, and validates that the - * index mappings are somewhat what we expect. - */ -export async function fetchInfo(client: MigrationEsClient, index: string): Promise { - const { body, statusCode } = await client.indices.get({ index }, { ignore: [404] }); - - if (statusCode === 404) { - return { - aliases: {}, - exists: false, - indexName: index, - mappings: { dynamic: 'strict', properties: {} }, - }; - } - - const [indexName, indexInfo] = Object.entries(body)[0]; - - // @ts-expect-error @elastic/elasticsearch IndexState.alias and IndexState.mappings should be required - return assertIsSupportedIndex({ ...indexInfo, exists: true, indexName }); -} - -/** - * Creates a reader function that serves up batches of documents from the index. We aren't using - * an async generator, as that feature currently breaks Kibana's tooling. - * - * @param client - The elastic search connection - * @param index - The index to be read from - * @param {opts} - * @prop batchSize - The number of documents to read at a time - * @prop scrollDuration - The scroll duration used for scrolling through the index - */ -export function reader( - client: MigrationEsClient, - index: string, - { batchSize = 10, scrollDuration = '15m' }: { batchSize: number; scrollDuration: string } -) { - const scroll = scrollDuration; - let scrollId: string | undefined; - - const nextBatch = () => - scrollId !== undefined - ? client.scroll({ - scroll, - scroll_id: scrollId, - }) - : client.search({ - body: { - size: batchSize, - query: excludeUnusedTypesQuery, - }, - index, - scroll, - }); - - const close = async () => scrollId && (await client.clearScroll({ scroll_id: scrollId })); - - return async function read() { - const result = await nextBatch(); - assertResponseIncludeAllShards(result.body); - - scrollId = result.body._scroll_id; - const docs = result.body.hits.hits; - if (!docs.length) { - await close(); - } - - return docs; - }; -} - -/** - * Writes the specified documents to the index, throws an exception - * if any of the documents fail to save. - */ -export async function write(client: MigrationEsClient, index: string, docs: RawDoc[]) { - const { body } = await client.bulk({ - body: docs.reduce((acc: object[], doc: RawDoc) => { - acc.push({ - index: { - _id: doc._id, - _index: index, - }, - }); - - acc.push(doc._source); - - return acc; - }, []), - }); - - const err = _.find(body.items, 'index.error.reason'); - - if (!err) { - return; - } - - const exception: any = new Error(err.index!.error!.reason); - exception.detail = err; - throw exception; -} - -/** - * Checks to see if the specified index is up to date. It does this by checking - * that the index has the expected mappings and by counting - * the number of documents that have a property which has migrations defined for it, - * but which has not had those migrations applied. We don't want to cache the - * results of this function (e.g. in context or somewhere), as it is important that - * it performs the check *each* time it is called, rather than memoizing itself, - * as this is used to determine if migrations are complete. - * - * @param client - The connection to ElasticSearch - * @param index - * @param migrationVersion - The latest versions of the migrations - */ -export async function migrationsUpToDate( - client: MigrationEsClient, - index: string, - migrationVersion: SavedObjectsMigrationVersion, - kibanaVersion: string, - retryCount: number = 10 -): Promise { - try { - const indexInfo = await fetchInfo(client, index); - - if (!indexInfo.mappings.properties?.migrationVersion) { - return false; - } - - // If no migrations are actually defined, we're up to date! - if (Object.keys(migrationVersion).length <= 0) { - return true; - } - - const { body } = await client.count({ - body: { - query: { - bool: { - should: [ - ...Object.entries(migrationVersion).map(([type, latestVersion]) => ({ - bool: { - must: [ - { exists: { field: type } }, - { - bool: { - must_not: { term: { [`migrationVersion.${type}`]: latestVersion } }, - }, - }, - ], - }, - })), - { - bool: { - must_not: { - term: { - coreMigrationVersion: kibanaVersion, - }, - }, - }, - }, - ], - }, - }, - }, - index, - }); - - assertResponseIncludeAllShards(body); - - return body.count === 0; - } catch (e) { - // retry for Service Unavailable - if (e.status !== 503 || retryCount === 0) { - throw e; - } - - await new Promise((r) => setTimeout(r, 1000)); - - return await migrationsUpToDate(client, index, migrationVersion, kibanaVersion, retryCount - 1); - } -} - -export async function createIndex( - client: MigrationEsClient, - index: string, - mappings?: IndexMapping -) { - await client.indices.create({ - body: { mappings, settings }, - index, - }); -} - -/** - * Converts an index to an alias. The `alias` parameter is the desired alias name which currently - * is a concrete index. This function will reindex `alias` into a new index, delete the `alias` - * index, and then create an alias `alias` that points to the new index. - * - * @param client - The ElasticSearch connection - * @param info - Information about the mappings and name of the new index - * @param alias - The name of the index being converted to an alias - */ -export async function convertToAlias( - client: MigrationEsClient, - info: FullIndexInfo, - alias: string, - batchSize: number, - script?: string -) { - await client.indices.create({ - body: { mappings: info.mappings, settings }, - index: info.indexName, - }); - - await reindex(client, alias, info.indexName, batchSize, script); - - await claimAlias(client, info.indexName, alias, [{ remove_index: { index: alias } }]); -} - -/** - * Points the specified alias to the specified index. This is an exclusive - * alias, meaning that it will only point to one index at a time, so we - * remove any other indices from the alias. - * - * @param {CallCluster} client - * @param {string} index - * @param {string} alias - * @param {AliasAction[]} aliasActions - Optional actions to be added to the updateAliases call - */ -export async function claimAlias( - client: MigrationEsClient, - index: string, - alias: string, - aliasActions: AliasAction[] = [] -) { - const { body, statusCode } = await client.indices.getAlias({ name: alias }, { ignore: [404] }); - const aliasInfo = statusCode === 404 ? {} : body; - const removeActions = Object.keys(aliasInfo).map((key) => ({ remove: { index: key, alias } })); - - await client.indices.updateAliases({ - body: { - actions: aliasActions.concat(removeActions).concat({ add: { index, alias } }), - }, - }); - - await client.indices.refresh({ index }); -} - -/** - * This is a rough check to ensure that the index being migrated satisfies at least - * some rudimentary expectations. Past Kibana indices had multiple root documents, etc - * and the migration system does not (yet?) handle those indices. They need to be upgraded - * via v5 -> v6 upgrade tools first. This file contains index-agnostic logic, and this - * check is itself index-agnostic, though the error hint is a bit Kibana specific. - * - * @param {FullIndexInfo} indexInfo - */ -function assertIsSupportedIndex(indexInfo: FullIndexInfo) { - const mappings = indexInfo.mappings as any; - const isV7Index = !!mappings.properties; - - if (!isV7Index) { - throw new Error( - `Index ${indexInfo.indexName} belongs to a version of Kibana ` + - `that cannot be automatically migrated. Reset it or use the X-Pack upgrade assistant.` - ); - } - - return indexInfo; -} - -/** - * Provides protection against reading/re-indexing against an index with missing - * shards which could result in data loss. This shouldn't be common, as the Saved - * Object indices should only ever have a single shard. This is more to handle - * instances where customers manually expand the shards of an index. - */ -function assertResponseIncludeAllShards({ _shards }: { _shards: estypes.ShardStatistics }) { - if (!_.has(_shards, 'total') || !_.has(_shards, 'successful')) { - return; - } - - const failed = _shards.total - _shards.successful; - - if (failed > 0) { - throw new Error( - `Re-index failed :: ${failed} of ${_shards.total} shards failed. ` + - `Check Elasticsearch cluster health for more information.` - ); - } -} - -/** - * Reindexes from source to dest, polling for the reindex completion. - */ -async function reindex( - client: MigrationEsClient, - source: string, - dest: string, - batchSize: number, - script?: string -) { - // We poll instead of having the request wait for completion, as for large indices, - // the request times out on the Elasticsearch side of things. We have a relatively tight - // polling interval, as the request is fairly efficient, and we don't - // want to block index migrations for too long on this. - const pollInterval = 250; - const { body: reindexBody } = await client.reindex({ - body: { - dest: { index: dest }, - source: { index: source, size: batchSize }, - script: script - ? { - source: script, - lang: 'painless', - } - : undefined, - }, - refresh: true, - wait_for_completion: false, - }); - - const task = reindexBody.task; - - let completed = false; - - while (!completed) { - await new Promise((r) => setTimeout(r, pollInterval)); - - const { body } = await client.tasks.get({ - task_id: String(task), - }); - - // @ts-expect-error @elastic/elasticsearch GetTaskResponse doesn't contain `error` property - const e = body.error; - if (e) { - throw new Error(`Re-index failed [${e.type}] ${e.reason} :: ${JSON.stringify(e)}`); - } - - completed = body.completed; - } -} diff --git a/src/core/server/saved_objects/migrations/core/index.ts b/src/core/server/saved_objects/migrations/core/index.ts index 1e51983a0ffbd..1aa488f96bd5c 100644 --- a/src/core/server/saved_objects/migrations/core/index.ts +++ b/src/core/server/saved_objects/migrations/core/index.ts @@ -7,11 +7,7 @@ */ export { DocumentMigrator } from './document_migrator'; -export { IndexMigrator } from './index_migrator'; export { buildActiveMappings } from './build_active_mappings'; -export type { CallCluster } from './call_cluster'; export type { LogFn, SavedObjectsMigrationLogger } from './migration_logger'; -export type { MigrationResult, MigrationStatus } from './migration_coordinator'; -export { createMigrationEsClient } from './migration_es_client'; -export type { MigrationEsClient } from './migration_es_client'; +export type { MigrationResult, MigrationStatus } from './types'; export { excludeUnusedTypesQuery } from './elastic_index'; diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts deleted file mode 100644 index fcc03f363139b..0000000000000 --- a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import _ from 'lodash'; -import type { estypes } from '@elastic/elasticsearch'; -import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; -import { SavedObjectUnsanitizedDoc, SavedObjectsSerializer } from '../../serialization'; -import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; -import { IndexMigrator } from './index_migrator'; -import { MigrationOpts } from './migration_context'; -import { loggingSystemMock } from '../../../logging/logging_system.mock'; - -describe('IndexMigrator', () => { - let testOpts: jest.Mocked & { - client: ReturnType; - }; - - beforeEach(() => { - testOpts = { - batchSize: 10, - client: elasticsearchClientMock.createElasticsearchClient(), - index: '.kibana', - kibanaVersion: '7.10.0', - log: loggingSystemMock.create().get(), - setStatus: jest.fn(), - mappingProperties: {}, - pollInterval: 1, - scrollDuration: '1m', - documentMigrator: { - migrationVersion: {}, - migrate: _.identity, - migrateAndConvert: _.identity, - prepareMigrations: jest.fn(), - }, - serializer: new SavedObjectsSerializer(new SavedObjectTypeRegistry()), - }; - }); - - test('creates the index if it does not exist', async () => { - const { client } = testOpts; - - testOpts.mappingProperties = { foo: { type: 'long' } as any }; - - withIndex(client, { index: { statusCode: 404 }, alias: { statusCode: 404 } }); - - await new IndexMigrator(testOpts).migrate(); - - expect(client.indices.create).toHaveBeenCalledWith({ - body: { - mappings: { - dynamic: 'strict', - _meta: { - migrationMappingPropertyHashes: { - foo: '18c78c995965207ed3f6e7fc5c6e55fe', - migrationVersion: '4a1746014a75ade3a714e1db5763276f', - namespace: '2f4316de49999235636386fe51dc06c1', - namespaces: '2f4316de49999235636386fe51dc06c1', - originId: '2f4316de49999235636386fe51dc06c1', - references: '7997cf5a56cc02bdc9c93361bde732b0', - coreMigrationVersion: '2f4316de49999235636386fe51dc06c1', - type: '2f4316de49999235636386fe51dc06c1', - updated_at: '00da57df13e94e9d98437d13ace4bfe0', - }, - }, - properties: { - foo: { type: 'long' }, - migrationVersion: { dynamic: 'true', type: 'object' }, - namespace: { type: 'keyword' }, - namespaces: { type: 'keyword' }, - originId: { type: 'keyword' }, - type: { type: 'keyword' }, - updated_at: { type: 'date' }, - references: { - type: 'nested', - properties: { - name: { type: 'keyword' }, - type: { type: 'keyword' }, - id: { type: 'keyword' }, - }, - }, - coreMigrationVersion: { type: 'keyword' }, - }, - }, - settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, - }, - index: '.kibana_1', - }); - }); - - test('returns stats about the migration', async () => { - const { client } = testOpts; - - withIndex(client, { index: { statusCode: 404 }, alias: { statusCode: 404 } }); - - const result = await new IndexMigrator(testOpts).migrate(); - - expect(result).toMatchObject({ - destIndex: '.kibana_1', - sourceIndex: '.kibana', - status: 'migrated', - }); - }); - - test('fails if there are multiple root doc types', async () => { - const { client } = testOpts; - - withIndex(client, { - index: { - '.kibana_1': { - aliases: {}, - mappings: { - foo: { properties: {} }, - doc: { - properties: { - author: { type: 'text' }, - }, - }, - }, - }, - }, - }); - - await expect(new IndexMigrator(testOpts).migrate()).rejects.toThrow( - /use the X-Pack upgrade assistant/ - ); - }); - - test('fails if root doc type is not "doc"', async () => { - const { client } = testOpts; - - withIndex(client, { - index: { - '.kibana_1': { - aliases: {}, - mappings: { - poc: { - properties: { - author: { type: 'text' }, - }, - }, - }, - }, - }, - }); - - await expect(new IndexMigrator(testOpts).migrate()).rejects.toThrow( - /use the X-Pack upgrade assistant/ - ); - }); - - test('retains unknown core field mappings from the previous index', async () => { - const { client } = testOpts; - - testOpts.mappingProperties = { foo: { type: 'text' } as any }; - - withIndex(client, { - index: { - '.kibana_1': { - aliases: {}, - mappings: { - properties: { - unknown_core_field: { type: 'text' }, - }, - }, - }, - }, - }); - - await new IndexMigrator(testOpts).migrate(); - - expect(client.indices.create).toHaveBeenCalledWith({ - body: { - mappings: { - dynamic: 'strict', - _meta: { - migrationMappingPropertyHashes: { - foo: '625b32086eb1d1203564cf85062dd22e', - migrationVersion: '4a1746014a75ade3a714e1db5763276f', - namespace: '2f4316de49999235636386fe51dc06c1', - namespaces: '2f4316de49999235636386fe51dc06c1', - originId: '2f4316de49999235636386fe51dc06c1', - references: '7997cf5a56cc02bdc9c93361bde732b0', - coreMigrationVersion: '2f4316de49999235636386fe51dc06c1', - type: '2f4316de49999235636386fe51dc06c1', - updated_at: '00da57df13e94e9d98437d13ace4bfe0', - }, - }, - properties: { - unknown_core_field: { type: 'text' }, - foo: { type: 'text' }, - migrationVersion: { dynamic: 'true', type: 'object' }, - namespace: { type: 'keyword' }, - namespaces: { type: 'keyword' }, - originId: { type: 'keyword' }, - type: { type: 'keyword' }, - updated_at: { type: 'date' }, - references: { - type: 'nested', - properties: { - name: { type: 'keyword' }, - type: { type: 'keyword' }, - id: { type: 'keyword' }, - }, - }, - coreMigrationVersion: { type: 'keyword' }, - }, - }, - settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, - }, - index: '.kibana_2', - }); - }); - - test('disables complex field mappings from unknown types in the previous index', async () => { - const { client } = testOpts; - - testOpts.mappingProperties = { foo: { type: 'text' } as any }; - - withIndex(client, { - index: { - '.kibana_1': { - aliases: {}, - mappings: { - properties: { - unknown_complex_field: { properties: { description: { type: 'text' } } }, - }, - }, - }, - }, - }); - - await new IndexMigrator(testOpts).migrate(); - - expect(client.indices.create).toHaveBeenCalledWith({ - body: { - mappings: { - dynamic: 'strict', - _meta: { - migrationMappingPropertyHashes: { - foo: '625b32086eb1d1203564cf85062dd22e', - migrationVersion: '4a1746014a75ade3a714e1db5763276f', - namespace: '2f4316de49999235636386fe51dc06c1', - namespaces: '2f4316de49999235636386fe51dc06c1', - originId: '2f4316de49999235636386fe51dc06c1', - references: '7997cf5a56cc02bdc9c93361bde732b0', - coreMigrationVersion: '2f4316de49999235636386fe51dc06c1', - type: '2f4316de49999235636386fe51dc06c1', - updated_at: '00da57df13e94e9d98437d13ace4bfe0', - }, - }, - properties: { - unknown_complex_field: { dynamic: false, properties: {} }, - foo: { type: 'text' }, - migrationVersion: { dynamic: 'true', type: 'object' }, - namespace: { type: 'keyword' }, - namespaces: { type: 'keyword' }, - originId: { type: 'keyword' }, - type: { type: 'keyword' }, - updated_at: { type: 'date' }, - references: { - type: 'nested', - properties: { - name: { type: 'keyword' }, - type: { type: 'keyword' }, - id: { type: 'keyword' }, - }, - }, - coreMigrationVersion: { type: 'keyword' }, - }, - }, - settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, - }, - index: '.kibana_2', - }); - }); - - test('points the alias at the dest index', async () => { - const { client } = testOpts; - - withIndex(client, { index: { statusCode: 404 }, alias: { statusCode: 404 } }); - - await new IndexMigrator(testOpts).migrate(); - - expect(client.indices.create).toHaveBeenCalledWith(expect.any(Object)); - expect(client.indices.updateAliases).toHaveBeenCalledWith({ - body: { actions: [{ add: { alias: '.kibana', index: '.kibana_1' } }] }, - }); - }); - - test('removes previous indices from the alias', async () => { - const { client } = testOpts; - - testOpts.documentMigrator.migrationVersion = { - dashboard: '2.4.5', - }; - - withIndex(client, { numOutOfDate: 1 }); - - await new IndexMigrator(testOpts).migrate(); - - expect(client.indices.create).toHaveBeenCalledWith(expect.any(Object)); - expect(client.indices.updateAliases).toHaveBeenCalledWith({ - body: { - actions: [ - { remove: { alias: '.kibana', index: '.kibana_1' } }, - { add: { alias: '.kibana', index: '.kibana_2' } }, - ], - }, - }); - }); - - test('transforms all docs from the original index', async () => { - let count = 0; - const { client } = testOpts; - const migrateAndConvertDoc = jest.fn((doc: SavedObjectUnsanitizedDoc) => { - return [{ ...doc, attributes: { name: ++count } }]; - }); - - testOpts.documentMigrator = { - migrationVersion: { foo: '1.2.3' }, - migrate: jest.fn(), - migrateAndConvert: migrateAndConvertDoc, - prepareMigrations: jest.fn(), - }; - - withIndex(client, { - numOutOfDate: 1, - docs: [ - [{ _id: 'foo:1', _source: { type: 'foo', foo: { name: 'Bar' } } }], - [{ _id: 'foo:2', _source: { type: 'foo', foo: { name: 'Baz' } } }], - ], - }); - - await new IndexMigrator(testOpts).migrate(); - - expect(count).toEqual(2); - expect(migrateAndConvertDoc).toHaveBeenNthCalledWith(1, { - id: '1', - type: 'foo', - attributes: { name: 'Bar' }, - migrationVersion: {}, - references: [], - }); - expect(migrateAndConvertDoc).toHaveBeenNthCalledWith(2, { - id: '2', - type: 'foo', - attributes: { name: 'Baz' }, - migrationVersion: {}, - references: [], - }); - - expect(client.bulk).toHaveBeenCalledTimes(2); - expect(client.bulk).toHaveBeenNthCalledWith(1, { - body: [ - { index: { _id: 'foo:1', _index: '.kibana_2' } }, - { foo: { name: 1 }, type: 'foo', migrationVersion: {}, references: [] }, - ], - }); - expect(client.bulk).toHaveBeenNthCalledWith(2, { - body: [ - { index: { _id: 'foo:2', _index: '.kibana_2' } }, - { foo: { name: 2 }, type: 'foo', migrationVersion: {}, references: [] }, - ], - }); - }); - - test('rejects when the migration function throws an error', async () => { - const { client } = testOpts; - const migrateAndConvertDoc = jest.fn((doc: SavedObjectUnsanitizedDoc) => { - throw new Error('error migrating document'); - }); - - testOpts.documentMigrator = { - migrationVersion: { foo: '1.2.3' }, - migrate: jest.fn(), - migrateAndConvert: migrateAndConvertDoc, - prepareMigrations: jest.fn(), - }; - - withIndex(client, { - numOutOfDate: 1, - docs: [ - [{ _id: 'foo:1', _source: { type: 'foo', foo: { name: 'Bar' } } }], - [{ _id: 'foo:2', _source: { type: 'foo', foo: { name: 'Baz' } } }], - ], - }); - - await expect(new IndexMigrator(testOpts).migrate()).rejects.toThrowErrorMatchingInlineSnapshot( - `"error migrating document"` - ); - }); -}); - -function withIndex( - client: ReturnType, - opts: any = {} -) { - const defaultIndex = { - '.kibana_1': { - aliases: { '.kibana': {} }, - mappings: { - dynamic: 'strict', - properties: { - migrationVersion: { dynamic: 'true', type: 'object' }, - }, - }, - }, - }; - const defaultAlias = { - '.kibana_1': {}, - }; - const { numOutOfDate = 0 } = opts; - const { alias = defaultAlias } = opts; - const { index = defaultIndex } = opts; - const { docs = [] } = opts; - const searchResult = (i: number) => ({ - _scroll_id: i, - _shards: { - successful: 1, - total: 1, - }, - hits: { - hits: docs[i] || [], - }, - }); - - let scrollCallCounter = 1; - - client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(index, { - statusCode: index.statusCode, - }) - ); - client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(alias, { - statusCode: index.statusCode, - }) - ); - client.reindex.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - task: 'zeid', - _shards: { successful: 1, total: 1 }, - } as estypes.ReindexResponse) - ); - client.tasks.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - completed: true, - } as estypes.GetTaskResponse) - ); - client.search.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(searchResult(0) as any) - ); - client.bulk.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - items: [] as any[], - } as estypes.BulkResponse) - ); - client.count.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - count: numOutOfDate, - _shards: { successful: 1, total: 1 }, - } as estypes.CountResponse) - ); - // @ts-expect-error - client.scroll.mockImplementation(() => { - if (scrollCallCounter <= docs.length) { - const result = searchResult(scrollCallCounter); - scrollCallCounter++; - return elasticsearchClientMock.createSuccessTransportRequestPromise(result); - } - return elasticsearchClientMock.createSuccessTransportRequestPromise({}); - }); -} diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.ts b/src/core/server/saved_objects/migrations/core/index_migrator.ts deleted file mode 100644 index 14dba1db9b624..0000000000000 --- a/src/core/server/saved_objects/migrations/core/index_migrator.ts +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { diffMappings } from './build_active_mappings'; -import * as Index from './elastic_index'; -import { migrateRawDocs } from './migrate_raw_docs'; -import { Context, migrationContext, MigrationOpts } from './migration_context'; -import { coordinateMigration, MigrationResult } from './migration_coordinator'; - -/* - * Core logic for migrating the mappings and documents in an index. - */ -export class IndexMigrator { - private opts: MigrationOpts; - - /** - * Creates an instance of IndexMigrator. - * - * @param {MigrationOpts} opts - */ - constructor(opts: MigrationOpts) { - this.opts = opts; - } - - /** - * Migrates the index, or, if another Kibana instance appears to be running the migration, - * waits for the migration to complete. - * - * @returns {Promise} - */ - public async migrate(): Promise { - const context = await migrationContext(this.opts); - - return coordinateMigration({ - log: context.log, - - pollInterval: context.pollInterval, - - setStatus: context.setStatus, - - async isMigrated() { - return !(await requiresMigration(context)); - }, - - async runMigration() { - if (await requiresMigration(context)) { - return migrateIndex(context); - } - - return { status: 'skipped' }; - }, - }); - } -} - -/** - * Determines what action the migration system needs to take (none, patch, migrate). - */ -async function requiresMigration(context: Context): Promise { - const { client, alias, documentMigrator, dest, kibanaVersion, log } = context; - - // Have all of our known migrations been run against the index? - const hasMigrations = await Index.migrationsUpToDate( - client, - alias, - documentMigrator.migrationVersion, - kibanaVersion - ); - - if (!hasMigrations) { - return true; - } - - // Is our index aliased? - const refreshedSource = await Index.fetchInfo(client, alias); - - if (!refreshedSource.aliases[alias]) { - return true; - } - - // Do the actual index mappings match our expectations? - const diffResult = diffMappings(refreshedSource.mappings, dest.mappings); - - if (diffResult) { - log.info(`Detected mapping change in "${diffResult.changedProp}"`); - - return true; - } - - return false; -} - -/** - * Performs an index migration if the source index exists, otherwise - * this simply creates the dest index with the proper mappings. - */ -async function migrateIndex(context: Context): Promise { - const startTime = Date.now(); - const { client, alias, source, dest, log } = context; - - await deleteIndexTemplates(context); - - log.info(`Creating index ${dest.indexName}.`); - - await Index.createIndex(client, dest.indexName, dest.mappings); - - await migrateSourceToDest(context); - - log.info(`Pointing alias ${alias} to ${dest.indexName}.`); - - await Index.claimAlias(client, dest.indexName, alias); - - const result: MigrationResult = { - status: 'migrated', - destIndex: dest.indexName, - sourceIndex: source.indexName, - elapsedMs: Date.now() - startTime, - }; - - log.info(`Finished in ${result.elapsedMs}ms.`); - - return result; -} - -/** - * If the obsoleteIndexTemplatePattern option is specified, this will delete any index templates - * that match it. - */ -async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern }: Context) { - if (!obsoleteIndexTemplatePattern) { - return; - } - - const { body: templates } = await client.cat.templates({ - format: 'json', - name: obsoleteIndexTemplatePattern, - }); - - if (!templates.length) { - return; - } - - const templateNames = templates.map((t) => t.name); - - log.info(`Removing index templates: ${templateNames}`); - - return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! }))); -} - -/** - * Moves all docs from sourceIndex to destIndex, migrating each as necessary. - * This moves documents from the concrete index, rather than the alias, to prevent - * a situation where the alias moves out from under us as we're migrating docs. - */ -async function migrateSourceToDest(context: Context) { - const { client, alias, dest, source, batchSize } = context; - const { scrollDuration, documentMigrator, log, serializer } = context; - - if (!source.exists) { - return; - } - - if (!source.aliases[alias]) { - log.info(`Reindexing ${alias} to ${source.indexName}`); - - await Index.convertToAlias(client, source, alias, batchSize, context.convertToAliasScript); - } - - const read = Index.reader(client, source.indexName, { batchSize, scrollDuration }); - - log.info(`Migrating ${source.indexName} saved objects to ${dest.indexName}`); - - while (true) { - const docs = await read(); - - if (!docs || !docs.length) { - return; - } - - log.debug(`Migrating saved objects ${docs.map((d) => d._id).join(', ')}`); - - await Index.write( - client, - dest.indexName, - await migrateRawDocs( - serializer, - documentMigrator.migrateAndConvert, - // @ts-expect-error @elastic/elasticsearch `Hit._id` may be a string | number in ES, but we always expect strings in the SO index. - docs - ) - ); - } -} diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts index d7f7aff45a470..7f63b30647ac5 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.ts @@ -6,124 +6,7 @@ * Side Public License, v 1. */ -/** - * The MigrationOpts interface defines the minimum set of data required - * in order to properly migrate an index. MigrationContext expands this - * with computed values and values from the index being migrated, and is - * serves as a central blueprint for what migrations will end up doing. - */ - -import { Logger } from '../../../logging'; -import { MigrationEsClient } from './migration_es_client'; -import { SavedObjectsSerializer } from '../../serialization'; -import { - SavedObjectsTypeMappingDefinitions, - SavedObjectsMappingProperties, - IndexMapping, -} from '../../mappings'; -import { buildActiveMappings } from './build_active_mappings'; -import { VersionedTransformer } from './document_migrator'; -import * as Index from './elastic_index'; -import { SavedObjectsMigrationLogger, MigrationLogger } from './migration_logger'; -import { KibanaMigratorStatus } from '../kibana'; - -export interface MigrationOpts { - batchSize: number; - pollInterval: number; - scrollDuration: string; - client: MigrationEsClient; - index: string; - kibanaVersion: string; - log: Logger; - setStatus: (status: KibanaMigratorStatus) => void; - mappingProperties: SavedObjectsTypeMappingDefinitions; - documentMigrator: VersionedTransformer; - serializer: SavedObjectsSerializer; - convertToAliasScript?: string; - - /** - * If specified, templates matching the specified pattern will be removed - * prior to running migrations. For example: 'kibana_index_template*' - */ - obsoleteIndexTemplatePattern?: string; -} - -/** - * @internal - */ -export interface Context { - client: MigrationEsClient; - alias: string; - source: Index.FullIndexInfo; - dest: Index.FullIndexInfo; - documentMigrator: VersionedTransformer; - kibanaVersion: string; - log: SavedObjectsMigrationLogger; - setStatus: (status: KibanaMigratorStatus) => void; - batchSize: number; - pollInterval: number; - scrollDuration: string; - serializer: SavedObjectsSerializer; - obsoleteIndexTemplatePattern?: string; - convertToAliasScript?: string; -} - -/** - * Builds up an uber object which has all of the config options, settings, - * and various info needed to migrate the source index. - */ -export async function migrationContext(opts: MigrationOpts): Promise { - const { log, client, setStatus } = opts; - const alias = opts.index; - const source = createSourceContext(await Index.fetchInfo(client, alias), alias); - const dest = createDestContext(source, alias, opts.mappingProperties); - - return { - client, - alias, - source, - dest, - kibanaVersion: opts.kibanaVersion, - log: new MigrationLogger(log), - setStatus, - batchSize: opts.batchSize, - documentMigrator: opts.documentMigrator, - pollInterval: opts.pollInterval, - scrollDuration: opts.scrollDuration, - serializer: opts.serializer, - obsoleteIndexTemplatePattern: opts.obsoleteIndexTemplatePattern, - convertToAliasScript: opts.convertToAliasScript, - }; -} - -function createSourceContext(source: Index.FullIndexInfo, alias: string) { - if (source.exists && source.indexName === alias) { - return { - ...source, - indexName: nextIndexName(alias, alias), - }; - } - - return source; -} - -function createDestContext( - source: Index.FullIndexInfo, - alias: string, - typeMappingDefinitions: SavedObjectsTypeMappingDefinitions -): Index.FullIndexInfo { - const targetMappings = disableUnknownTypeMappingFields( - buildActiveMappings(typeMappingDefinitions), - source.mappings - ); - - return { - aliases: {}, - exists: false, - indexName: nextIndexName(source.indexName, alias), - mappings: targetMappings, - }; -} +import { SavedObjectsMappingProperties, IndexMapping } from '../../mappings'; /** * Merges the active mappings and the source mappings while disabling the @@ -175,14 +58,3 @@ export function disableUnknownTypeMappingFields( }, }; } - -/** - * Gets the next index name in a sequence, based on specified current index's info. - * We're using a numeric counter to create new indices. So, `.kibana_1`, `.kibana_2`, etc - * There are downsides to this, but it seemed like a simple enough approach. - */ -function nextIndexName(indexName: string, alias: string) { - const indexSuffix = (indexName.match(/[\d]+$/) || [])[0]; - const indexNum = parseInt(indexSuffix, 10) || 0; - return `${alias}_${indexNum + 1}`; -} diff --git a/src/core/server/saved_objects/migrations/core/migration_coordinator.test.ts b/src/core/server/saved_objects/migrations/core/migration_coordinator.test.ts deleted file mode 100644 index 63476a15d77cd..0000000000000 --- a/src/core/server/saved_objects/migrations/core/migration_coordinator.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { coordinateMigration } from './migration_coordinator'; -import { createSavedObjectsMigrationLoggerMock } from '../mocks'; - -describe('coordinateMigration', () => { - const log = createSavedObjectsMigrationLoggerMock(); - - test('waits for isMigrated, if there is an index conflict', async () => { - const pollInterval = 1; - const runMigration = jest.fn(() => { - // eslint-disable-next-line no-throw-literal - throw { body: { error: { index: '.foo', type: 'resource_already_exists_exception' } } }; - }); - const isMigrated = jest.fn(); - const setStatus = jest.fn(); - - isMigrated.mockResolvedValueOnce(false).mockResolvedValueOnce(true); - - await coordinateMigration({ - log, - runMigration, - pollInterval, - isMigrated, - setStatus, - }); - - expect(runMigration).toHaveBeenCalledTimes(1); - expect(isMigrated).toHaveBeenCalledTimes(2); - const warnings = log.warning.mock.calls.filter((msg: any) => /deleting index \.foo/.test(msg)); - expect(warnings.length).toEqual(1); - }); - - test('does not poll if the runMigration succeeds', async () => { - const pollInterval = 1; - const runMigration = jest.fn(() => Promise.resolve()); - const isMigrated = jest.fn(() => Promise.resolve(true)); - const setStatus = jest.fn(); - - await coordinateMigration({ - log, - runMigration, - pollInterval, - isMigrated, - setStatus, - }); - expect(isMigrated).not.toHaveBeenCalled(); - }); - - test('does not swallow exceptions', async () => { - const pollInterval = 1; - const runMigration = jest.fn(() => { - throw new Error('Doh'); - }); - const isMigrated = jest.fn(() => Promise.resolve(true)); - const setStatus = jest.fn(); - - await expect( - coordinateMigration({ - log, - runMigration, - pollInterval, - isMigrated, - setStatus, - }) - ).rejects.toThrow(/Doh/); - expect(isMigrated).not.toHaveBeenCalled(); - }); -}); diff --git a/src/core/server/saved_objects/migrations/core/migration_coordinator.ts b/src/core/server/saved_objects/migrations/core/migration_coordinator.ts deleted file mode 100644 index 5b99f050b0ece..0000000000000 --- a/src/core/server/saved_objects/migrations/core/migration_coordinator.ts +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/* - * This provides a mechanism for preventing multiple Kibana instances from - * simultaneously running migrations on the same index. It synchronizes this - * by handling index creation conflicts, and putting this instance into a - * poll loop that periodically checks to see if the index is migrated. - * - * The reason we have to coordinate this, rather than letting each Kibana instance - * perform duplicate work, is that if we allowed each Kibana to simply run migrations in - * parallel, they would each try to reindex and each try to create the destination index. - * If those indices already exist, it may be due to contention between multiple Kibana - * instances (which is safe to ignore), but it may be due to a partially completed migration, - * or someone tampering with the Kibana alias. In these cases, it's not clear that we should - * just migrate data into an existing index. Such an action could result in data loss. Instead, - * we should probably fail, and the Kibana sys-admin should clean things up before relaunching - * Kibana. - */ - -import _ from 'lodash'; -import { KibanaMigratorStatus } from '../kibana'; -import { SavedObjectsMigrationLogger } from './migration_logger'; - -const DEFAULT_POLL_INTERVAL = 15000; - -export type MigrationStatus = - | 'waiting_to_start' - | 'waiting_for_other_nodes' - | 'running' - | 'completed'; - -export type MigrationResult = - | { status: 'skipped' } - | { status: 'patched' } - | { - status: 'migrated'; - destIndex: string; - sourceIndex: string; - elapsedMs: number; - }; - -interface Opts { - runMigration: () => Promise; - isMigrated: () => Promise; - setStatus: (status: KibanaMigratorStatus) => void; - log: SavedObjectsMigrationLogger; - pollInterval?: number; -} - -/** - * Runs the migration specified by opts. If the migration fails due to an index - * creation conflict, this falls into a polling loop, checking every pollInterval - * milliseconds if the index is migrated. - * - * @export - * @param {Opts} opts - * @prop {Migration} runMigration - A function that runs the index migration - * @prop {IsMigrated} isMigrated - A function which checks if the index is already migrated - * @prop {Logger} log - The migration logger - * @prop {number} pollInterval - How often, in ms, to check that the index is migrated - * @returns - */ -export async function coordinateMigration(opts: Opts): Promise { - try { - return await opts.runMigration(); - } catch (error) { - const waitingIndex = handleIndexExists(error, opts.log); - if (waitingIndex) { - opts.setStatus({ status: 'waiting_for_other_nodes', waitingIndex }); - await waitForMigration(opts.isMigrated, opts.pollInterval); - return { status: 'skipped' }; - } - throw error; - } -} - -/** - * If the specified error is an index exists error, this logs a warning, - * and is the cue for us to fall into a polling loop, waiting for some - * other Kibana instance to complete the migration. - */ -function handleIndexExists(error: any, log: SavedObjectsMigrationLogger): string | undefined { - const isIndexExistsError = - _.get(error, 'body.error.type') === 'resource_already_exists_exception'; - if (!isIndexExistsError) { - return undefined; - } - - const index = _.get(error, 'body.error.index'); - - log.warning( - `Another Kibana instance appears to be migrating the index. Waiting for ` + - `that migration to complete. If no other Kibana instance is attempting ` + - `migrations, you can get past this message by deleting index ${index} and ` + - `restarting Kibana.` - ); - - return index; -} - -/** - * Polls isMigrated every pollInterval milliseconds until it returns true. - */ -async function waitForMigration( - isMigrated: () => Promise, - pollInterval = DEFAULT_POLL_INTERVAL -) { - while (true) { - if (await isMigrated()) { - return; - } - await sleep(pollInterval); - } -} - -function sleep(ms: number) { - return new Promise((r) => setTimeout(r, ms)); -} diff --git a/src/core/server/saved_objects/migrations/core/migration_es_client.test.ts b/src/core/server/saved_objects/migrations/core/migration_es_client.test.ts deleted file mode 100644 index 75dbdf55e55fc..0000000000000 --- a/src/core/server/saved_objects/migrations/core/migration_es_client.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { migrationRetryCallClusterMock } from './migration_es_client.test.mock'; - -import { createMigrationEsClient, MigrationEsClient } from './migration_es_client'; -import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; -import { loggerMock } from '../../../logging/logger.mock'; -import { SavedObjectsErrorHelpers } from '../../service/lib/errors'; - -describe('MigrationEsClient', () => { - let client: ReturnType; - let migrationEsClient: MigrationEsClient; - - beforeEach(() => { - client = elasticsearchClientMock.createElasticsearchClient(); - migrationEsClient = createMigrationEsClient(client, loggerMock.create()); - migrationRetryCallClusterMock.mockClear(); - }); - - it('delegates call to ES client method', async () => { - expect(migrationEsClient.bulk).toStrictEqual(expect.any(Function)); - await migrationEsClient.bulk({ body: [] }); - expect(client.bulk).toHaveBeenCalledTimes(1); - }); - - it('wraps a method call in migrationRetryCallClusterMock', async () => { - await migrationEsClient.bulk({ body: [] }); - expect(migrationRetryCallClusterMock).toHaveBeenCalledTimes(1); - }); - - it('sets maxRetries: 0 to delegate retry logic to migrationRetryCallCluster', async () => { - expect(migrationEsClient.bulk).toStrictEqual(expect.any(Function)); - await migrationEsClient.bulk({ body: [] }); - expect(client.bulk).toHaveBeenCalledWith( - expect.any(Object), - expect.objectContaining({ maxRetries: 0 }) - ); - }); - - it('do not transform elasticsearch errors into saved objects errors', async () => { - expect.assertions(1); - client.bulk = jest.fn().mockRejectedValue(new Error('reason')); - try { - await migrationEsClient.bulk({ body: [] }); - } catch (e) { - expect(SavedObjectsErrorHelpers.isSavedObjectsClientError(e)).toBe(false); - } - }); -}); diff --git a/src/core/server/saved_objects/migrations/core/migration_es_client.ts b/src/core/server/saved_objects/migrations/core/migration_es_client.ts deleted file mode 100644 index e8dc9c94b7861..0000000000000 --- a/src/core/server/saved_objects/migrations/core/migration_es_client.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { TransportRequestOptions } from '@elastic/elasticsearch/lib/Transport'; -import { get } from 'lodash'; -import { set } from '@elastic/safer-lodash-set'; - -import { ElasticsearchClient } from '../../../elasticsearch'; -import { migrationRetryCallCluster } from '../../../elasticsearch/client/retry_call_cluster'; -import { Logger } from '../../../logging'; - -const methods = [ - 'bulk', - 'cat.templates', - 'clearScroll', - 'count', - 'indices.create', - 'indices.deleteTemplate', - 'indices.get', - 'indices.getAlias', - 'indices.refresh', - 'indices.updateAliases', - 'reindex', - 'search', - 'scroll', - 'tasks.get', -] as const; - -type MethodName = typeof methods[number]; - -export interface MigrationEsClient { - bulk: ElasticsearchClient['bulk']; - cat: { - templates: ElasticsearchClient['cat']['templates']; - }; - clearScroll: ElasticsearchClient['clearScroll']; - count: ElasticsearchClient['count']; - indices: { - create: ElasticsearchClient['indices']['create']; - delete: ElasticsearchClient['indices']['delete']; - deleteTemplate: ElasticsearchClient['indices']['deleteTemplate']; - get: ElasticsearchClient['indices']['get']; - getAlias: ElasticsearchClient['indices']['getAlias']; - refresh: ElasticsearchClient['indices']['refresh']; - updateAliases: ElasticsearchClient['indices']['updateAliases']; - }; - reindex: ElasticsearchClient['reindex']; - search: ElasticsearchClient['search']; - scroll: ElasticsearchClient['scroll']; - tasks: { - get: ElasticsearchClient['tasks']['get']; - }; -} - -export function createMigrationEsClient( - client: ElasticsearchClient, - log: Logger, - delay?: number -): MigrationEsClient { - return methods.reduce((acc: MigrationEsClient, key: MethodName) => { - set(acc, key, async (params?: unknown, options?: TransportRequestOptions) => { - const fn = get(client, key); - if (!fn) { - throw new Error(`unknown ElasticsearchClient client method [${key}]`); - } - return await migrationRetryCallCluster( - () => fn.call(client, params, { maxRetries: 0, ...options }), - log, - delay - ); - }); - return acc; - }, {} as MigrationEsClient); -} diff --git a/src/core/server/saved_objects/migrations/core/migration_es_client.test.mock.ts b/src/core/server/saved_objects/migrations/core/types.ts similarity index 53% rename from src/core/server/saved_objects/migrations/core/migration_es_client.test.mock.ts rename to src/core/server/saved_objects/migrations/core/types.ts index 593973ad2e9ba..61985d8f10996 100644 --- a/src/core/server/saved_objects/migrations/core/migration_es_client.test.mock.ts +++ b/src/core/server/saved_objects/migrations/core/types.ts @@ -6,7 +6,18 @@ * Side Public License, v 1. */ -export const migrationRetryCallClusterMock = jest.fn((fn) => fn()); -jest.doMock('../../../elasticsearch/client/retry_call_cluster', () => ({ - migrationRetryCallCluster: migrationRetryCallClusterMock, -})); +export type MigrationStatus = + | 'waiting_to_start' + | 'waiting_for_other_nodes' + | 'running' + | 'completed'; + +export type MigrationResult = + | { status: 'skipped' } + | { status: 'patched' } + | { + status: 'migrated'; + destIndex: string; + sourceIndex: string; + elapsedMs: number; + }; diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts index 530203e659086..5d3bcfc1ca1fa 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts @@ -40,8 +40,6 @@ const createMigrator = ( scrollDuration: '15m', pollInterval: 1500, skip: false, - // TODO migrationsV2: remove/deprecate once we remove migrations v1 - enableV2: false, retryAttempts: 10, }, runMigrations: jest.fn(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index c6dfd2c2d1809..db736bb937818 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -7,7 +7,7 @@ */ import { take } from 'rxjs/operators'; -import { estypes, errors as esErrors } from '@elastic/elasticsearch'; +import { estypes } from '@elastic/elasticsearch'; import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; import { KibanaMigratorOptions, KibanaMigrator } from './kibana_migrator'; @@ -15,6 +15,7 @@ import { loggingSystemMock } from '../../../logging/logging_system.mock'; import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry'; import { SavedObjectsType } from '../../types'; import { DocumentMigrator } from '../core/document_migrator'; + jest.mock('../core/document_migrator', () => { return { // Create a mock for spying on the constructor @@ -147,155 +148,76 @@ describe('KibanaMigrator', () => { expect(options.client.cat.templates).toHaveBeenCalledTimes(1); }); - describe('when enableV2 = false', () => { - it('when enableV2 = false creates an IndexMigrator which retries NoLivingConnectionsError errors from ES client', async () => { - const options = mockOptions(); - - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - options.client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - options.client.indices.create = jest - .fn() - .mockReturnValueOnce( - elasticsearchClientMock.createErrorTransportRequestPromise( - new esErrors.NoLivingConnectionsError('reason', {} as any) - ) - ) - .mockImplementationOnce(() => - elasticsearchClientMock.createSuccessTransportRequestPromise('success') - ); - - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); + beforeEach(() => { + jest.clearAllMocks(); + }); - migrator.prepareMigrations(); - await migrator.runMigrations(); + it('emits results on getMigratorResult$()', async () => { + const options = mockV2MigrationOptions(); + const migrator = new KibanaMigrator(options); + const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); + migrator.prepareMigrations(); + await migrator.runMigrations(); - expect(options.client.indices.create).toHaveBeenCalledTimes(3); - const { status } = await migratorStatus; - return expect(status).toEqual('completed'); + const { status, result } = await migratorStatus; + expect(status).toEqual('completed'); + expect(result![0]).toMatchObject({ + destIndex: '.my-index_8.2.3_001', + sourceIndex: '.my-index_pre8.2.3_001', + elapsedMs: expect.any(Number), + status: 'migrated', }); - - it('emits results on getMigratorResult$()', async () => { - const options = mockOptions(); - - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - options.client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - migrator.prepareMigrations(); - await migrator.runMigrations(); - const { status, result } = await migratorStatus; - expect(status).toEqual('completed'); - expect(result![0]).toMatchObject({ - destIndex: '.my-index_1', - elapsedMs: expect.any(Number), - sourceIndex: '.my-index', - status: 'migrated', - }); - expect(result![1]).toMatchObject({ - destIndex: 'other-index_1', - elapsedMs: expect.any(Number), - sourceIndex: 'other-index', - status: 'migrated', - }); + expect(result![1]).toMatchObject({ + destIndex: 'other-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'patched', }); }); - describe('when enableV2 = true', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('emits results on getMigratorResult$()', async () => { - const options = mockV2MigrationOptions(); - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - migrator.prepareMigrations(); - await migrator.runMigrations(); - - const { status, result } = await migratorStatus; - expect(status).toEqual('completed'); - expect(result![0]).toMatchObject({ - destIndex: '.my-index_8.2.3_001', - sourceIndex: '.my-index_pre8.2.3_001', - elapsedMs: expect.any(Number), - status: 'migrated', - }); - expect(result![1]).toMatchObject({ - destIndex: 'other-index_8.2.3_001', - elapsedMs: expect.any(Number), - status: 'patched', - }); - }); - it('rejects when the migration state machine terminates in a FATAL state', () => { - const options = mockV2MigrationOptions(); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - { - '.my-index_8.2.4_001': { - aliases: { - '.my-index': {}, - '.my-index_8.2.4': {}, - }, - mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } }, - settings: {}, + it('rejects when the migration state machine terminates in a FATAL state', () => { + const options = mockV2MigrationOptions(); + options.client.indices.get.mockReturnValue( + elasticsearchClientMock.createSuccessTransportRequestPromise( + { + '.my-index_8.2.4_001': { + aliases: { + '.my-index': {}, + '.my-index_8.2.4': {}, }, + mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } }, + settings: {}, }, - { statusCode: 200 } - ) - ); + }, + { statusCode: 200 } + ) + ); - const migrator = new KibanaMigrator(options); - migrator.prepareMigrations(); - return expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.my-index] index: The .my-index alias is pointing to a newer version of Kibana: v8.2.4]` - ); - }); - it('rejects when an unexpected exception occurs in an action', async () => { - const options = mockV2MigrationOptions(); - options.client.tasks.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - completed: true, - error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, - failures: [], - task: { description: 'task description' } as any, - }) - ); + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + return expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot( + `[Error: Unable to complete saved object migrations for the [.my-index] index: The .my-index alias is pointing to a newer version of Kibana: v8.2.4]` + ); + }); + it('rejects when an unexpected exception occurs in an action', async () => { + const options = mockV2MigrationOptions(); + options.client.tasks.get.mockReturnValue( + elasticsearchClientMock.createSuccessTransportRequestPromise({ + completed: true, + error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, + failures: [], + task: { description: 'task description' } as any, + }) + ); - const migrator = new KibanaMigrator(options); - migrator.prepareMigrations(); - await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); - expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` + expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` [Error: Reindex failed with the following error: {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); - }); }); }); }); @@ -305,7 +227,7 @@ type MockedOptions = KibanaMigratorOptions & { }; const mockV2MigrationOptions = () => { - const options = mockOptions({ enableV2: true }); + const options = mockOptions(); options.client.indices.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise( @@ -357,7 +279,7 @@ const mockV2MigrationOptions = () => { return options; }; -const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) => { +const mockOptions = () => { const options: MockedOptions = { logger: loggingSystemMock.create().get(), kibanaVersion: '8.2.3', @@ -395,7 +317,6 @@ const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) pollInterval: 20000, scrollDuration: '10m', skip: false, - enableV2, retryAttempts: 20, }, client: elasticsearchClientMock.createElasticsearchClient(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index e09284b49c86e..2843ebf7aa0c7 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -22,13 +22,7 @@ import { SavedObjectsSerializer, SavedObjectsRawDoc, } from '../../serialization'; -import { - buildActiveMappings, - createMigrationEsClient, - IndexMigrator, - MigrationResult, - MigrationStatus, -} from '../core'; +import { buildActiveMappings, MigrationResult, MigrationStatus } from '../core'; import { DocumentMigrator, VersionedTransformer } from '../core/document_migrator'; import { createIndexMap } from '../core/build_index_map'; import { SavedObjectsMigrationConfigType } from '../../saved_objects_config'; @@ -44,7 +38,6 @@ export interface KibanaMigratorOptions { kibanaConfig: KibanaConfigType; kibanaVersion: string; logger: Logger; - migrationsRetryDelay?: number; } export type IKibanaMigrator = Pick; @@ -71,7 +64,6 @@ export class KibanaMigrator { status: 'waiting_to_start', }); private readonly activeMappings: IndexMapping; - private migrationsRetryDelay?: number; // TODO migrationsV2: make private once we remove migrations v1 public readonly kibanaVersion: string; // TODO migrationsV2: make private once we remove migrations v1 @@ -87,7 +79,6 @@ export class KibanaMigrator { soMigrationsConfig, kibanaVersion, logger, - migrationsRetryDelay, }: KibanaMigratorOptions) { this.client = client; this.kibanaConfig = kibanaConfig; @@ -105,7 +96,6 @@ export class KibanaMigrator { // Building the active mappings (and associated md5sums) is an expensive // operation so we cache the result this.activeMappings = buildActiveMappings(this.mappingProperties); - this.migrationsRetryDelay = migrationsRetryDelay; } /** @@ -174,43 +164,22 @@ export class KibanaMigrator { }); const migrators = Object.keys(indexMap).map((index) => { - // TODO migrationsV2: remove old migrations algorithm - if (this.soMigrationsConfig.enableV2) { - return { - migrate: (): Promise => { - return runResilientMigrator({ - client: this.client, - kibanaVersion: this.kibanaVersion, - targetMappings: buildActiveMappings(indexMap[index].typeMappings), - logger: this.log, - preMigrationScript: indexMap[index].script, - transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => - migrateRawDocs(this.serializer, this.documentMigrator.migrateAndConvert, rawDocs), - migrationVersionPerType: this.documentMigrator.migrationVersion, - indexPrefix: index, - migrationsConfig: this.soMigrationsConfig, - }); - }, - }; - } else { - return new IndexMigrator({ - batchSize: this.soMigrationsConfig.batchSize, - client: createMigrationEsClient(this.client, this.log, this.migrationsRetryDelay), - documentMigrator: this.documentMigrator, - index, - kibanaVersion: this.kibanaVersion, - log: this.log, - mappingProperties: indexMap[index].typeMappings, - setStatus: (status) => this.status$.next(status), - pollInterval: this.soMigrationsConfig.pollInterval, - scrollDuration: this.soMigrationsConfig.scrollDuration, - serializer: this.serializer, - // Only necessary for the migrator of the kibana index. - obsoleteIndexTemplatePattern: - index === kibanaIndexName ? 'kibana_index_template*' : undefined, - convertToAliasScript: indexMap[index].script, - }); - } + return { + migrate: (): Promise => { + return runResilientMigrator({ + client: this.client, + kibanaVersion: this.kibanaVersion, + targetMappings: buildActiveMappings(indexMap[index].typeMappings), + logger: this.log, + preMigrationScript: indexMap[index].script, + transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => + migrateRawDocs(this.serializer, this.documentMigrator.migrateAndConvert, rawDocs), + migrationVersionPerType: this.documentMigrator.migrationVersion, + indexPrefix: index, + migrationsConfig: this.soMigrationsConfig, + }); + }, + }; }); return Promise.all(migrators.map((migrator) => migrator.migrate())); diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts index 48bb282da18f6..abc2dd29aef21 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts @@ -27,7 +27,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration.test.ts index 37dfe9bc717d0..3432265022dc1 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/migration.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration.test.ts @@ -50,7 +50,6 @@ describe('migration v2', () => { { migrations: { skip: false, - enableV2: true, // There are 53 docs in fixtures. Batch size configured to enforce 3 migration steps. batchSize: 20, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7.7.2_xpack_100k.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7.7.2_xpack_100k.test.ts index 0e51c886f7f30..291d86758d7f6 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7.7.2_xpack_100k.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_7.7.2_xpack_100k.test.ts @@ -51,7 +51,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts index 9f7e32c49ef15..129b36e1b2e0e 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts @@ -62,7 +62,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index bffe590a39432..109f90308dbc7 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -39,7 +39,6 @@ describe('migrationsStateActionMachine', () => { pollInterval: 0, scrollDuration: '0s', skip: false, - enableV2: true, retryAttempts: 5, }, }); diff --git a/src/core/server/saved_objects/saved_objects_config.ts b/src/core/server/saved_objects/saved_objects_config.ts index 7182df74c597f..a46c323d75a82 100644 --- a/src/core/server/saved_objects/saved_objects_config.ts +++ b/src/core/server/saved_objects/saved_objects_config.ts @@ -15,24 +15,17 @@ const migrationSchema = schema.object({ scrollDuration: schema.string({ defaultValue: '15m' }), pollInterval: schema.number({ defaultValue: 1_500 }), skip: schema.boolean({ defaultValue: false }), - enableV2: schema.boolean({ defaultValue: true }), retryAttempts: schema.number({ defaultValue: 15 }), }); export type SavedObjectsMigrationConfigType = TypeOf; -const migrationDeprecations: ConfigDeprecationProvider = () => [ - (settings, fromPath, addDeprecation) => { - const migrationsConfig = settings[fromPath]; - if (migrationsConfig?.enableV2 !== undefined) { - addDeprecation({ - message: - '"migrations.enableV2" is deprecated and will be removed in an upcoming release without any further notice.', - documentationUrl: 'https://ela.st/kbn-so-migration-v2', - }); - } - return settings; - }, +const migrationDeprecations: ConfigDeprecationProvider = ({ unused }) => [ + unused('enableV2', { + message: + '"migrations.enableV2" is deprecated and will be removed in an upcoming release without any further notice.', + documentationUrl: 'https://ela.st/kbn-so-migration-v2', + }), ]; export const savedObjectsMigrationConfig: ServiceConfigDescriptor = { diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index b95f187cd44ca..7b7ccebbf41b9 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -341,10 +341,10 @@ export class SavedObjectsService }; } - public async start( - { elasticsearch, pluginsInitialized = true }: SavedObjectsStartDeps, - migrationsRetryDelay?: number - ): Promise { + public async start({ + elasticsearch, + pluginsInitialized = true, + }: SavedObjectsStartDeps): Promise { if (!this.setupDeps || !this.config) { throw new Error('#setup() needs to be run first'); } @@ -360,8 +360,7 @@ export class SavedObjectsService const migrator = this.createMigrator( kibanaConfig, this.config.migration, - elasticsearch.client.asInternalUser, - migrationsRetryDelay + elasticsearch.client.asInternalUser ); this.migrator$.next(migrator); @@ -477,8 +476,7 @@ export class SavedObjectsService private createMigrator( kibanaConfig: KibanaConfigType, soMigrationsConfig: SavedObjectsMigrationConfigType, - client: ElasticsearchClient, - migrationsRetryDelay?: number + client: ElasticsearchClient ): IKibanaMigrator { return new KibanaMigrator({ typeRegistry: this.typeRegistry, @@ -487,7 +485,6 @@ export class SavedObjectsService soMigrationsConfig, kibanaConfig, client, - migrationsRetryDelay, }); } } diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 8faa476b77bfa..2c59e70827ab3 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -2047,16 +2047,7 @@ export class SavedObjectsRepository { * @param type - the type */ private getIndexForType(type: string) { - // TODO migrationsV2: Remove once we remove migrations v1 - // This is a hacky, but it required the least amount of changes to - // existing code to support a migrations v2 index. Long term we would - // want to always use the type registry to resolve a type's index - // (including the default index). - if (this._migrator.soMigrationsConfig.enableV2) { - return `${this._registry.getIndex(type) || this._index}_${this._migrator.kibanaVersion}`; - } else { - return this._registry.getIndex(type) || this._index; - } + return `${this._registry.getIndex(type) || this._index}_${this._migrator.kibanaVersion}`; } /** diff --git a/test/api_integration/apis/saved_objects/index.ts b/test/api_integration/apis/saved_objects/index.ts index 2af1df01c0f92..a77fb7e719c70 100644 --- a/test/api_integration/apis/saved_objects/index.ts +++ b/test/api_integration/apis/saved_objects/index.ts @@ -19,7 +19,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./find')); loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./import')); - loadTestFile(require.resolve('./migrations')); loadTestFile(require.resolve('./resolve')); loadTestFile(require.resolve('./resolve_import_errors')); loadTestFile(require.resolve('./update')); diff --git a/test/api_integration/apis/saved_objects/migrations.ts b/test/api_integration/apis/saved_objects/migrations.ts deleted file mode 100644 index dcd34c604dc31..0000000000000 --- a/test/api_integration/apis/saved_objects/migrations.ts +++ /dev/null @@ -1,760 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -/* - * Smokescreen tests for core migration logic - */ - -import uuidv5 from 'uuid/v5'; -import { set } from '@elastic/safer-lodash-set'; -import _ from 'lodash'; -import expect from '@kbn/expect'; -import { ElasticsearchClient, SavedObjectsType } from 'src/core/server'; - -import { - DocumentMigrator, - IndexMigrator, - createMigrationEsClient, -} from '../../../../src/core/server/saved_objects/migrations/core'; -import { SavedObjectsTypeMappingDefinitions } from '../../../../src/core/server/saved_objects/mappings'; - -import { - SavedObjectsSerializer, - SavedObjectTypeRegistry, -} from '../../../../src/core/server/saved_objects'; -import { FtrProviderContext } from '../../ftr_provider_context'; - -const KIBANA_VERSION = '99.9.9'; -const FOO_TYPE: SavedObjectsType = { - name: 'foo', - hidden: false, - namespaceType: 'single', - mappings: { properties: {} }, -}; -const BAR_TYPE: SavedObjectsType = { - name: 'bar', - hidden: false, - namespaceType: 'single', - mappings: { properties: {} }, -}; -const BAZ_TYPE: SavedObjectsType = { - name: 'baz', - hidden: false, - namespaceType: 'single', - mappings: { properties: {} }, -}; -const FLEET_AGENT_EVENT_TYPE: SavedObjectsType = { - name: 'fleet-agent-event', - hidden: false, - namespaceType: 'single', - mappings: { properties: {} }, -}; - -function getLogMock() { - return { - debug() {}, - error() {}, - fatal() {}, - info() {}, - log() {}, - trace() {}, - warn() {}, - get: getLogMock, - }; -} -export default ({ getService }: FtrProviderContext) => { - const esClient = getService('es'); - const esDeleteAllIndices = getService('esDeleteAllIndices'); - - describe('Kibana index migration', () => { - before(() => esDeleteAllIndices('.migrate-*')); - - it('Migrates an existing index that has never been migrated before', async () => { - const index = '.migration-a'; - const originalDocs = [ - { id: 'foo:a', type: 'foo', foo: { name: 'Foo A' } }, - { id: 'foo:e', type: 'foo', foo: { name: 'Fooey' } }, - { id: 'bar:i', type: 'bar', bar: { nomnom: 33 } }, - { id: 'bar:o', type: 'bar', bar: { nomnom: 2 } }, - { id: 'baz:u', type: 'baz', baz: { title: 'Terrific!' } }, - ]; - - const mappingProperties = { - foo: { properties: { name: { type: 'text' } } }, - bar: { properties: { mynum: { type: 'integer' } } }, - }; - - const savedObjectTypes: SavedObjectsType[] = [ - { - ...FOO_TYPE, - migrations: { - '1.0.0': (doc) => set(doc, 'attributes.name', doc.attributes.name.toUpperCase()), - }, - }, - { - ...BAR_TYPE, - migrations: { - '1.0.0': (doc) => set(doc, 'attributes.nomnom', doc.attributes.nomnom + 1), - '1.3.0': (doc) => set(doc, 'attributes', { mynum: doc.attributes.nomnom }), - '1.9.0': (doc) => set(doc, 'attributes.mynum', doc.attributes.mynum * 2), - }, - }, - ]; - - await createIndex({ esClient, index, esDeleteAllIndices }); - await createDocs({ esClient, index, docs: originalDocs }); - - // Test that unrelated index templates are unaffected - await esClient.indices.putTemplate({ - name: 'migration_test_a_template', - body: { - index_patterns: ['migration_test_a'], - mappings: { - dynamic: 'strict', - properties: { baz: { type: 'text' } }, - }, - }, - }); - - // Test that obsolete index templates get removed - await esClient.indices.putTemplate({ - name: 'migration_a_template', - body: { - index_patterns: [index], - mappings: { - dynamic: 'strict', - properties: { baz: { type: 'text' } }, - }, - }, - }); - - const migrationATemplate = await esClient.indices.existsTemplate({ - name: 'migration_a_template', - }); - expect(migrationATemplate.body).to.be.ok(); - - const result = await migrateIndex({ - esClient, - index, - savedObjectTypes, - mappingProperties, - obsoleteIndexTemplatePattern: 'migration_a*', - }); - - const migrationATemplateAfter = await esClient.indices.existsTemplate({ - name: 'migration_a_template', - }); - - expect(migrationATemplateAfter.body).not.to.be.ok(); - const migrationTestATemplateAfter = await esClient.indices.existsTemplate({ - name: 'migration_test_a_template', - }); - - expect(migrationTestATemplateAfter.body).to.be.ok(); - expect(_.omit(result, 'elapsedMs')).to.eql({ - destIndex: '.migration-a_2', - sourceIndex: '.migration-a_1', - status: 'migrated', - }); - - // The docs in the original index are unchanged - expect(await fetchDocs(esClient, `${index}_1`)).to.eql(originalDocs.sort(sortByTypeAndId)); - - // The docs in the alias have been migrated - expect(await fetchDocs(esClient, index)).to.eql([ - { - id: 'bar:i', - type: 'bar', - migrationVersion: { bar: '1.9.0' }, - bar: { mynum: 68 }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'bar:o', - type: 'bar', - migrationVersion: { bar: '1.9.0' }, - bar: { mynum: 6 }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'baz:u', - type: 'baz', - baz: { title: 'Terrific!' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'foo:a', - type: 'foo', - migrationVersion: { foo: '1.0.0' }, - foo: { name: 'FOO A' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'foo:e', - type: 'foo', - migrationVersion: { foo: '1.0.0' }, - foo: { name: 'FOOEY' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - ]); - }); - - it('migrates a previously migrated index, if migrations change', async () => { - const index = '.migration-b'; - const originalDocs = [ - { id: 'foo:a', type: 'foo', foo: { name: 'Foo A' } }, - { id: 'foo:e', type: 'foo', foo: { name: 'Fooey' } }, - { id: 'bar:i', type: 'bar', bar: { nomnom: 33 } }, - { id: 'bar:o', type: 'bar', bar: { nomnom: 2 } }, - ]; - - const mappingProperties = { - foo: { properties: { name: { type: 'text' } } }, - bar: { properties: { mynum: { type: 'integer' } } }, - }; - - let savedObjectTypes: SavedObjectsType[] = [ - { - ...FOO_TYPE, - migrations: { - '1.0.0': (doc) => set(doc, 'attributes.name', doc.attributes.name.toUpperCase()), - }, - }, - { - ...BAR_TYPE, - migrations: { - '1.0.0': (doc) => set(doc, 'attributes.nomnom', doc.attributes.nomnom + 1), - '1.3.0': (doc) => set(doc, 'attributes', { mynum: doc.attributes.nomnom }), - '1.9.0': (doc) => set(doc, 'attributes.mynum', doc.attributes.mynum * 2), - }, - }, - ]; - - await createIndex({ esClient, index, esDeleteAllIndices }); - await createDocs({ esClient, index, docs: originalDocs }); - - await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }); - - // @ts-expect-error name doesn't exist on mynum type - mappingProperties.bar.properties.name = { type: 'keyword' }; - savedObjectTypes = [ - { - ...FOO_TYPE, - migrations: { - '2.0.1': (doc) => set(doc, 'attributes.name', `${doc.attributes.name}v2`), - }, - }, - { - ...BAR_TYPE, - migrations: { - '2.3.4': (doc) => set(doc, 'attributes.name', `NAME ${doc.id}`), - }, - }, - ]; - - await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }); - - // The index for the initial migration has not been destroyed... - expect(await fetchDocs(esClient, `${index}_2`)).to.eql([ - { - id: 'bar:i', - type: 'bar', - migrationVersion: { bar: '1.9.0' }, - bar: { mynum: 68 }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'bar:o', - type: 'bar', - migrationVersion: { bar: '1.9.0' }, - bar: { mynum: 6 }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'foo:a', - type: 'foo', - migrationVersion: { foo: '1.0.0' }, - foo: { name: 'FOO A' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'foo:e', - type: 'foo', - migrationVersion: { foo: '1.0.0' }, - foo: { name: 'FOOEY' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - ]); - - // The docs were migrated again... - expect(await fetchDocs(esClient, index)).to.eql([ - { - id: 'bar:i', - type: 'bar', - migrationVersion: { bar: '2.3.4' }, - bar: { mynum: 68, name: 'NAME i' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'bar:o', - type: 'bar', - migrationVersion: { bar: '2.3.4' }, - bar: { mynum: 6, name: 'NAME o' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'foo:a', - type: 'foo', - migrationVersion: { foo: '2.0.1' }, - foo: { name: 'FOO Av2' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'foo:e', - type: 'foo', - migrationVersion: { foo: '2.0.1' }, - foo: { name: 'FOOEYv2' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - ]); - }); - - it('drops fleet-agent-event saved object types when doing a migration', async () => { - const index = '.migration-b'; - const originalDocs = [ - { - id: 'fleet-agent-event:a', - type: 'fleet-agent-event', - 'fleet-agent-event': { name: 'Foo A' }, - }, - { - id: 'fleet-agent-event:e', - type: 'fleet-agent-event', - 'fleet-agent-event': { name: 'Fooey' }, - }, - { id: 'bar:i', type: 'bar', bar: { nomnom: 33 } }, - { id: 'bar:o', type: 'bar', bar: { nomnom: 2 } }, - ]; - - const mappingProperties = { - 'fleet-agent-event': { properties: { name: { type: 'text' } } }, - bar: { properties: { mynum: { type: 'integer' } } }, - }; - - let savedObjectTypes: SavedObjectsType[] = [ - FLEET_AGENT_EVENT_TYPE, - { - ...BAR_TYPE, - migrations: { - '1.0.0': (doc) => set(doc, 'attributes.nomnom', doc.attributes.nomnom + 1), - '1.3.0': (doc) => set(doc, 'attributes', { mynum: doc.attributes.nomnom }), - '1.9.0': (doc) => set(doc, 'attributes.mynum', doc.attributes.mynum * 2), - }, - }, - ]; - - await createIndex({ esClient, index, esDeleteAllIndices }); - await createDocs({ esClient, index, docs: originalDocs }); - - await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }); - - // @ts-expect-error name doesn't exist on mynum type - mappingProperties.bar.properties.name = { type: 'keyword' }; - savedObjectTypes = [ - FLEET_AGENT_EVENT_TYPE, - { - ...BAR_TYPE, - migrations: { - '2.3.4': (doc) => set(doc, 'attributes.name', `NAME ${doc.id}`), - }, - }, - ]; - - await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }); - - // Assert that fleet-agent-events were dropped - expect(await fetchDocs(esClient, index)).to.eql([ - { - id: 'bar:i', - type: 'bar', - migrationVersion: { bar: '2.3.4' }, - bar: { mynum: 68, name: 'NAME i' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'bar:o', - type: 'bar', - migrationVersion: { bar: '2.3.4' }, - bar: { mynum: 6, name: 'NAME o' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - ]); - }); - - it('Coordinates migrations across the Kibana cluster', async () => { - const index = '.migration-c'; - const originalDocs = [{ id: 'foo:lotr', type: 'foo', foo: { name: 'Lord of the Rings' } }]; - - const mappingProperties = { - foo: { properties: { name: { type: 'text' } } }, - }; - - const savedObjectTypes: SavedObjectsType[] = [ - { - ...FOO_TYPE, - migrations: { - '1.0.0': (doc) => set(doc, 'attributes.name', 'LOTR'), - }, - }, - ]; - - await createIndex({ esClient, index, esDeleteAllIndices }); - await createDocs({ esClient, index, docs: originalDocs }); - - const result = await Promise.all([ - migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }), - migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }), - ]); - - // The polling instance and the migrating instance should both - // return a similar migration result. - expect( - result - // @ts-expect-error destIndex exists only on MigrationResult status: 'migrated'; - .map(({ status, destIndex }) => ({ status, destIndex })) - .sort(({ destIndex: a }, { destIndex: b }) => - // sort by destIndex in ascending order, keeping falsy values at the end - (a && !b) || a < b ? -1 : (!a && b) || a > b ? 1 : 0 - ) - ).to.eql([ - { status: 'migrated', destIndex: '.migration-c_2' }, - { status: 'skipped', destIndex: undefined }, - ]); - - const { body } = await esClient.cat.indices({ index: '.migration-c*', format: 'json' }); - // It only created the original and the dest - expect(_.map(body, 'index').sort()).to.eql(['.migration-c_1', '.migration-c_2']); - - // The docs in the original index are unchanged - expect(await fetchDocs(esClient, `${index}_1`)).to.eql([ - { id: 'foo:lotr', type: 'foo', foo: { name: 'Lord of the Rings' } }, - ]); - - // The docs in the alias have been migrated - expect(await fetchDocs(esClient, index)).to.eql([ - { - id: 'foo:lotr', - type: 'foo', - migrationVersion: { foo: '1.0.0' }, - foo: { name: 'LOTR' }, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - ]); - }); - - it('Correctly applies reference transforms and conversion transforms', async () => { - const index = '.migration-d'; - const originalDocs = [ - { id: 'foo:1', type: 'foo', foo: { name: 'Foo 1 default' } }, - { id: 'spacex:foo:1', type: 'foo', foo: { name: 'Foo 1 spacex' }, namespace: 'spacex' }, - { - id: 'bar:1', - type: 'bar', - bar: { nomnom: 1 }, - references: [{ type: 'foo', id: '1', name: 'Foo 1 default' }], - }, - { - id: 'spacex:bar:1', - type: 'bar', - bar: { nomnom: 2 }, - references: [{ type: 'foo', id: '1', name: 'Foo 1 spacex' }], - namespace: 'spacex', - }, - { - id: 'baz:1', - type: 'baz', - baz: { title: 'Baz 1 default' }, - references: [{ type: 'bar', id: '1', name: 'Bar 1 default' }], - }, - { - id: 'spacex:baz:1', - type: 'baz', - baz: { title: 'Baz 1 spacex' }, - references: [{ type: 'bar', id: '1', name: 'Bar 1 spacex' }], - namespace: 'spacex', - }, - ]; - - const mappingProperties = { - foo: { properties: { name: { type: 'text' } } }, - bar: { properties: { nomnom: { type: 'integer' } } }, - baz: { properties: { title: { type: 'keyword' } } }, - }; - - const savedObjectTypes: SavedObjectsType[] = [ - { - ...FOO_TYPE, - namespaceType: 'multiple', - convertToMultiNamespaceTypeVersion: '1.0.0', - }, - { - ...BAR_TYPE, - namespaceType: 'multiple-isolated', - convertToMultiNamespaceTypeVersion: '2.0.0', - }, - BAZ_TYPE, // must be registered for reference transforms to be applied to objects of this type - ]; - - await createIndex({ esClient, index, esDeleteAllIndices }); - await createDocs({ esClient, index, docs: originalDocs }); - - await migrateIndex({ - esClient, - index, - savedObjectTypes, - mappingProperties, - obsoleteIndexTemplatePattern: 'migration_a*', - }); - - // The docs in the original index are unchanged - expect(await fetchDocs(esClient, `${index}_1`)).to.eql(originalDocs.sort(sortByTypeAndId)); - - // The docs in the alias have been migrated - const migratedDocs = await fetchDocs(esClient, index); - - // each newly converted multi-namespace object in a non-default space has its ID deterministically regenerated, and a legacy-url-alias - // object is created which links the old ID to the new ID - const newFooId = uuidv5('spacex:foo:1', uuidv5.DNS); - const newBarId = uuidv5('spacex:bar:1', uuidv5.DNS); - - expect(migratedDocs).to.eql( - [ - { - id: 'foo:1', - type: 'foo', - foo: { name: 'Foo 1 default' }, - references: [], - namespaces: ['default'], - migrationVersion: { foo: '1.0.0' }, - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: `foo:${newFooId}`, - type: 'foo', - foo: { name: 'Foo 1 spacex' }, - references: [], - namespaces: ['spacex'], - originId: '1', - migrationVersion: { foo: '1.0.0' }, - coreMigrationVersion: KIBANA_VERSION, - }, - { - // new object - id: 'legacy-url-alias:spacex:foo:1', - type: 'legacy-url-alias', - 'legacy-url-alias': { - targetId: newFooId, - targetNamespace: 'spacex', - targetType: 'foo', - }, - migrationVersion: {}, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'bar:1', - type: 'bar', - bar: { nomnom: 1 }, - references: [{ type: 'foo', id: '1', name: 'Foo 1 default' }], - namespaces: ['default'], - migrationVersion: { bar: '2.0.0' }, - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: `bar:${newBarId}`, - type: 'bar', - bar: { nomnom: 2 }, - references: [{ type: 'foo', id: newFooId, name: 'Foo 1 spacex' }], - namespaces: ['spacex'], - originId: '1', - migrationVersion: { bar: '2.0.0' }, - coreMigrationVersion: KIBANA_VERSION, - }, - { - // new object - id: 'legacy-url-alias:spacex:bar:1', - type: 'legacy-url-alias', - 'legacy-url-alias': { - targetId: newBarId, - targetNamespace: 'spacex', - targetType: 'bar', - }, - migrationVersion: {}, - references: [], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'baz:1', - type: 'baz', - baz: { title: 'Baz 1 default' }, - references: [{ type: 'bar', id: '1', name: 'Bar 1 default' }], - coreMigrationVersion: KIBANA_VERSION, - }, - { - id: 'spacex:baz:1', - type: 'baz', - baz: { title: 'Baz 1 spacex' }, - references: [{ type: 'bar', id: newBarId, name: 'Bar 1 spacex' }], - namespace: 'spacex', - coreMigrationVersion: KIBANA_VERSION, - }, - ].sort(sortByTypeAndId) - ); - }); - }); -}; - -async function createIndex({ - esClient, - index, - esDeleteAllIndices, -}: { - esClient: ElasticsearchClient; - index: string; - esDeleteAllIndices: (pattern: string) => Promise; -}) { - await esDeleteAllIndices(`${index}*`); - - const properties = { - type: { type: 'keyword' }, - foo: { properties: { name: { type: 'keyword' } } }, - bar: { properties: { nomnom: { type: 'integer' } } }, - baz: { properties: { title: { type: 'keyword' } } }, - 'legacy-url-alias': { - properties: { - targetNamespace: { type: 'text' }, - targetType: { type: 'text' }, - targetId: { type: 'text' }, - lastResolved: { type: 'date' }, - resolveCounter: { type: 'integer' }, - disabled: { type: 'boolean' }, - }, - }, - namespace: { type: 'keyword' }, - namespaces: { type: 'keyword' }, - originId: { type: 'keyword' }, - references: { - type: 'nested', - properties: { - name: { type: 'keyword' }, - type: { type: 'keyword' }, - id: { type: 'keyword' }, - }, - }, - coreMigrationVersion: { - type: 'keyword', - }, - }; - await esClient.indices.create({ - index, - body: { mappings: { dynamic: 'strict', properties } }, - }); -} - -async function createDocs({ - esClient, - index, - docs, -}: { - esClient: ElasticsearchClient; - index: string; - docs: any[]; -}) { - await esClient.bulk({ - body: docs.reduce((acc, doc) => { - acc.push({ index: { _id: doc.id, _index: index } }); - acc.push(_.omit(doc, 'id')); - return acc; - }, []), - }); - await esClient.indices.refresh({ index }); -} - -async function migrateIndex({ - esClient, - index, - savedObjectTypes, - mappingProperties, - obsoleteIndexTemplatePattern, -}: { - esClient: ElasticsearchClient; - index: string; - savedObjectTypes: SavedObjectsType[]; - mappingProperties: SavedObjectsTypeMappingDefinitions; - obsoleteIndexTemplatePattern?: string; -}) { - const typeRegistry = new SavedObjectTypeRegistry(); - savedObjectTypes.forEach((type) => typeRegistry.registerType(type)); - - const documentMigrator = new DocumentMigrator({ - kibanaVersion: KIBANA_VERSION, - typeRegistry, - minimumConvertVersion: '0.0.0', // bypass the restriction of a minimum version of 8.0.0 for these integration tests - log: getLogMock(), - }); - - documentMigrator.prepareMigrations(); - - const migrator = new IndexMigrator({ - client: createMigrationEsClient(esClient, getLogMock()), - documentMigrator, - index, - kibanaVersion: KIBANA_VERSION, - obsoleteIndexTemplatePattern, - mappingProperties, - batchSize: 10, - log: getLogMock(), - setStatus: () => {}, - pollInterval: 50, - scrollDuration: '5m', - serializer: new SavedObjectsSerializer(typeRegistry), - }); - - return await migrator.migrate(); -} - -async function fetchDocs(esClient: ElasticsearchClient, index: string) { - const { body } = await esClient.search({ index }); - - return body.hits.hits - .map((h) => ({ - ...h._source, - id: h._id, - })) - .sort(sortByTypeAndId); -} - -function sortByTypeAndId(a: { type: string; id: string }, b: { type: string; id: string }) { - return a.type.localeCompare(b.type) || a.id.localeCompare(b.id); -}