From bcb8a13e66cc05a756725d92976348cb90d5c620 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Fri, 23 Dec 2022 20:19:27 +0100 Subject: [PATCH] [Migrations] Only reindex indices if needed (#147371) Fixes https://github.com/elastic/kibana/issues/124946 ## Summary Takes a step toward optimising our migration paths by only reindexing (an expensive operation) when needed by checking whether the current SO mappings have "changed". By "changed" we mean that we have detected a new md5 checksum of any registered saved object type relative to the hashes we have stored. ## How to test These changes are constrained to the `model.ts`, a test was added for correctly detecting that mappings are the same during the `INIT` phase to send us down the correct migration path. Additionally, we have a new Jest integration test `skip_reindex.test.ts` that runs Kibana and inspects the logs for the expected model transitions. Everything else should remain the same. ## Happy path for skipping reindex ``` RUN INIT IF !versionMigrationIsComplete AND !kibana indexBelongsToLaterVersion AND !waitForMigrationCompletion AND mappingsAreUnchanged THEN the migration operations must target the existing .kibana_x.y.z_001 index RUN PREPARE_COMPATIBLE_MIGRATION (new state) we remove old version aliases (prevent old reads/writes), and set the current version alias (prevent old migrations) SKIP LEGACY_SET_WRITE_BLOCK SKIP ... SKIP SET_SOURCE_WRITE_BLOCK SKIP ... SKIP REFRESH_TARGET RUN OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT ... RUN MARK_VERSION_INDEX_READY DONE ``` ## Notes * This optimisation will only be triggered when there are no mappings changes AND we are upgrading to a new version. This does not cover all cases. In future we will make this more sophisticated by checking for incompatible changes to mappings and only reindexing when those occur. ## Related * https://github.com/elastic/kibana/pull/147503 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Gerard Soldevila --- .../src/model/helpers.test.ts | 20 +++ .../src/model/helpers.ts | 18 ++- .../src/model/model.test.ts | 152 ++++++++++++++++-- .../src/model/model.ts | 84 ++++++++-- .../src/next.ts | 5 +- .../src/state.ts | 23 ++- .../migrations/check_target_mappings.test.ts | 53 +----- .../migrations/skip_reindex.test.ts | 118 ++++++++++++++ .../saved_objects/migrations/test_utils.ts | 3 + 9 files changed, 394 insertions(+), 82 deletions(-) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts index c364e053c1ff6..5cfd4bc04c465 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts @@ -12,6 +12,7 @@ import { addMustClausesToBoolQuery, addMustNotClausesToBoolQuery, getAliases, + buildRemoveAliasActions, versionMigrationCompleted, } from './helpers'; @@ -267,3 +268,22 @@ describe('versionMigrationCompleted', () => { expect(versionMigrationCompleted('.current-alias', '.version-alias', {})).toBe(false); }); }); + +describe('buildRemoveAliasActions', () => { + test('empty', () => { + expect(buildRemoveAliasActions('.kibana_test_123', [], [])).toEqual([]); + }); + test('no exclusions', () => { + expect(buildRemoveAliasActions('.kibana_test_123', ['a', 'b', 'c'], [])).toEqual([ + { remove: { index: '.kibana_test_123', alias: 'a', must_exist: true } }, + { remove: { index: '.kibana_test_123', alias: 'b', must_exist: true } }, + { remove: { index: '.kibana_test_123', alias: 'c', must_exist: true } }, + ]); + }); + test('with exclusions', () => { + expect(buildRemoveAliasActions('.kibana_test_123', ['a', 'b', 'c'], ['b'])).toEqual([ + { remove: { index: '.kibana_test_123', alias: 'a', must_exist: true } }, + { remove: { index: '.kibana_test_123', alias: 'c', must_exist: true } }, + ]); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts index f7377401c16bf..a7e71bc99e9e0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts @@ -14,7 +14,7 @@ import type { import * as Either from 'fp-ts/lib/Either'; import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import type { State } from '../state'; -import type { FetchIndexResponse } from '../actions'; +import type { AliasAction, FetchIndexResponse } from '../actions'; /** * A helper function/type for ensuring that all control state's are handled. @@ -189,3 +189,19 @@ export function getAliases( return Either.right(aliases); } + +/** + * Build a list of alias actions to remove the provided aliases from the given index. + */ +export function buildRemoveAliasActions( + index: string, + aliases: string[], + exclude: string[] +): AliasAction[] { + return aliases.flatMap((alias) => { + if (exclude.includes(alias)) { + return []; + } + return [{ remove: { index, alias, must_exist: true } }]; + }); +} diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts index c812cd14969ef..8072675b96b7e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts @@ -9,6 +9,7 @@ import * as Either from 'fp-ts/lib/Either'; import * as Option from 'fp-ts/lib/Option'; import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; +import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import type { FatalState, State, @@ -45,6 +46,7 @@ import type { CheckVersionIndexReadyActions, UpdateTargetMappingsMeta, CheckTargetMappingsState, + PrepareCompatibleMigration, } from '../state'; import { type TransformErrorObjects, TransformSavedObjectDocumentError } from '../core'; import type { AliasAction, RetryableEsClientError } from '../actions'; @@ -53,6 +55,20 @@ import { createInitialProgress } from './progress'; import { model } from './model'; describe('migrations v2 model', () => { + const indexMapping: IndexMapping = { + properties: { + new_saved_object_type: { + properties: { + value: { type: 'text' }, + }, + }, + }, + _meta: { + migrationMappingPropertyHashes: { + new_saved_object_type: '4a11183eee21e6fbad864f7a30b39ad0', + }, + }, + }; const baseState: BaseState = { controlState: '', legacyIndex: '.kibana', @@ -67,20 +83,7 @@ describe('migrations v2 model', () => { discardCorruptObjects: false, indexPrefix: '.kibana', outdatedDocumentsQuery: {}, - targetIndexMappings: { - properties: { - new_saved_object_type: { - properties: { - value: { type: 'text' }, - }, - }, - }, - _meta: { - migrationMappingPropertyHashes: { - new_saved_object_type: '4a11183eee21e6fbad864f7a30b39ad0', - }, - }, - }, + targetIndexMappings: indexMapping, tempIndexMappings: { properties: {} }, preMigrationScript: Option.none, currentAlias: '.kibana', @@ -265,7 +268,7 @@ describe('migrations v2 model', () => { settings: {}, }, }); - const newState = model(initState, res); + const newState = model(initState, res) as OutdatedDocumentsSearchOpenPit; expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'); // This snapshot asserts that we merge the @@ -292,6 +295,22 @@ describe('migrations v2 model', () => { }, } `); + expect(newState.targetIndexRawMappings).toEqual({ + _meta: { + migrationMappingPropertyHashes: { + disabled_saved_object_type: '7997cf5a56cc02bdc9c93361bde732b0', + }, + }, + properties: { + disabled_saved_object_type: { + properties: { + value: { + type: 'keyword', + }, + }, + }, + }, + }); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); @@ -498,6 +517,7 @@ describe('migrations v2 model', () => { expect(newState.retryDelay).toEqual(2000); }); }); + describe('if waitForMigrationCompletion=false', () => { const initState = Object.assign({}, initBaseState, { waitForMigrationCompletion: false, @@ -787,6 +807,67 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + + describe('when upgrading to a new stack version', () => { + const unchangedMappingsState: State = { + ...baseState, + controlState: 'INIT', + kibanaVersion: '7.12.0', // new version! + currentAlias: '.kibana', + versionAlias: '.kibana_7.12.0', + versionIndex: '.kibana_7.11.0_001', + }; + it('INIT -> PREPARE_COMPATIBLE_MIGRATION when the mappings have not changed', () => { + const res: ResponseType<'INIT'> = Either.right({ + '.kibana_7.11.0_001': { + aliases: { + '.kibana': {}, + '.kibana_7.11.0': {}, + }, + mappings: indexMapping, + settings: {}, + }, + }); + const newState = model(unchangedMappingsState, res) as PrepareCompatibleMigration; + + expect(newState.controlState).toEqual('PREPARE_COMPATIBLE_MIGRATION'); + expect(newState.targetIndexRawMappings).toEqual({ + _meta: { + migrationMappingPropertyHashes: { + new_saved_object_type: '4a11183eee21e6fbad864f7a30b39ad0', + }, + }, + properties: { + new_saved_object_type: { + properties: { + value: { + type: 'text', + }, + }, + }, + }, + }); + expect(newState.versionAlias).toEqual('.kibana_7.12.0'); + expect(newState.currentAlias).toEqual('.kibana'); + // will point to + expect(newState.targetIndex).toEqual('.kibana_7.11.0_001'); + expect(newState.preTransformDocsActions).toEqual([ + { + add: { + alias: '.kibana_7.12.0', + index: '.kibana_7.11.0_001', + }, + }, + { + remove: { + alias: '.kibana_7.11.0', + index: '.kibana_7.11.0_001', + must_exist: true, + }, + }, + ]); + }); + }); }); }); @@ -1893,6 +1974,47 @@ describe('migrations v2 model', () => { }); }); + describe('PREPARE_COMPATIBLE_MIGRATIONS', () => { + const someAliasAction: AliasAction = { add: { index: '.kibana', alias: '.kibana_8.7.0' } }; + const state: PrepareCompatibleMigration = { + ...baseState, + controlState: 'PREPARE_COMPATIBLE_MIGRATION', + versionIndexReadyActions: Option.none, + sourceIndex: Option.some('.kibana') as Option.Some, + targetIndex: '.kibana_7.11.0_001', + preTransformDocsActions: [someAliasAction], + }; + + it('PREPARE_COMPATIBLE_MIGRATIONS -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT if action succeeds', () => { + const res: ResponseType<'PREPARE_COMPATIBLE_MIGRATION'> = Either.right( + 'update_aliases_succeeded' + ); + const newState = model(state, res) as OutdatedDocumentsSearchOpenPit; + expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'); + expect(newState.versionIndexReadyActions).toEqual(Option.none); + }); + + it('PREPARE_COMPATIBLE_MIGRATIONS -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT if action fails because the alias is not found', () => { + const res: ResponseType<'PREPARE_COMPATIBLE_MIGRATION'> = Either.left({ + type: 'alias_not_found_exception', + }); + + const newState = model(state, res) as OutdatedDocumentsSearchOpenPit; + expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'); + expect(newState.versionIndexReadyActions).toEqual(Option.none); + }); + + it('throws an exception if action fails with an error other than a missing alias', () => { + const res: ResponseType<'PREPARE_COMPATIBLE_MIGRATION'> = Either.left({ + type: 'remove_index_not_a_concrete_index', + }); + + expect(() => model(state, res)).toThrowErrorMatchingInlineSnapshot( + `"PREPARE_COMPATIBLE_MIGRATION received unexpected action response: {\\"type\\":\\"remove_index_not_a_concrete_index\\"}"` + ); + }); + }); + describe('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', () => { const state: OutdatedDocumentsSearchOpenPit = { ...baseState, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 1b0fa490fda51..44de571c13f5d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -38,9 +38,11 @@ import { throwBadControlState, throwBadResponse, versionMigrationCompleted, + buildRemoveAliasActions, } from './helpers'; import { createBatches } from './create_batches'; import type { MigrationLog } from '../types'; +import { diffMappings } from '../core/build_active_mappings'; export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; @@ -71,7 +73,6 @@ export const model = (currentState: State, resW: ResponseType): if (stateP.controlState === 'INIT') { const res = resW as ExcludeRetryableEsError>; - if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'incompatible_cluster_routing_allocation')) { @@ -97,12 +98,13 @@ export const model = (currentState: State, resW: ResponseType): const aliases = aliasesRes.right; + // The source index .kibana is pointing to. E.g: ".kibana_8.7.0_001" + const source = aliases[stateP.currentAlias]; + if ( // This version's migration has already been completed. versionMigrationCompleted(stateP.currentAlias, stateP.versionAlias, aliases) ) { - const source = aliases[stateP.currentAlias]!; - return { ...stateP, // Skip to 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' so that if a new plugin was @@ -112,15 +114,15 @@ export const model = (currentState: State, resW: ResponseType): // Source is a none because we didn't do any migration from a source // index sourceIndex: Option.none, - targetIndex: `${stateP.indexPrefix}_${stateP.kibanaVersion}_001`, - sourceIndexMappings: indices[source].mappings, + targetIndex: source!, + sourceIndexMappings: indices[source!].mappings, // in this scenario, a .kibana_X.Y.Z_001 index exists that matches the current kibana version // aka we are NOT upgrading to a newer version // we inject the target index's current mappings in the state, to check them later - targetIndexCurrentMappings: indices[source].mappings, + targetIndexRawMappings: indices[source!].mappings, targetIndexMappings: mergeMigrationMappingPropertyHashes( stateP.targetIndexMappings, - indices[aliases[stateP.currentAlias]!].mappings + indices[source!].mappings ), versionIndexReadyActions: Option.none, }; @@ -156,17 +158,51 @@ export const model = (currentState: State, resW: ResponseType): }, ], }; + } else if ( + // source exists + Boolean(indices[source!]?.mappings?._meta?.migrationMappingPropertyHashes) && + // ...and mappings are unchanged + !diffMappings( + /* actual */ + indices[source!].mappings, + /* expected */ + stateP.targetIndexMappings + ) + ) { + const targetIndex = source!; + const sourceMappings = indices[source!].mappings; + + return { + ...stateP, + controlState: 'PREPARE_COMPATIBLE_MIGRATION', + sourceIndex: Option.none, + targetIndex, + targetIndexRawMappings: sourceMappings, + targetIndexMappings: mergeMigrationMappingPropertyHashes( + stateP.targetIndexMappings, + sourceMappings + ), + preTransformDocsActions: [ + // Point the version alias to the source index. This let's other Kibana + // instances know that a migration for the current version is "done" + // even though we may be waiting for document transformations to finish. + { add: { index: source!, alias: stateP.versionAlias } }, + ...buildRemoveAliasActions(source!, Object.keys(aliases), [ + stateP.currentAlias, + stateP.versionAlias, + ]), + ], + versionIndexReadyActions: Option.none, + }; } else if ( // If the `.kibana` alias exists - aliases[stateP.currentAlias] != null + source != null ) { - // The source index is the index the `.kibana` alias points to - const source = aliases[stateP.currentAlias]!; return { ...stateP, controlState: 'WAIT_FOR_YELLOW_SOURCE', - sourceIndex: Option.some(source) as Option.Some, - sourceIndexMappings: indices[source].mappings, + sourceIndex: Option.some(source!) as Option.Some, + sourceIndexMappings: indices[source!].mappings, }; } else if (indices[stateP.legacyIndex] != null) { // Migrate from a legacy index @@ -276,6 +312,30 @@ export const model = (currentState: State, resW: ResponseType): ], }; } + } else if (stateP.controlState === 'PREPARE_COMPATIBLE_MIGRATION') { + const res = resW as ExcludeRetryableEsError>; + if (Either.isRight(res)) { + return { + ...stateP, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; + } else if (Either.isLeft(res)) { + // Note: if multiple newer Kibana versions are competing with each other to perform a migration, + // it might happen that another Kibana instance has deleted this instance's version index. + // NIT to handle this in properly, we'd have to add a PREPARE_COMPATIBLE_MIGRATION_CONFLICT step, + // similar to MARK_VERSION_INDEX_READY_CONFLICT. + if (isTypeof(res.left, 'alias_not_found_exception')) { + // We assume that the alias was already deleted by another Kibana instance + return { + ...stateP, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; + } else { + throwBadResponse(stateP, res.left as never); + } + } else { + throwBadResponse(stateP, res); + } } else if (stateP.controlState === 'LEGACY_SET_WRITE_BLOCK') { const res = resW as ExcludeRetryableEsError>; // If the write block is successfully in place diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts index 3382f2aa9996a..386786baf60c8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts @@ -42,6 +42,7 @@ import type { CalculateExcludeFiltersState, WaitForMigrationCompletionState, CheckTargetMappingsState, + PrepareCompatibleMigration, } from './state'; import type { TransformRawDocs } from './types'; import * as Actions from './actions'; @@ -62,6 +63,8 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra return { INIT: (state: InitState) => Actions.initAction({ client, indices: [state.currentAlias, state.versionAlias] }), + PREPARE_COMPATIBLE_MIGRATION: (state: PrepareCompatibleMigration) => + Actions.updateAliases({ client, aliasActions: state.preTransformDocsActions }), WAIT_FOR_MIGRATION_COMPLETION: (state: WaitForMigrationCompletionState) => Actions.fetchIndices({ client, indices: [state.currentAlias, state.versionAlias] }), WAIT_FOR_YELLOW_SOURCE: (state: WaitForYellowSourceState) => @@ -132,7 +135,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra Actions.refreshIndex({ client, targetIndex: state.targetIndex }), CHECK_TARGET_MAPPINGS: (state: CheckTargetMappingsState) => Actions.checkTargetMappings({ - actualMappings: state.targetIndexCurrentMappings, + actualMappings: state.targetIndexRawMappings, expectedMappings: state.targetIndexMappings, }), UPDATE_TARGET_MAPPINGS: (state: UpdateTargetMappingsState) => diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts index 324877f6cf13e..e3873122329fd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts @@ -173,7 +173,12 @@ export interface PostInitState extends BaseState { readonly sourceIndex: Option.Option; /** The target index is the index to which the migration writes */ readonly targetIndex: string; - readonly targetIndexCurrentMappings?: IndexMapping; + /** + * Unaltered mappings retrieved from the current target index. + * + * See also {@link BaseState['targetIndexMappings']}. + */ + readonly targetIndexRawMappings?: IndexMapping; readonly versionIndexReadyActions: Option.Option; readonly outdatedDocumentsQuery: QueryDslQueryContainer; } @@ -183,6 +188,21 @@ export interface DoneState extends PostInitState { readonly controlState: 'DONE'; } +/** + * Compatibe migrations do not require migrating to a new index because all + * schema changes are compatible with current index mappings. + * + * Before running the compatible migration we need to prepare. For example, we + * need to make sure that no older Kibana versions are still writing to target + * index. + */ +export interface PrepareCompatibleMigration extends PostInitState { + /** We have found a schema-compatible migration, this means we can optimise our migration steps */ + readonly controlState: 'PREPARE_COMPATIBLE_MIGRATION'; + /** Alias-level actions that prepare for this migration */ + readonly preTransformDocsActions: AliasAction[]; +} + export interface FatalState extends BaseState { /** Migration terminated with a failure */ readonly controlState: 'FATAL'; @@ -452,6 +472,7 @@ export interface LegacyDeleteState extends LegacyBaseState { export type State = Readonly< | FatalState | InitState + | PrepareCompatibleMigration | WaitForMigrationCompletionState | DoneState | WaitForYellowSourceState diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_target_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_target_mappings.test.ts index 5dd147de85b51..38b8c00a701a1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/check_target_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/check_target_mappings.test.ts @@ -8,7 +8,6 @@ import Path from 'path'; import fs from 'fs/promises'; -import { SemVer } from 'semver'; import JSON5 from 'json5'; import { Env } from '@kbn/config'; import { REPO_ROOT } from '@kbn/repo-info'; @@ -20,11 +19,10 @@ import { createTestServers, type TestElasticsearchUtils, } from '@kbn/core-test-helpers-kbn-server'; +import { delay } from './test_utils'; const logFilePath = Path.join(__dirname, 'check_target_mappings.log'); -const delay = (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); - async function removeLogFile() { // ignore errors if it doesn't exist await fs.unlink(logFilePath).catch(() => void 0); @@ -160,55 +158,6 @@ describe('migration v2 - CHECK_TARGET_MAPPINGS', () => { expect(logIncludes(logs, 'MARK_VERSION_INDEX_READY -> DONE')).toEqual(true); expect(logIncludes(logs, 'Migration completed')).toEqual(true); }); - - it('runs UPDATE_TARGET_MAPPINGS even if the mappings have NOT changed', async () => { - const { startES } = createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: { - es: { - license: 'basic', - }, - }, - }); - - esServer = await startES(); - - // start Kibana a first time to create the system indices - root = createRoot(); - await root.preboot(); - await root.setup(); - await root.start(); - - // stop Kibana and remove logs - await root.shutdown(); - await delay(10); - await removeLogFile(); - - const nextMinor = new SemVer(currentVersion).inc('patch').format(); - root = createRoot(undefined, nextMinor); - await root.preboot(); - await root.setup(); - await root.start(); - - // Check for migration steps present in the logs - logs = await parseLogFile(); - expect(logIncludes(logs, 'CREATE_NEW_TARGET')).toEqual(false); - expect(logIncludes(logs, 'CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS')).toEqual(true); - expect( - logIncludes(logs, 'UPDATE_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK') - ).toEqual(true); - expect( - logIncludes(logs, 'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META') - ).toEqual(true); - expect( - logIncludes(logs, 'UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS') - ).toEqual(true); - expect( - logIncludes(logs, 'CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY') - ).toEqual(true); - expect(logIncludes(logs, 'MARK_VERSION_INDEX_READY -> DONE')).toEqual(true); - expect(logIncludes(logs, 'Migration completed')).toEqual(true); - }); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts b/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts new file mode 100644 index 0000000000000..df64ccb853753 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/skip_reindex.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import Path from 'path'; +import fs from 'fs/promises'; +import { Env } from '@kbn/config'; +import { getEnvOptions } from '@kbn/config-mocks'; +import { REPO_ROOT } from '@kbn/repo-info'; +import type { Root } from '@kbn/core-root-server-internal'; +import { + createRootWithCorePlugins, + createTestServers, + type TestElasticsearchUtils, +} from '@kbn/core-test-helpers-kbn-server'; +import { delay } from './test_utils'; +import { SemVer } from 'semver'; + +const logFilePath = Path.join(__dirname, 'skip_reindex.log'); + +describe('skip reindexing', () => { + const currentVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; + let esServer: TestElasticsearchUtils['es']; + let root: Root; + + afterEach(async () => { + await root?.shutdown(); + await esServer?.stop(); + await delay(10); + }); + + it('when migrating to a new version, but mappings remain the same', async () => { + let logs: string; + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + esServer = await startES(); + root = createRoot(); + + // Run initial migrations + await root.preboot(); + await root.setup(); + await root.start(); + + // stop Kibana and remove logs + await root.shutdown(); + await delay(10); + await fs.unlink(logFilePath).catch(() => {}); + + const nextPatch = new SemVer(currentVersion).inc('patch').format(); + root = createRoot(nextPatch); + await root.preboot(); + await root.setup(); + await root.start(); + + logs = await fs.readFile(logFilePath, 'utf-8'); + + expect(logs).toMatch('INIT -> PREPARE_COMPATIBLE_MIGRATION'); + expect(logs).toMatch('PREPARE_COMPATIBLE_MIGRATION -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'); + expect(logs).toMatch('CHECK_TARGET_MAPPINGS -> CHECK_VERSION_INDEX_READY_ACTIONS'); + expect(logs).toMatch('CHECK_VERSION_INDEX_READY_ACTIONS -> DONE'); + + expect(logs).not.toMatch('CREATE_NEW_TARGET'); + expect(logs).not.toMatch('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS'); + + // We restart Kibana again after doing a "compatible migration" to ensure that + // the next time state is loaded everything still works as expected. + // For instance, we might see something like: + // Unable to complete saved object migrations for the [.kibana] index. Please check the health of your Elasticsearch cluster and try again. Unexpected Elasticsearch ResponseError: statusCode: 404, method: POST, url: /.kibana_8.7.1_001/_pit?keep_alive=10m error: [index_not_found_exception]: no such index [.kibana_8.7.1_001] + await root.shutdown(); + await delay(10); + await fs.unlink(logFilePath).catch(() => {}); + + root = createRoot(nextPatch); + await root.preboot(); + await root.setup(); + await root.start(); + + logs = await fs.readFile(logFilePath, 'utf-8'); + expect(logs).toMatch('INIT -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'); + expect(logs).not.toMatch('INIT -> PREPARE_COMPATIBLE_MIGRATION'); + }); +}); + +function createRoot(kibanaVersion?: string): Root { + return createRootWithCorePlugins( + { + logging: { + appenders: { + file: { + type: 'file', + fileName: logFilePath, + layout: { + type: 'json', + }, + }, + }, + loggers: [ + { + name: 'root', + level: 'info', + appenders: ['file'], + }, + ], + }, + }, + { oss: true }, + kibanaVersion + ); +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts index 5e519890e5adc..aa78d8dbc5d90 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts @@ -21,3 +21,6 @@ export const getMigrationDocLink = () => { const docLinks = getDocLinks({ kibanaBranch: env.packageInfo.branch }); return docLinks.kibanaUpgradeSavedObjects; }; + +export const delay = (seconds: number) => + new Promise((resolve) => setTimeout(resolve, seconds * 1000));