diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 722694a32f6ad..f93602bd99350 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -45,4 +45,15 @@ export { isVirtualModelVersion, virtualVersionToModelVersion, modelVersionToVirtualVersion, + getModelVersionMapForTypes, + getLatestModelVersion, + type ModelVersionMap, + compareModelVersions, + type CompareModelVersionMapParams, + type CompareModelVersionStatus, + type CompareModelVersionDetails, + type CompareModelVersionResult, + getModelVersionsFromMappings, + getModelVersionsFromMappingMeta, + getModelVersionDelta, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index c93abc2064fb6..0267f2ce27c1a 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -57,8 +57,24 @@ export interface IndexMapping { /** @internal */ export interface IndexMappingMeta { - // A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa') - // with each key being a root-level mapping property, and each value being - // the md5 hash of that mapping's value when the index was created. + /** + * A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa') + * with each key being a root-level mapping property, and each value being + * the md5 hash of that mapping's value when the index was created. + * + * @remark: Only defined for indices using the v2 migration algorithm. + */ migrationMappingPropertyHashes?: { [k: string]: string }; + /** + * The current model versions of the mapping of the index. + * + * @remark: Only defined for indices using the zdt migration algorithm. + */ + mappingVersions?: { [k: string]: number }; + /** + * The current model versions of the documents of the index. + * + * @remark: Only defined for indices using the zdt migration algorithm. + */ + docVersions?: { [k: string]: number }; } diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts new file mode 100644 index 0000000000000..521ddd2a0efc3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getModelVersionDelta } from './get_version_delta'; + +describe('getModelVersionDelta', () => { + it('generates an upward delta', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 1, + b: 1, + }, + targetVersions: { + a: 2, + b: 3, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('upward'); + expect(result.diff).toEqual([ + { + name: 'a', + current: 1, + target: 2, + }, + { + name: 'b', + current: 1, + target: 3, + }, + ]); + }); + + it('generates a downward delta', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 4, + b: 2, + }, + targetVersions: { + a: 1, + b: 1, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('downward'); + expect(result.diff).toEqual([ + { + name: 'a', + current: 4, + target: 1, + }, + { + name: 'b', + current: 2, + target: 1, + }, + ]); + }); + + it('generates a noop delta', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 4, + b: 2, + }, + targetVersions: { + a: 4, + b: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('noop'); + expect(result.diff).toEqual([]); + }); + + it('ignores deleted types', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 1, + b: 3, + }, + targetVersions: { + a: 2, + }, + deletedTypes: ['b'], + }); + + expect(result.status).toEqual('upward'); + expect(result.diff).toEqual([ + { + name: 'a', + current: 1, + target: 2, + }, + ]); + }); + + it('throws if the provided version maps are in conflict', () => { + expect(() => + getModelVersionDelta({ + currentVersions: { + a: 1, + b: 2, + }, + targetVersions: { + a: 2, + b: 1, + }, + deletedTypes: [], + }) + ).toThrow(); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts new file mode 100644 index 0000000000000..f39c52b47f9f7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ModelVersionMap } from './version_map'; +import { compareModelVersions } from './version_compare'; + +interface GetModelVersionDeltaOpts { + currentVersions: ModelVersionMap; + targetVersions: ModelVersionMap; + deletedTypes: string[]; +} + +type ModelVersionDeltaResultStatus = 'upward' | 'downward' | 'noop'; + +interface ModelVersionDeltaResult { + status: ModelVersionDeltaResultStatus; + diff: ModelVersionDeltaTypeResult[]; +} + +interface ModelVersionDeltaTypeResult { + /** the name of the type */ + name: string; + /** the current version the type is at */ + current: number; + /** the target version the type should go to */ + target: number; +} + +/** + * Will generate the difference to go from `currentVersions` to `targetVersions`. + * + * @remarks: will throw if the version maps are in conflict + */ +export const getModelVersionDelta = ({ + currentVersions, + targetVersions, + deletedTypes, +}: GetModelVersionDeltaOpts): ModelVersionDeltaResult => { + const compared = compareModelVersions({ + indexVersions: currentVersions, + appVersions: targetVersions, + deletedTypes, + }); + + if (compared.status === 'conflict') { + throw new Error('Cannot generate model version difference: conflict between versions'); + } + + const status: ModelVersionDeltaResultStatus = + compared.status === 'lesser' ? 'downward' : compared.status === 'greater' ? 'upward' : 'noop'; + + const result: ModelVersionDeltaResult = { + status, + diff: [], + }; + + if (compared.status === 'greater') { + compared.details.greater.forEach((type) => { + result.diff.push(getTypeDelta({ type, currentVersions, targetVersions })); + }); + } else if (compared.status === 'lesser') { + compared.details.lesser.forEach((type) => { + result.diff.push(getTypeDelta({ type, currentVersions, targetVersions })); + }); + } + + return result; +}; + +const getTypeDelta = ({ + type, + currentVersions, + targetVersions, +}: { + type: string; + currentVersions: ModelVersionMap; + targetVersions: ModelVersionMap; +}): ModelVersionDeltaTypeResult => { + const currentVersion = currentVersions[type]; + const targetVersion = targetVersions[type]; + if (currentVersion === undefined || targetVersion === undefined) { + // should never occur given we've been checking consistency numerous times before getting there + // but better safe than sorry. + throw new Error( + `Consistency error: trying to generate delta with missing entry for type ${type}` + ); + } + return { + name: type, + current: currentVersion, + target: targetVersion, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 5301c0a4d219c..2179199921a82 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -13,3 +13,20 @@ export { modelVersionToVirtualVersion, virtualVersionToModelVersion, } from './conversion'; +export { + getModelVersionMapForTypes, + getLatestModelVersion, + type ModelVersionMap, +} from './version_map'; +export { + compareModelVersions, + type CompareModelVersionMapParams, + type CompareModelVersionStatus, + type CompareModelVersionDetails, + type CompareModelVersionResult, +} from './version_compare'; +export { + getModelVersionsFromMappings, + getModelVersionsFromMappingMeta, +} from './model_version_from_mappings'; +export { getModelVersionDelta } from './get_version_delta'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts new file mode 100644 index 0000000000000..8fea10f11f6b1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMapping, IndexMappingMeta } from '../mappings'; +import { getModelVersionsFromMappings } from './model_version_from_mappings'; + +describe('getModelVersionsFromMappings', () => { + const createIndexMapping = (parts: Partial = {}): IndexMapping => ({ + properties: {}, + _meta: { + ...parts, + }, + }); + + it('retrieves the version map from docVersions', () => { + const mappings = createIndexMapping({ + docVersions: { + foo: 3, + bar: 5, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + + expect(versionMap).toEqual({ + foo: 3, + bar: 5, + }); + }); + + it('retrieves the version map from mappingVersions', () => { + const mappings = createIndexMapping({ + mappingVersions: { + foo: 2, + bar: 7, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + + expect(versionMap).toEqual({ + foo: 2, + bar: 7, + }); + }); + + it('returns undefined for docVersions if meta field is not present', () => { + const mappings = createIndexMapping({ + mappingVersions: { + foo: 3, + bar: 5, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + + expect(versionMap).toBeUndefined(); + }); + + it('returns undefined for mappingVersions if meta field is not present', () => { + const mappings = createIndexMapping({ + docVersions: { + foo: 3, + bar: 5, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + + expect(versionMap).toBeUndefined(); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts new file mode 100644 index 0000000000000..8e7816a12fb53 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMapping, IndexMappingMeta } from '../mappings'; +import type { ModelVersionMap } from './version_map'; +import { assertValidModelVersion } from './conversion'; + +/** + * Build the version map from the specified source of the provided mappings. + */ +export const getModelVersionsFromMappings = ({ + mappings, + source, +}: { + mappings: IndexMapping; + source: 'mappingVersions' | 'docVersions'; +}): ModelVersionMap | undefined => { + if (!mappings._meta) { + return undefined; + } + + return getModelVersionsFromMappingMeta({ + meta: mappings._meta, + source, + }); +}; + +/** + * Build the version map from the specified source of the provided mappings meta. + */ +export const getModelVersionsFromMappingMeta = ({ + meta, + source, +}: { + meta: IndexMappingMeta; + source: 'mappingVersions' | 'docVersions'; +}): ModelVersionMap | undefined => { + const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; + if (!indexVersions) { + return undefined; + } + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { + map[type] = assertValidModelVersion(rawVersion); + return map; + }, {}); +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts new file mode 100644 index 0000000000000..eba6fe1837cce --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { compareModelVersions } from './version_compare'; + +describe('compareModelVersions', () => { + it('returns the correct value for greater app version', () => { + const result = compareModelVersions({ + appVersions: { + foo: 3, + bar: 2, + }, + indexVersions: { + foo: 2, + bar: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('greater'); + }); + + it('returns the correct value for lesser app version', () => { + const result = compareModelVersions({ + appVersions: { + foo: 1, + bar: 2, + }, + indexVersions: { + foo: 2, + bar: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('lesser'); + }); + + it('returns the correct value for equal versions', () => { + const result = compareModelVersions({ + appVersions: { + foo: 2, + bar: 2, + }, + indexVersions: { + foo: 2, + bar: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('equal'); + }); + + it('handles new types not being present in the index', () => { + const result = compareModelVersions({ + appVersions: { + foo: 2, + new: 1, + }, + indexVersions: { + foo: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('greater'); + }); + + it('handles types not being present in the app', () => { + const result = compareModelVersions({ + appVersions: { + foo: 3, + }, + indexVersions: { + foo: 2, + old: 1, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('conflict'); + }); + + it('returns the correct value for conflicts', () => { + const result = compareModelVersions({ + appVersions: { + a: 3, + b: 3, + c: 3, + }, + indexVersions: { + a: 2, + b: 3, + c: 4, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('conflict'); + }); + + it('properly lists the details', () => { + const result = compareModelVersions({ + appVersions: { + a: 3, + b: 3, + c: 3, + }, + indexVersions: { + a: 2, + b: 3, + c: 4, + }, + deletedTypes: [], + }); + + expect(result.details.lesser).toEqual(['c']); + expect(result.details.equal).toEqual(['b']); + expect(result.details.greater).toEqual(['a']); + }); + + it('ignores deleted types when comparing', () => { + const result = compareModelVersions({ + appVersions: { + a: 3, + }, + indexVersions: { + a: 2, + b: 3, + }, + deletedTypes: ['b'], + }); + + expect(result.status).toEqual('greater'); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts new file mode 100644 index 0000000000000..9b8d14b7fd862 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ModelVersionMap } from './version_map'; + +export interface CompareModelVersionMapParams { + /** The latest model version of the types registered in the application */ + appVersions: ModelVersionMap; + /** The model version stored in the index */ + indexVersions: ModelVersionMap; + /** The list of deleted types to exclude during the compare process */ + deletedTypes: string[]; +} + +/** + * The overall status of the model version comparison: + * - `greater`: app version is greater than the index version + * - `lesser`: app version is lesser than the index version + * - `equal`: app version is equal to the index version + * - `conflict`: app and index versions are incompatible (versions for some types are higher, and for other types lower) + */ +export type CompareModelVersionStatus = 'greater' | 'lesser' | 'equal' | 'conflict'; + +export interface CompareModelVersionDetails { + greater: string[]; + lesser: string[]; + equal: string[]; +} + +export interface CompareModelVersionResult { + status: CompareModelVersionStatus; + details: CompareModelVersionDetails; +} + +export const compareModelVersions = ({ + appVersions, + indexVersions, + deletedTypes, +}: CompareModelVersionMapParams): CompareModelVersionResult => { + const allTypes = [ + ...new Set([...Object.keys(appVersions), ...Object.keys(indexVersions)]), + ].filter((type) => !deletedTypes.includes(type)); + + const details: CompareModelVersionDetails = { + greater: [], + lesser: [], + equal: [], + }; + + allTypes.forEach((type) => { + const appVersion = appVersions[type] ?? 0; + const indexVersion = indexVersions[type] ?? 0; + + if (appVersion > indexVersion) { + details.greater.push(type); + } else if (appVersion < indexVersion) { + details.lesser.push(type); + } else { + details.equal.push(type); + } + }); + + const hasGreater = details.greater.length > 0; + const hasLesser = details.lesser.length > 0; + const status: CompareModelVersionStatus = + hasGreater && hasLesser ? 'conflict' : hasGreater ? 'greater' : hasLesser ? 'lesser' : 'equal'; + + return { + status, + details, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts new file mode 100644 index 0000000000000..aafb83ab96009 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType, SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { getModelVersionMapForTypes, getLatestModelVersion } from './version_map'; + +describe('ModelVersion map utilities', () => { + const buildType = (parts: Partial = {}): SavedObjectsType => ({ + name: 'test-type', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, + }); + + const dummyModelVersion = (): SavedObjectsModelVersion => ({ + modelChange: { + type: 'expansion', + }, + }); + + describe('getLatestModelVersion', () => { + it('returns 0 when no model versions are registered', () => { + expect(getLatestModelVersion(buildType({ modelVersions: {} }))).toEqual(0); + expect(getLatestModelVersion(buildType({ modelVersions: undefined }))).toEqual(0); + }); + + it('throws if an invalid version is provided', () => { + expect(() => + getLatestModelVersion( + buildType({ + modelVersions: { + foo: dummyModelVersion(), + }, + }) + ) + ).toThrow(); + }); + + it('returns the latest registered version', () => { + expect( + getLatestModelVersion( + buildType({ + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersion(), + '3': dummyModelVersion(), + }, + }) + ) + ).toEqual(3); + }); + + it('accepts provider functions', () => { + expect( + getLatestModelVersion( + buildType({ + modelVersions: () => ({ + '1': dummyModelVersion(), + '2': dummyModelVersion(), + '3': dummyModelVersion(), + }), + }) + ) + ).toEqual(3); + }); + + it('supports unordered maps', () => { + expect( + getLatestModelVersion( + buildType({ + modelVersions: { + '3': dummyModelVersion(), + '1': dummyModelVersion(), + '2': dummyModelVersion(), + }, + }) + ) + ).toEqual(3); + }); + }); + + describe('getModelVersionMapForTypes', () => { + it('returns a map with the latest version of the provided types', () => { + expect( + getModelVersionMapForTypes([ + buildType({ + name: 'foo', + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersion(), + }, + }), + buildType({ + name: 'bar', + modelVersions: {}, + }), + buildType({ + name: 'dolly', + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersion(), + '3': dummyModelVersion(), + }, + }), + ]) + ).toEqual({ + foo: 2, + bar: 0, + dolly: 3, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts new file mode 100644 index 0000000000000..dd05e64dbcbef --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { assertValidModelVersion } from './conversion'; + +export type ModelVersionMap = Record; + +/** + * Returns the latest registered model version number for the given type. + */ +export const getLatestModelVersion = (type: SavedObjectsType): number => { + const versionMap = + typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {}; + return Object.keys(versionMap).reduce((memo, current) => { + return Math.max(memo, assertValidModelVersion(current)); + }, 0); +}; + +/** + * Build a version map for the given types. + */ +export const getModelVersionMapForTypes = (types: SavedObjectsType[]): ModelVersionMap => { + return types.reduce((versionMap, type) => { + versionMap[type.name] = getLatestModelVersion(type); + return versionMap; + }, {}); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index 7e380d3a7ad17..7aec839dca066 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -37,7 +37,11 @@ export { removeWriteBlock } from './remove_write_block'; export type { CloneIndexResponse, CloneIndexParams } from './clone_index'; export { cloneIndex } from './clone_index'; -export type { WaitForIndexStatusParams, IndexNotYellowTimeout } from './wait_for_index_status'; +export type { + WaitForIndexStatusParams, + IndexNotYellowTimeout, + IndexNotGreenTimeout, +} from './wait_for_index_status'; import type { IndexNotGreenTimeout, IndexNotYellowTimeout } from './wait_for_index_status'; import { waitForIndexStatus } from './wait_for_index_status'; @@ -78,7 +82,7 @@ export { cleanupUnknownAndExcluded } from './cleanup_unknown_and_excluded'; export { waitForDeleteByQueryTask } from './wait_for_delete_by_query_task'; -export type { CreateIndexParams } from './create_index'; +export type { CreateIndexParams, ClusterShardLimitExceeded } from './create_index'; export { createIndex } from './create_index'; export { checkTargetMappings } from './check_target_mappings'; @@ -91,7 +95,7 @@ export type { } from './update_and_pickup_mappings'; export { updateAndPickupMappings } from './update_and_pickup_mappings'; -export { updateMappings } from './update_mappings'; +export { updateMappings, type IncompatibleMappingException } from './update_mappings'; import type { UnknownDocsFound } from './check_for_unknown_docs'; import type { IncompatibleClusterRoutingAllocation } from './initialize_action'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts new file mode 100644 index 0000000000000..5dfdb05a0bca8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts new file mode 100644 index 0000000000000..3701621a96a12 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createDelayFn } from './delay'; + +const nextTick = () => new Promise((resolve) => resolve()); + +describe('createDelayFn', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('adds a delay effect to the provided function', async () => { + const handler = jest.fn(); + + const wrapped = createDelayFn({ retryDelay: 2000, retryCount: 0 })(handler); + + wrapped(); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(500); + await nextTick(); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(1500); + await nextTick(); + + expect(handler).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts new file mode 100644 index 0000000000000..8a8a87ec65def --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface RetryableState { + retryCount: number; + retryDelay: number; +} + +/** + * HOC wrapping the function with a delay. + */ +export const createDelayFn = + (state: RetryableState) => + any>(fn: F): (() => ReturnType) => { + return () => { + return state.retryDelay > 0 + ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) + : fn(); + }; + }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts index 962f40b87db02..3801e180ed024 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts @@ -7,3 +7,4 @@ */ export { logActionResponse, logStateTransition, type LogAwareState } from './logs'; +export { createDelayFn, type RetryableState } from './delay'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts new file mode 100644 index 0000000000000..e3bb8451b0ae9 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { omit } from 'lodash'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { logStateTransition, type LogAwareState } from './logs'; + +describe('logStateTransition', () => { + let logger: MockedLogger; + + const messagePrefix = '[PREFIX] '; + + beforeEach(() => { + logger = loggerMock.create(); + }); + + it('logs the offset of messages between the old and the new state', () => { + const previous: LogAwareState = { + controlState: 'PREVIOUS', + logs: [], + }; + const next: LogAwareState = { + controlState: 'NEXT', + logs: [ + ...previous.logs, + { level: 'info', message: 'info message' }, + { level: 'warning', message: 'warning message' }, + ], + }; + + logStateTransition(logger, messagePrefix, previous, next, 500); + + expect(omit(loggerMock.collect(logger), 'debug')).toEqual({ + error: [], + fatal: [], + info: [['[PREFIX] info message'], ['[PREFIX] PREVIOUS -> NEXT. took: 500ms.']], + log: [], + trace: [], + warn: [['[PREFIX] warning message']], + }); + }); + + it('logs a debug message with the correct meta', () => { + const previous: LogAwareState = { + controlState: 'PREVIOUS', + logs: [], + }; + const next: LogAwareState = { + controlState: 'NEXT', + logs: [ + ...previous.logs, + { level: 'info', message: 'info message' }, + { level: 'warning', message: 'warning message' }, + ], + }; + + logStateTransition(logger, messagePrefix, previous, next, 500); + + expect(loggerMock.collect(logger).debug).toEqual([ + [ + '[PREFIX] PREVIOUS -> NEXT. took: 500ms.', + { + kibana: { + migrations: { + duration: 500, + state: expect.objectContaining({ + controlState: 'NEXT', + }), + }, + }, + }, + ], + ]); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts index f14de6bc72ee0..3d8648ae33274 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts @@ -27,7 +27,7 @@ import type { export function buildActiveMappings( typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties ): IndexMapping { - const mapping = defaultMapping(); + const mapping = getBaseMappings(); const mergedProperties = validateAndMerge(mapping.properties, typeDefinitions); @@ -114,7 +114,7 @@ function findChangedProp(actual: any, expected: any) { * * @returns {IndexMapping} */ -function defaultMapping(): IndexMapping { +export function getBaseMappings(): IndexMapping { return { dynamic: 'strict', properties: { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts index a113e5e5f77bc..1503fdcf19814 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export { buildActiveMappings } from './build_active_mappings'; +export { buildActiveMappings, getBaseMappings } from './build_active_mappings'; export type { LogFn } from './migration_logger'; export { excludeUnusedTypesQuery, REMOVED_TYPES } from './unused_types'; export { TransformSavedObjectDocumentError } from './transform_saved_object_document_error'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index 408b277444995..d63cc69d8f7f4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -145,6 +145,7 @@ export class KibanaMigrator implements IKibanaMigrator { private runMigrationZdt(): Promise { return runZeroDowntimeMigration({ + kibanaVersion: this.kibanaVersion, kibanaIndexPrefix: this.kibanaIndex, typeRegistry: this.typeRegistry, logger: this.log, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index dbaacdbdd6808..45dbcc877bbdb 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -44,9 +44,9 @@ import { import { createBatches } from './create_batches'; import type { MigrationLog } from '../types'; import { diffMappings } from '../core/build_active_mappings'; +import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../common/constants'; export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; -const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; export const model = (currentState: State, resW: ResponseType): State => { // The action response `resW` is weakly typed, the type includes all action diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts index 78042acd7a6a4..1109d52039d3e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts @@ -50,6 +50,7 @@ import type { WaitForMigrationCompletionState, WaitForYellowSourceState, } from './state'; +import { createDelayFn } from './common/utils'; import type { TransformRawDocs } from './types'; import * as Actions from './actions'; import { REMOVED_TYPES } from './core'; @@ -253,13 +254,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra export const next = (client: ElasticsearchClient, transformRawDocs: TransformRawDocs) => { const map = nextActionMap(client, transformRawDocs); return (state: State) => { - const delay = any>(fn: F): (() => ReturnType) => { - return () => { - return state.retryDelay > 0 - ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) - : fn(); - }; - }; + const delay = createDelayFn(state); if (state.controlState === 'DONE' || state.controlState === 'FATAL') { // Return null if we're in one of the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index 92334d396adc0..bb135c115ce92 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -10,11 +10,22 @@ import type { IncompatibleClusterRoutingAllocation, RetryableEsClientError, WaitForTaskCompletionTimeout, + IndexNotYellowTimeout, + IndexNotGreenTimeout, + ClusterShardLimitExceeded, IndexNotFound, + AliasNotFound, + IncompatibleMappingException, } from '../../actions'; export { initAction as init, + waitForIndexStatus, + createIndex, + updateAliases, + updateMappings, + updateAndPickupMappings, + waitForPickupUpdatedMappingsTask, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -27,6 +38,11 @@ export interface ActionErrorTypeMap { incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation; retryable_es_client_error: RetryableEsClientError; index_not_found_exception: IndexNotFound; + index_not_green_timeout: IndexNotGreenTimeout; + index_not_yellow_timeout: IndexNotYellowTimeout; + cluster_shard_limit_exceeded: ClusterShardLimitExceeded; + alias_not_found_exception: AliasNotFound; + incompatible_mapping_exception: IncompatibleMappingException; } /** Type guard for narrowing the type of a left */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index cc4c7b63d3993..7a660ea470443 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -6,8 +6,10 @@ * Side Public License, v 1. */ -import type { MigratorContext } from './types'; +import { getModelVersionMapForTypes } from '@kbn/core-saved-objects-base-server-internal'; +import { REMOVED_TYPES } from '../../core'; import type { MigrateIndexOptions } from '../migrate_index'; +import type { MigratorContext } from './types'; export type CreateContextOps = Omit; @@ -15,6 +17,7 @@ export type CreateContextOps = Omit; * Create the context object that will be used for this index migration. */ export const createContext = ({ + kibanaVersion, types, docLinks, migrationConfig, @@ -24,12 +27,15 @@ export const createContext = ({ serializer, }: CreateContextOps): MigratorContext => { return { + kibanaVersion, indexPrefix, types, + typeModelVersions: getModelVersionMapForTypes(types.map((type) => typeRegistry.getType(type)!)), elasticsearchClient, typeRegistry, serializer, maxRetryAttempts: migrationConfig.retryAttempts, migrationDocLinks: docLinks.links.kibanaUpgradeSavedObjects, + deletedTypes: REMOVED_TYPES, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 2603a5b69a681..5b6d4b2fe27e9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -11,16 +11,21 @@ import type { ISavedObjectTypeRegistry, ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; +import type { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; /** * The set of static, precomputed values and services used by the ZDT migration */ export interface MigratorContext { + /** The current Kibana version */ + readonly kibanaVersion: string; /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ readonly indexPrefix: string; /** Name of the types that are living in the index */ readonly types: string[]; + /** Model versions for the registered types */ + readonly typeModelVersions: ModelVersionMap; /** The client to use for communications with ES */ readonly elasticsearchClient: ElasticsearchClient; /** The maximum number of retries to attempt for a failing action */ @@ -31,4 +36,6 @@ export interface MigratorContext { readonly serializer: ISavedObjectsSerializer; /** The SO type registry to use for the migration */ readonly typeRegistry: ISavedObjectTypeRegistry; + /** List of types that are no longer registered */ + readonly deletedTypes: string[]; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts index 72ee369236e16..6c3850d5dc8e9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts @@ -25,6 +25,7 @@ import { model } from './model'; import { createInitialState } from './state'; export interface MigrateIndexOptions { + kibanaVersion: string; indexPrefix: string; types: string[]; /** The SO type registry to use for the migration */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts index 70faab1fdc8d5..b59f1ef7b7aa4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts @@ -11,7 +11,7 @@ const realStages = jest.requireActual('./stages'); export const StageMocks = Object.keys(realStages).reduce((mocks, key) => { mocks[key] = jest.fn().mockImplementation((state: unknown) => state); return mocks; -}, {} as Record); +}, {} as Record>); jest.doMock('./stages', () => { return StageMocks; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts index b6c91e702bd2b..c0316b954e5f3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts @@ -10,7 +10,7 @@ import { StageMocks } from './model.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { createContextMock, MockedMigratorContext } from '../test_helpers'; import type { RetryableEsClientError } from '../../actions'; -import type { State, BaseState, FatalState } from '../state'; +import type { State, BaseState, FatalState, AllActionStates } from '../state'; import type { StateActionResponse } from './types'; import { model } from './model'; @@ -113,22 +113,39 @@ describe('model', () => { }); describe('dispatching to correct stage', () => { - test('dispatching INIT state', () => { - const state: State = { + const createStubState = (controlState: AllActionStates): State => + ({ ...baseState, - controlState: 'INIT', - }; - const res: StateActionResponse<'INIT'> = Either.right({ + controlState, + } as unknown as State); + + const createStubResponse = () => + Either.right({ '.kibana_7.11.0_001': { aliases: {}, mappings: { properties: {} }, settings: {}, }, }); - model(state, res, context); - expect(StageMocks.init).toHaveBeenCalledTimes(1); - expect(StageMocks.init).toHaveBeenCalledWith(state, res, context); + const stageMapping: Record = { + INIT: StageMocks.init, + CREATE_TARGET_INDEX: StageMocks.createTargetIndex, + UPDATE_INDEX_MAPPINGS: StageMocks.updateIndexMappings, + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: StageMocks.updateIndexMappingsWaitForTask, + UPDATE_MAPPING_MODEL_VERSIONS: StageMocks.updateMappingModelVersion, + UPDATE_ALIASES: StageMocks.updateAliases, + }; + + Object.entries(stageMapping).forEach(([stage, handler]) => { + test(`dispatch ${stage} state`, () => { + const state = createStubState(stage as AllActionStates); + const res = createStubResponse(); + model(state, res, context); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith(state, res, context); + }); }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 2ea48bd1a58af..62971c3a614aa 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -32,6 +32,36 @@ export const model = ( switch (current.controlState) { case 'INIT': return Stages.init(current, response as StateActionResponse<'INIT'>, context); + case 'CREATE_TARGET_INDEX': + return Stages.createTargetIndex( + current, + response as StateActionResponse<'CREATE_TARGET_INDEX'>, + context + ); + case 'UPDATE_ALIASES': + return Stages.updateAliases( + current, + response as StateActionResponse<'UPDATE_ALIASES'>, + context + ); + case 'UPDATE_INDEX_MAPPINGS': + return Stages.updateIndexMappings( + current, + response as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': + return Stages.updateIndexMappingsWaitForTask( + current, + response as StateActionResponse<'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'>, + context + ); + case 'UPDATE_MAPPING_MODEL_VERSIONS': + return Stages.updateMappingModelVersion( + current, + response as StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'>, + context + ); case 'DONE': case 'FATAL': // The state-action machine will never call the model in the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts new file mode 100644 index 0000000000000..b85e06bde59bc --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getAliasActionsMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getAliasActions: getAliasActionsMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts new file mode 100644 index 0000000000000..cd15594d32b29 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getAliasActionsMock } from './create_target_index.test.mocks'; +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CreateTargetIndexState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { createTargetIndex } from './create_target_index'; + +describe('Stage: createTargetIndex', () => { + let context: MockedMigratorContext; + + const createState = (parts: Partial = {}): CreateTargetIndexState => ({ + ...createPostInitState(), + controlState: 'CREATE_TARGET_INDEX', + indexMappings: { properties: { foo: { type: 'text' } }, _meta: {} }, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + getAliasActionsMock.mockReset().mockReturnValue([]); + }); + + describe('In case of left return', () => { + it('CREATE_TARGET_INDEX -> CREATE_TARGET_INDEX in case of index_not_green_timeout exception', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.left({ + type: 'index_not_green_timeout', + message: 'index not green', + }); + + const result = createTargetIndex(state, res, context); + + expect(result).toEqual({ + ...state, + controlState: 'CREATE_TARGET_INDEX', + retryCount: 1, + retryDelay: 2000, + logs: expect.any(Array), + }); + }); + + it('CREATE_TARGET_INDEX -> FATAL in case of cluster_shard_limit_exceeded exception', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.left({ + type: 'cluster_shard_limit_exceeded', + }); + + const result = createTargetIndex(state, res, context); + + expect(result).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.stringContaining('[cluster_shard_limit_exceeded]'), + }); + }); + }); + + describe('In case of right return', () => { + it('calls getAliasActions with the correct parameters', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + createTargetIndex(state, res, context); + + expect(getAliasActionsMock).toHaveBeenCalledTimes(1); + expect(getAliasActionsMock).toHaveBeenCalledWith({ + currentIndex: state.currentIndex, + existingAliases: [], + indexPrefix: context.indexPrefix, + kibanaVersion: context.kibanaVersion, + }); + }); + + it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + const aliasActions = [{ add: { index: '.kibana_1', alias: '.kibana' } }]; + getAliasActionsMock.mockReturnValue(aliasActions); + + const newState = createTargetIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_ALIASES', + previousMappings: state.indexMappings, + currentIndexMeta: state.indexMappings._meta, + aliases: [], + aliasActions, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts new file mode 100644 index 0000000000000..bb0697a70cf51 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { cloneDeep } from 'lodash'; +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { throwBadResponse } from '../../../model/helpers'; +import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../../../common/constants'; +import { isTypeof } from '../../actions'; +import { getAliasActions } from '../../utils'; +import type { ModelStage } from '../types'; + +export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASES' | 'FATAL'> = ( + state, + res, + context +) => { + if (Either.isLeft(res)) { + const left = res.left; + if (isTypeof(left, 'index_not_green_timeout')) { + // cluster might just be busy so we retry the action for a set number of times. + const retryErrorMessage = `${left.message} Refer to ${context.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; + return delayRetryState(state, retryErrorMessage, context.maxRetryAttempts); + } else if (isTypeof(left, 'cluster_shard_limit_exceeded')) { + return { + ...state, + controlState: 'FATAL', + reason: `${CLUSTER_SHARD_LIMIT_EXCEEDED_REASON} See ${context.migrationDocLinks.clusterShardLimitExceeded}`, + }; + } else { + return throwBadResponse(state, left); + } + } + + const aliasActions = getAliasActions({ + currentIndex: state.currentIndex, + existingAliases: [], + indexPrefix: context.indexPrefix, + kibanaVersion: context.kibanaVersion, + }); + + const currentIndexMeta = cloneDeep(state.indexMappings._meta!); + + return { + ...state, + controlState: 'UPDATE_ALIASES', + previousMappings: state.indexMappings, + currentIndexMeta, + aliases: [], + aliasActions, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index aa12c6bed1b22..0322f92eb35aa 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -7,3 +7,8 @@ */ export { init } from './init'; +export { createTargetIndex } from './create_target_index'; +export { updateAliases } from './update_aliases'; +export { updateIndexMappings } from './update_index_mappings'; +export { updateIndexMappingsWaitForTask } from './update_index_mappings_wait_for_task'; +export { updateMappingModelVersion } from './update_mapping_model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts new file mode 100644 index 0000000000000..8f773fe951171 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getCurrentIndexMock = jest.fn(); +export const checkVersionCompatibilityMock = jest.fn(); +export const buildIndexMappingsMock = jest.fn(); +export const generateAdditiveMappingDiffMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getCurrentIndex: getCurrentIndexMock, + checkVersionCompatibility: checkVersionCompatibilityMock, + buildIndexMappings: buildIndexMappingsMock, + generateAdditiveMappingDiff: generateAdditiveMappingDiffMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index 8105449c7fce8..ea6f4424404ef 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -6,15 +6,24 @@ * Side Public License, v 1. */ +import { + getCurrentIndexMock, + checkVersionCompatibilityMock, + buildIndexMappingsMock, + generateAdditiveMappingDiffMock, +} from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; +import { FetchIndexResponse } from '../../../actions'; import { createContextMock, MockedMigratorContext } from '../../test_helpers'; import type { InitState } from '../../state'; import type { StateActionResponse } from '../types'; import { init } from './init'; -describe('Action: init', () => { +describe('Stage: init', () => { let context: MockedMigratorContext; + const currentIndex = '.kibana_1'; + const createState = (parts: Partial = {}): InitState => ({ controlState: 'INIT', retryDelay: 0, @@ -23,29 +32,40 @@ describe('Action: init', () => { ...parts, }); - beforeEach(() => { - context = createContextMock(); + const createResponse = (): FetchIndexResponse => ({ + [currentIndex]: { + aliases: {}, + mappings: { + properties: {}, + _meta: { mappingVersions: { foo: 1, bar: 1 } }, + }, + settings: {}, + }, }); - test('INIT -> DONE because its not implemented yet', () => { - const state = createState(); - const res: StateActionResponse<'INIT'> = Either.right({ - '.kibana_8.7.0_001': { - aliases: { - '.kibana': {}, - '.kibana_8.7.0': {}, - }, - mappings: { properties: {} }, - settings: {}, - }, + beforeEach(() => { + getCurrentIndexMock.mockReset().mockReturnValue(currentIndex); + checkVersionCompatibilityMock.mockReset().mockReturnValue({ + status: 'equal', }); + generateAdditiveMappingDiffMock.mockReset().mockReturnValue({}); - const newState = init(state, res, context); - - expect(newState.controlState).toEqual('DONE'); + context = createContextMock({ indexPrefix: '.kibana', types: ['foo', 'bar'] }); + context.typeRegistry.registerType({ + name: 'foo', + mappings: { properties: {} }, + namespaceType: 'single', + hidden: false, + }); + context.typeRegistry.registerType({ + name: 'bar', + mappings: { properties: {} }, + namespaceType: 'single', + hidden: false, + }); }); - test('INIT -> INIT when cluster routing allocation is incompatible', () => { + it('loops to INIT when cluster routing allocation is incompatible', () => { const state = createState(); const res: StateActionResponse<'INIT'> = Either.left({ type: 'incompatible_cluster_routing_allocation', @@ -58,4 +78,236 @@ describe('Action: init', () => { expect(newState.retryDelay).toEqual(2000); expect(newState.logs).toHaveLength(1); }); + + it('calls getCurrentIndex with the correct parameters', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + init(state, res, context); + + expect(getCurrentIndexMock).toHaveBeenCalledTimes(1); + expect(getCurrentIndexMock).toHaveBeenCalledWith(fetchIndexResponse, context.indexPrefix); + }); + + it('calls checkVersionCompatibility with the correct parameters', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + init(state, res, context); + + expect(checkVersionCompatibilityMock).toHaveBeenCalledTimes(1); + expect(checkVersionCompatibilityMock).toHaveBeenCalledWith({ + mappings: fetchIndexResponse[currentIndex].mappings, + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + source: 'mappingVersions', + deletedTypes: context.deletedTypes, + }); + }); + + describe('when getCurrentIndex returns undefined', () => { + beforeEach(() => { + getCurrentIndexMock.mockReturnValue(undefined); + }); + + it('calls buildIndexMappings with the correct parameters', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + init(state, res, context); + + expect(buildIndexMappingsMock).toHaveBeenCalledTimes(1); + expect(buildIndexMappingsMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + }); + }); + + it('forwards to CREATE_TARGET_INDEX', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + const mockMappings = { properties: { someMappings: 'string' } }; + buildIndexMappingsMock.mockReturnValue(mockMappings); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'CREATE_TARGET_INDEX', + currentIndex: '.kibana_1', + indexMappings: mockMappings, + }) + ); + }); + }); + + describe('when checkVersionCompatibility returns `greater`', () => { + it('calls generateAdditiveMappingDiff with the correct parameters', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'greater', + }); + + init(state, res, context); + + expect(generateAdditiveMappingDiffMock).toHaveBeenCalledTimes(1); + expect(generateAdditiveMappingDiffMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + meta: fetchIndexResponse[currentIndex].mappings._meta, + deletedTypes: context.deletedTypes, + }); + }); + + it('forwards to UPDATE_INDEX_MAPPINGS', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'greater', + }); + generateAdditiveMappingDiffMock.mockReturnValue({ someToken: {} }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'UPDATE_INDEX_MAPPINGS', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + additiveMappingChanges: { someToken: {} }, + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'greater', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: greater`, + ]); + }); + }); + + describe('when checkVersionCompatibility returns `equal`', () => { + it('forwards to UPDATE_ALIASES', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'UPDATE_ALIASES', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: equal`, + ]); + }); + }); + + describe('when checkVersionCompatibility returns `lesser`', () => { + it('forwards to FATAL', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'lesser', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'FATAL', + reason: 'Downgrading model version is currently unsupported', + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'lesser', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: lesser`, + ]); + }); + }); + + describe('when checkVersionCompatibility returns `conflict`', () => { + it('forwards to FATAL', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'conflict', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'FATAL', + reason: 'Model version conflict: inconsistent higher/lower versions', + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'conflict', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: conflict`, + ]); + }); + }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index 78dccf237afca..da138730364f4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -6,14 +6,25 @@ * Side Public License, v 1. */ +import { cloneDeep } from 'lodash'; import * as Either from 'fp-ts/lib/Either'; import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; +import type { MigrationLog } from '../../../types'; import { isTypeof } from '../../actions'; -import type { State } from '../../state'; +import { + getCurrentIndex, + checkVersionCompatibility, + buildIndexMappings, + getAliasActions, + generateAdditiveMappingDiff, +} from '../../utils'; import type { ModelStage } from '../types'; -export const init: ModelStage<'INIT', 'DONE' | 'FATAL'> = (state, res, context): State => { +export const init: ModelStage< + 'INIT', + 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_ALIASES' | 'FATAL' +> = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'incompatible_cluster_routing_allocation')) { @@ -24,9 +35,101 @@ export const init: ModelStage<'INIT', 'DONE' | 'FATAL'> = (state, res, context): } } - // nothing implemented yet, just going to 'DONE' - return { - ...state, - controlState: 'DONE', - }; + const types = context.types.map((type) => context.typeRegistry.getType(type)!); + const logs: MigrationLog[] = [...state.logs]; + + const indices = res.right; + const currentIndex = getCurrentIndex(indices, context.indexPrefix); + + // No indices were found, likely because it is the first time Kibana boots. + // In that case, we just create the index. + if (!currentIndex) { + return { + ...state, + logs, + controlState: 'CREATE_TARGET_INDEX', + currentIndex: `${context.indexPrefix}_1`, + indexMappings: buildIndexMappings({ types }), + }; + } + + // Index was found. This is the standard scenario, we check the model versions + // compatibility before going further. + const currentMappings = indices[currentIndex].mappings; + const versionCheck = checkVersionCompatibility({ + mappings: currentMappings, + types, + source: 'mappingVersions', + deletedTypes: context.deletedTypes, + }); + + logs.push({ + level: 'info', + message: `Mappings model version check result: ${versionCheck.status}`, + }); + + const aliases = Object.keys(indices[currentIndex].aliases); + const aliasActions = getAliasActions({ + existingAliases: aliases, + currentIndex, + indexPrefix: context.indexPrefix, + kibanaVersion: context.kibanaVersion, + }); + // cloning as we may be mutating it in later stages. + const currentIndexMeta = cloneDeep(currentMappings._meta!); + + switch (versionCheck.status) { + // app version is greater than the index mapping version. + // scenario of an upgrade: we need to update the mappings + case 'greater': + const additiveMappingChanges = generateAdditiveMappingDiff({ + types, + meta: currentMappings._meta ?? {}, + deletedTypes: context.deletedTypes, + }); + return { + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS', + logs, + currentIndex, + currentIndexMeta, + aliases, + aliasActions, + previousMappings: currentMappings, + additiveMappingChanges, + }; + // app version and index mapping version are the same. + // either application upgrade without model change, or a simple reboot on the same version. + // In that case we jump directly to alias update + case 'equal': + return { + ...state, + controlState: 'UPDATE_ALIASES', + logs, + currentIndex, + currentIndexMeta, + aliases, + aliasActions, + previousMappings: currentMappings, + }; + // app version is lower than the index mapping version. + // likely a rollback scenario - unsupported for the initial implementation + case 'lesser': + return { + ...state, + controlState: 'FATAL', + reason: 'Downgrading model version is currently unsupported', + logs, + }; + // conflicts: version for some types are greater, some are lower + // shouldn't occur in any normal scenario - cannot recover + case 'conflict': + default: + return { + ...state, + controlState: 'FATAL', + reason: 'Model version conflict: inconsistent higher/lower versions', + logs, + }; + } }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts new file mode 100644 index 0000000000000..4fac3d02db044 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateAliasesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateAliases } from './update_aliases'; + +describe('Stage: updateAliases', () => { + let context: MockedMigratorContext; + + const createState = (parts: Partial = {}): UpdateAliasesState => ({ + ...createPostInitState(), + controlState: 'UPDATE_ALIASES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_ALIASES -> FATAL in case of alias_not_found_exception', () => { + const state = createState(); + const res: StateActionResponse<'UPDATE_ALIASES'> = Either.left({ + type: 'alias_not_found_exception', + }); + + const newState = updateAliases(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: `Alias missing during alias update`, + }); + }); + + it('UPDATE_ALIASES -> FATAL in case of index_not_found_exception', () => { + const state = createState(); + const res: StateActionResponse<'UPDATE_ALIASES'> = Either.left({ + type: 'index_not_found_exception', + index: '.test', + }); + + const newState = updateAliases(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: `Index .test missing during alias update`, + }); + }); + + it('UPDATE_ALIASES -> DONE if successful', () => { + const state = createState(); + const res: StateActionResponse<'UPDATE_ALIASES'> = Either.right('update_aliases_succeeded'); + + const newState = updateAliases(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts new file mode 100644 index 0000000000000..4d91eb116871b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( + state, + res, + context +) => { + if (Either.isLeft(res)) { + const left = res.left; + if (isTypeof(left, 'alias_not_found_exception')) { + // Should never occur given a single operator is supposed to perform the migration. + // we just terminate in that case + return { + ...state, + controlState: 'FATAL', + reason: `Alias missing during alias update`, + }; + } else if (isTypeof(left, 'index_not_found_exception')) { + // Should never occur given a single operator is supposed to perform the migration. + // we just terminate in that case + return { + ...state, + controlState: 'FATAL', + reason: `Index ${left.index} missing during alias update`, + }; + } else { + throwBadResponse(state, left as never); + } + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts new file mode 100644 index 0000000000000..971482d3262b7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { UpdateIndexMappingsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateIndexMappings } from './update_index_mappings'; + +describe('Stage: updateIndexMappings', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateIndexMappingsState => ({ + ...createPostInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: {}, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ + taskId: '42', + }); + + const newState = updateIndexMappings( + state, + res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: '42', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts new file mode 100644 index 0000000000000..ffcd67aed2b78 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateIndexMappings: ModelStage< + 'UPDATE_INDEX_MAPPINGS', + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' | 'FATAL' +> = (state, res, context) => { + if (Either.isRight(res)) { + const right = res.right; + return { + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: right.taskId, + }; + } + + throwBadResponse(state, res as never); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts new file mode 100644 index 0000000000000..971482d3262b7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { UpdateIndexMappingsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateIndexMappings } from './update_index_mappings'; + +describe('Stage: updateIndexMappings', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateIndexMappingsState => ({ + ...createPostInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: {}, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ + taskId: '42', + }); + + const newState = updateIndexMappings( + state, + res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: '42', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts new file mode 100644 index 0000000000000..9856cb0c5a1e5 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import {} from 'lodash'; +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const updateIndexMappingsWaitForTask: ModelStage< + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + 'UPDATE_MAPPING_MODEL_VERSIONS' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + const left = res.left; + if (isTypeof(left, 'wait_for_task_completion_timeout')) { + // After waiting for the specified timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticsearch task succeeds or fails. + return delayRetryState(state, left.message, Number.MAX_SAFE_INTEGER); + } else { + throwBadResponse(state, left); + } + } + + return { + ...state, + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + currentIndexMeta: { + ...state.currentIndexMeta, + mappingVersions: { + ...context.typeModelVersions, + }, + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts new file mode 100644 index 0000000000000..971482d3262b7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { UpdateIndexMappingsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateIndexMappings } from './update_index_mappings'; + +describe('Stage: updateIndexMappings', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateIndexMappingsState => ({ + ...createPostInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: {}, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ + taskId: '42', + }); + + const newState = updateIndexMappings( + state, + res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: '42', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts new file mode 100644 index 0000000000000..8b4df56fc83a7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateMappingModelVersion: ModelStage< + 'UPDATE_MAPPING_MODEL_VERSIONS', + 'DONE' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index 329ba194b5957..a85e9bfde6b56 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -6,9 +6,19 @@ * Side Public License, v 1. */ -import type { AllActionStates, InitState, State } from './state'; +import type { + AllActionStates, + State, + InitState, + CreateTargetIndexState, + UpdateIndexMappingsState, + UpdateIndexMappingsWaitForTaskState, + UpdateMappingModelVersionState, + UpdateAliasesState, +} from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; +import { createDelayFn } from '../common/utils'; export type ActionMap = ReturnType; @@ -23,9 +33,45 @@ export type ResponseType = Awaited< >; export const nextActionMap = (context: MigratorContext) => { + const client = context.elasticsearchClient; return { INIT: (state: InitState) => - Actions.init({ client: context.elasticsearchClient, indices: [context.indexPrefix] }), + Actions.init({ + client, + indices: [`${context.indexPrefix}_*`], + }), + CREATE_TARGET_INDEX: (state: CreateTargetIndexState) => + Actions.createIndex({ + client, + indexName: state.currentIndex, + mappings: state.indexMappings, + }), + UPDATE_INDEX_MAPPINGS: (state: UpdateIndexMappingsState) => + Actions.updateAndPickupMappings({ + client, + index: state.currentIndex, + mappings: { properties: state.additiveMappingChanges }, + }), + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: (state: UpdateIndexMappingsWaitForTaskState) => + Actions.waitForPickupUpdatedMappingsTask({ + client, + taskId: state.updateTargetMappingsTaskId, + timeout: '60s', + }), + UPDATE_MAPPING_MODEL_VERSIONS: (state: UpdateMappingModelVersionState) => + Actions.updateMappings({ + client, + index: state.currentIndex, + mappings: { + properties: {}, + _meta: state.currentIndexMeta, + }, + }), + UPDATE_ALIASES: (state: UpdateAliasesState) => + Actions.updateAliases({ + client, + aliasActions: state.aliasActions, + }), }; }; @@ -33,13 +79,7 @@ export const next = (context: MigratorContext) => { const map = nextActionMap(context); return (state: State) => { - const delay = any>(fn: F): (() => ReturnType) => { - return () => { - return state.retryDelay > 0 - ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) - : fn(); - }; - }; + const delay = createDelayFn(state); if (state.controlState === 'DONE' || state.controlState === 'FATAL') { // Return null if we're in one of the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts index 8a2686e23764b..44a067944bd05 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts @@ -22,7 +22,9 @@ import { buildMigratorConfigs } from './utils'; import { migrateIndex } from './migrate_index'; export interface RunZeroDowntimeMigrationOpts { - /** The kibana system index prefix. e.g `.kibana` */ + /** The current Kibana version */ + kibanaVersion: string; + /** The Kibana system index prefix. e.g `.kibana` or `.kibana_task_manager` */ kibanaIndexPrefix: string; /** The SO type registry to use for the migration */ typeRegistry: ISavedObjectTypeRegistry; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts index ceb68b42b4a72..f5132d2ad2739 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts @@ -9,6 +9,9 @@ import type { InitState, State } from './types'; import type { MigratorContext } from '../context'; +/** + * Create the initial state to be used for the ZDT migrator. + */ export const createInitialState = (context: MigratorContext): State => { const initialState: InitState = { controlState: 'INIT', diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index bff3ea15da4c2..0f7d28507bb4a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -9,6 +9,11 @@ export type { BaseState, InitState, + CreateTargetIndexState, + UpdateIndexMappingsState, + UpdateIndexMappingsWaitForTaskState, + UpdateMappingModelVersionState, + UpdateAliasesState, DoneState, FatalState, State, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 90a888a8a947c..d43c6e49dd5e5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -6,8 +6,11 @@ * Side Public License, v 1. */ +import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; +import type { IndexMapping, IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrationLog } from '../../types'; import type { ControlState } from '../../state_action_machine'; +import type { AliasAction } from '../../actions'; export interface BaseState extends ControlState { readonly retryCount: number; @@ -15,10 +18,60 @@ export interface BaseState extends ControlState { readonly logs: MigrationLog[]; } +/** Initial state before any action is performed */ export interface InitState extends BaseState { readonly controlState: 'INIT'; } +export interface PostInitState extends BaseState { + /** + * The index we're currently migrating. + */ + readonly currentIndex: string; + /** + * The aliases that are already present for the current index. + */ + readonly aliases: string[]; + /** + * The alias actions to perform to update the aliases. + */ + readonly aliasActions: AliasAction[]; + /** + * The *previous* mappings (and _meta), as they were when we resolved the index + * information. This shouldn't be updated once populated. + */ + readonly previousMappings: IndexMapping; + /** + * The *current* _meta field of the index. + * All operations updating this field will update in the state accordingly. + */ + readonly currentIndexMeta: IndexMappingMeta; +} + +export interface CreateTargetIndexState extends BaseState { + readonly controlState: 'CREATE_TARGET_INDEX'; + readonly currentIndex: string; + readonly indexMappings: IndexMapping; +} + +export interface UpdateIndexMappingsState extends PostInitState { + readonly controlState: 'UPDATE_INDEX_MAPPINGS'; + readonly additiveMappingChanges: SavedObjectsMappingProperties; +} + +export interface UpdateIndexMappingsWaitForTaskState extends PostInitState { + readonly controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'; + readonly updateTargetMappingsTaskId: string; +} + +export interface UpdateMappingModelVersionState extends PostInitState { + readonly controlState: 'UPDATE_MAPPING_MODEL_VERSIONS'; +} + +export interface UpdateAliasesState extends PostInitState { + readonly controlState: 'UPDATE_ALIASES'; +} + /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; @@ -31,7 +84,15 @@ export interface FatalState extends BaseState { readonly reason: string; } -export type State = InitState | DoneState | FatalState; +export type State = + | InitState + | DoneState + | FatalState + | CreateTargetIndexState + | UpdateIndexMappingsState + | UpdateIndexMappingsWaitForTaskState + | UpdateMappingModelVersionState + | UpdateAliasesState; export type AllControlStates = State['controlState']; @@ -44,6 +105,11 @@ export interface ControlStateMap { INIT: InitState; FATAL: FatalState; DONE: DoneState; + CREATE_TARGET_INDEX: CreateTargetIndexState; + UPDATE_INDEX_MAPPINGS: UpdateIndexMappingsState; + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; + UPDATE_MAPPING_MODEL_VERSIONS: UpdateMappingModelVersionState; + UPDATE_ALIASES: UpdateAliasesState; } /** diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index ded69aa02e7de..faf9f9c89c9f5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -17,6 +17,7 @@ import type { MigratorContext } from '../context'; export type MockedMigratorContext = Omit & { elasticsearchClient: ElasticsearchClientMock; + typeRegistry: SavedObjectTypeRegistry; }; export const createContextMock = ( @@ -25,13 +26,19 @@ export const createContextMock = ( const typeRegistry = new SavedObjectTypeRegistry(); return { + kibanaVersion: '8.7.0', indexPrefix: '.kibana', types: ['foo', 'bar'], + typeModelVersions: { + foo: 1, + bar: 2, + }, elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(), maxRetryAttempts: 15, migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects, typeRegistry, serializer: serializerMock.create(), + deletedTypes: ['deleted-type'], ...parts, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts index 8b0c329317c03..5658828fc2e0c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts @@ -7,3 +7,5 @@ */ export { createContextMock, type MockedMigratorContext } from './context'; +export { createPostInitState } from './state'; +export { createType } from './saved_object_type'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts new file mode 100644 index 0000000000000..5f1f2ee8676f1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; + +export const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test-type', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts new file mode 100644 index 0000000000000..bd95881abbba4 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PostInitState } from '../state/types'; + +export const createPostInitState = (): PostInitState => ({ + controlState: 'INIT', + retryDelay: 0, + retryCount: 0, + logs: [], + currentIndex: '.kibana_1', + aliases: ['.kibana'], + aliasActions: [], + previousMappings: { properties: {} }, + currentIndexMeta: {}, +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts new file mode 100644 index 0000000000000..f4536cf1c75b0 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; +import { createType } from '../test_helpers'; + +const getTestTypes = () => { + const foo = createType({ + name: 'foo', + switchToModelVersionAt: '8.7.0', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { fooField: { type: 'text' } } }, + }); + const bar = createType({ + name: 'bar', + switchToModelVersionAt: '8.7.0', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { barField: { type: 'text' } } }, + }); + const dolly = createType({ + name: 'dolly', + switchToModelVersionAt: '8.7.0', + modelVersions: () => ({ + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + 3: { modelChange: { type: 'expansion' } }, + }), + mappings: { properties: { dollyField: { type: 'text' } } }, + }); + + return { foo, bar, dolly }; +}; + +describe('buildIndexMappings', () => { + it('builds the mappings used when creating a new index', () => { + const { foo, bar, dolly } = getTestTypes(); + const mappings = buildIndexMappings({ + types: [foo, bar, dolly], + }); + + expect(mappings).toEqual({ + dynamic: 'strict', + properties: expect.objectContaining({ + foo: foo.mappings, + bar: bar.mappings, + dolly: dolly.mappings, + }), + _meta: buildIndexMeta({ types: [foo, bar, dolly] }), + }); + }); +}); + +describe('buildIndexMeta', () => { + it('builds the _meta field value of the mapping', () => { + const { foo, bar, dolly } = getTestTypes(); + const meta = buildIndexMeta({ + types: [foo, bar, dolly], + }); + + expect(meta).toEqual({ + mappingVersions: { + foo: 2, + bar: 1, + dolly: 3, + }, + docVersions: { + foo: 2, + bar: 1, + dolly: 3, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts new file mode 100644 index 0000000000000..6221221ab993c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { cloneDeep } from 'lodash'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + type IndexMapping, + type IndexMappingMeta, + getModelVersionMapForTypes, +} from '@kbn/core-saved-objects-base-server-internal'; +import { getBaseMappings, buildTypesMappings } from '../../core'; + +interface BuildIndexMappingsOpts { + types: SavedObjectsType[]; +} + +/** + * Build the mappings to use when creating a new index. + * + * @param types The list of all registered SO types. + */ +export const buildIndexMappings = ({ types }: BuildIndexMappingsOpts): IndexMapping => { + const mappings: IndexMapping = cloneDeep(getBaseMappings()); + const typeMappings = buildTypesMappings(types); + + mappings.properties = { + ...mappings.properties, + ...typeMappings, + }; + + mappings._meta = buildIndexMeta({ types }); + + return mappings; +}; + +interface BuildIndexMetaOpts { + types: SavedObjectsType[]; +} + +/** + * Build the mapping _meta field to use when creating a new index. + * + * @param types The list of all registered SO types. + */ +export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta => { + const modelVersions = getModelVersionMapForTypes(types); + + return { + mappingVersions: modelVersions, + docVersions: modelVersions, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts new file mode 100644 index 0000000000000..a0e03552ed946 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getModelVersionsFromMappingsMock = jest.fn(); +export const compareModelVersionsMock = jest.fn(); +export const getModelVersionMapForTypesMock = jest.fn(); + +jest.doMock('@kbn/core-saved-objects-base-server-internal', () => { + const actual = jest.requireActual('@kbn/core-saved-objects-base-server-internal'); + return { + ...actual, + getModelVersionsFromMappings: getModelVersionsFromMappingsMock, + compareModelVersions: compareModelVersionsMock, + getModelVersionMapForTypes: getModelVersionMapForTypesMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts new file mode 100644 index 0000000000000..6ad12656229fc --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + compareModelVersionsMock, + getModelVersionsFromMappingsMock, + getModelVersionMapForTypesMock, +} from './check_version_compatibility.test.mocks'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import type { + IndexMapping, + ModelVersionMap, + CompareModelVersionResult, +} from '@kbn/core-saved-objects-base-server-internal'; +import { checkVersionCompatibility } from './check_version_compatibility'; +import { createType } from '../test_helpers'; + +describe('checkVersionCompatibility', () => { + const deletedTypes = ['some-deleted-type']; + + let types: SavedObjectsType[]; + let mappings: IndexMapping; + + beforeEach(() => { + compareModelVersionsMock.mockReset().mockReturnValue({}); + getModelVersionsFromMappingsMock.mockReset().mockReturnValue({}); + getModelVersionMapForTypesMock.mockReset().mockReturnValue({ status: 'equal' }); + + types = [createType({ name: 'foo' }), createType({ name: 'bar' })]; + + mappings = { + properties: { foo: { type: 'boolean' } }, + }; + }); + + it('calls getModelVersionMapForTypes with the correct parameters', () => { + checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(getModelVersionMapForTypesMock).toHaveBeenCalledTimes(1); + expect(getModelVersionMapForTypesMock).toHaveBeenCalledWith(types); + }); + + it('calls getModelVersionsFromMappings with the correct parameters', () => { + checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(getModelVersionsFromMappingsMock).toHaveBeenCalledTimes(1); + expect(getModelVersionsFromMappingsMock).toHaveBeenCalledWith({ + mappings, + source: 'mappingVersions', + }); + }); + + it('calls compareModelVersions with the correct parameters', () => { + const appVersions: ModelVersionMap = { foo: 2, bar: 2 }; + const indexVersions: ModelVersionMap = { foo: 1, bar: 1 }; + + getModelVersionMapForTypesMock.mockReturnValue(appVersions); + getModelVersionsFromMappingsMock.mockReturnValue(indexVersions); + + checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(compareModelVersionsMock).toHaveBeenCalledTimes(1); + expect(compareModelVersionsMock).toHaveBeenCalledWith({ + appVersions, + indexVersions, + deletedTypes, + }); + }); + + it('returns the result of the compareModelVersions call', () => { + const expected: CompareModelVersionResult = { + status: 'lesser', + details: { + greater: [], + lesser: [], + equal: [], + }, + }; + compareModelVersionsMock.mockReturnValue(expected); + + const result = checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts new file mode 100644 index 0000000000000..4499ce419d34a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + getModelVersionsFromMappings, + compareModelVersions, + getModelVersionMapForTypes, + type IndexMapping, + type CompareModelVersionResult, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface CheckVersionCompatibilityOpts { + mappings: IndexMapping; + types: SavedObjectsType[]; + source: 'docVersions' | 'mappingVersions'; + deletedTypes: string[]; +} + +export const checkVersionCompatibility = ({ + mappings, + types, + source, + deletedTypes, +}: CheckVersionCompatibilityOpts): CompareModelVersionResult => { + const appVersions = getModelVersionMapForTypes(types); + const indexVersions = getModelVersionsFromMappings({ mappings, source }); + if (!indexVersions) { + throw new Error(`Cannot check version: ${source} not present in the mapping meta`); + } + return compareModelVersions({ appVersions, indexVersions, deletedTypes }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts new file mode 100644 index 0000000000000..ce0cba20f427c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; +import { createType } from '../test_helpers'; + +describe('generateAdditiveMappingDiff', () => { + const deletedTypes = ['deletedType']; + + const getTypes = () => { + const foo = createType({ + name: 'foo', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { fooProp: { type: 'text' } } }, + }); + const bar = createType({ + name: 'bar', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { barProp: { type: 'text' } } }, + }); + + return { foo, bar }; + }; + + it('aggregates the mappings of the types with versions higher than in the index', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + }, + }; + + const addedMappings = generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }); + + expect(addedMappings).toEqual({ + foo: foo.mappings, + bar: bar.mappings, + }); + }); + + it('ignores mapping from types already up to date', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 2, + }, + }; + + const addedMappings = generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }); + + expect(addedMappings).toEqual({ + foo: foo.mappings, + }); + }); + + it('ignores deleted types', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + deletedType: 42, + }, + }; + + const addedMappings = generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }); + + expect(addedMappings).toEqual({ + foo: foo.mappings, + bar: bar.mappings, + }); + }); + + it('throws an error in case of version conflict', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 3, + }, + }; + + expect(() => + generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }) + ).toThrowErrorMatchingInlineSnapshot( + `"Cannot generate model version difference: conflict between versions"` + ); + }); + + it('throws an error if mappingVersions is not present on the index meta', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = {}; + + expect(() => + generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }) + ).toThrowErrorMatchingInlineSnapshot( + `"Cannot generate additive mapping diff: mappingVersions not present on index meta"` + ); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts new file mode 100644 index 0000000000000..f23b1e84a87ea --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + SavedObjectsType, + SavedObjectsMappingProperties, +} from '@kbn/core-saved-objects-server'; +import { + IndexMappingMeta, + getModelVersionsFromMappingMeta, + getModelVersionMapForTypes, + getModelVersionDelta, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface GenerateAdditiveMappingsDiffOpts { + types: SavedObjectsType[]; + meta: IndexMappingMeta; + deletedTypes: string[]; +} + +/** + * Generates the additive mapping diff we will need to update the index mapping with. + * + * @param types The types to generate the diff for + * @param meta The meta field of the index we're migrating + * @param deletedTypes The list of deleted types to ignore during diff/comparison + */ +export const generateAdditiveMappingDiff = ({ + types, + meta, + deletedTypes, +}: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { + const typeVersions = getModelVersionMapForTypes(types); + const mappingVersion = getModelVersionsFromMappingMeta({ meta, source: 'mappingVersions' }); + if (!mappingVersion) { + // should never occur given we checked previously in the flow but better safe than sorry. + throw new Error( + 'Cannot generate additive mapping diff: mappingVersions not present on index meta' + ); + } + + const delta = getModelVersionDelta({ + currentVersions: mappingVersion, + targetVersions: typeVersions, + deletedTypes, + }); + const typeMap = types.reduce>((map, type) => { + map[type.name] = type; + return map; + }, {}); + + // TODO: later we will want to generate the proper diff from `SavedObjectsModelExpansionChange.addedMappings` + // for this first implementation this is acceptable given we only allow compatible mapping changes anyway. + // we may want to implement the proper logic before this get used by real (non-test) type owners. + + const changedTypes = delta.diff.map((diff) => diff.name); + + const addedMappings: SavedObjectsMappingProperties = {}; + changedTypes.forEach((type) => { + addedMappings[type] = typeMap[type].mappings; + }); + + return addedMappings; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts new file mode 100644 index 0000000000000..42d7bcf27d11a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getAliasActions } from './get_alias_actions'; + +describe('getAliasActions', () => { + it('creates the global and version aliases', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: [], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([ + { add: { alias: '.kibana', index: '.kibana_1' } }, + { add: { alias: '.kibana_8.7.0', index: '.kibana_1' } }, + ]); + }); + + it('does not create the version alias when already present', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana_8.7.0'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([{ add: { alias: '.kibana', index: '.kibana_1' } }]); + }); + + it('does not create the global alias when already present', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([{ add: { alias: '.kibana_8.7.0', index: '.kibana_1' } }]); + }); + + it('creates nothing when both aliases are present', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana', '.kibana_8.7.0'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([]); + }); + + it('ignores other aliases', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana_8.6.0', '.kibana_old'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([ + { add: { alias: '.kibana', index: '.kibana_1' } }, + { add: { alias: '.kibana_8.7.0', index: '.kibana_1' } }, + ]); + }); + + it('accepts other prefixes', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana_task_manager', + currentIndex: '.kibana_task_manager_2', + existingAliases: [], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([ + { add: { alias: '.kibana_task_manager', index: '.kibana_task_manager_2' } }, + { add: { alias: '.kibana_task_manager_8.7.0', index: '.kibana_task_manager_2' } }, + ]); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts new file mode 100644 index 0000000000000..185b454987b45 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { AliasAction } from '../../actions'; + +interface GetAliasActionOpts { + indexPrefix: string; + currentIndex: string; + existingAliases: string[]; + kibanaVersion: string; +} + +/** + * Build the list of alias actions to perform, depending on the current state of the cluster. + */ +export const getAliasActions = ({ + indexPrefix, + currentIndex, + existingAliases, + kibanaVersion, +}: GetAliasActionOpts): AliasAction[] => { + const actions: AliasAction[] = []; + + const globalAlias = indexPrefix; + const versionAlias = `${indexPrefix}_${kibanaVersion}`; + const allAliases = [globalAlias, versionAlias]; + allAliases.forEach((alias) => { + if (!existingAliases.includes(alias)) { + actions.push({ + add: { index: currentIndex, alias }, + }); + } + }); + + return actions; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts new file mode 100644 index 0000000000000..d6db500dca633 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getCurrentIndex } from './get_current_index'; +import type { FetchIndexResponse } from '../../actions'; + +describe('getCurrentIndex', () => { + const createIndexResponse = (...indexNames: string[]): FetchIndexResponse => { + return indexNames.reduce((resp, indexName) => { + resp[indexName] = { aliases: {}, mappings: { properties: {} }, settings: {} }; + return resp; + }, {}); + }; + + it('returns the highest numbered index matching the index prefix', () => { + const resp = createIndexResponse('.kibana_1', '.kibana_2'); + expect(getCurrentIndex(resp, '.kibana')).toEqual('.kibana_2'); + }); + + it('ignores other indices', () => { + const resp = createIndexResponse('.kibana_1', '.kibana_2', '.foo_3'); + expect(getCurrentIndex(resp, '.kibana')).toEqual('.kibana_2'); + }); + + it('ignores other indices including the prefix', () => { + const resp = createIndexResponse('.kibana_1', '.kibana_2', '.kibana_task_manager_3'); + expect(getCurrentIndex(resp, '.kibana')).toEqual('.kibana_2'); + }); + + it('ignores other indices including a subpart of the prefix', () => { + const resp = createIndexResponse( + '.kibana_3', + '.kibana_task_manager_1', + '.kibana_task_manager_2' + ); + expect(getCurrentIndex(resp, '.kibana_task_manager')).toEqual('.kibana_task_manager_2'); + }); + + it('returns undefined if no indices match', () => { + const resp = createIndexResponse('.kibana_task_manager_1', '.kibana_task_manager_2'); + expect(getCurrentIndex(resp, '.kibana')).toBeUndefined(); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts new file mode 100644 index 0000000000000..59eea5c550804 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { escapeRegExp } from 'lodash'; +import type { FetchIndexResponse } from '../../actions'; + +export const getCurrentIndex = ( + indices: FetchIndexResponse, + indexPrefix: string +): string | undefined => { + const matcher = new RegExp(`^${escapeRegExp(indexPrefix)}[_](?\\d+)$`); + + let lastCount = -1; + Object.keys(indices).forEach((indexName) => { + const match = matcher.exec(indexName); + if (match && match.groups?.counter) { + const suffix = parseInt(match.groups.counter, 10); + lastCount = Math.max(lastCount, suffix); + } + }); + + return lastCount === -1 ? undefined : `${indexPrefix}_${lastCount}`; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index f96ea531e460d..ebc22e623f600 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -7,3 +7,8 @@ */ export { buildMigratorConfigs, type MigratorConfig } from './get_migrator_configs'; +export { getCurrentIndex } from './get_current_index'; +export { checkVersionCompatibility } from './check_version_compatibility'; +export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; +export { getAliasActions } from './get_alias_actions'; +export { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; diff --git a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts index aa78d8dbc5d90..0a84b9bc4b7e8 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts @@ -10,6 +10,7 @@ import { Env } from '@kbn/config'; import { getDocLinksMeta, getDocLinks } from '@kbn/doc-links'; import { REPO_ROOT } from '@kbn/repo-info'; import { getEnvOptions } from '@kbn/config-mocks'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; export const getDocVersion = () => { const env = Env.createDefault(REPO_ROOT, getEnvOptions()); @@ -24,3 +25,11 @@ export const getMigrationDocLink = () => { export const delay = (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); + +export const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test-type', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts new file mode 100644 index 0000000000000..9bfac3ac5fd49 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { createType } from '../test_utils'; + +export const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +export const getFooType = () => { + return createType({ + name: 'foo', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + '2': dummyModelVersion, + }, + }); +}; + +export const getBarType = () => { + return createType({ + name: 'bar', + mappings: { + properties: { + aKeyword: { type: 'keyword' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts new file mode 100644 index 0000000000000..e3f45bb561cf6 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; +import { LogRecord } from '@kbn/logging'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { getFooType, getBarType } from './base_types.fixtures'; + +export const logFilePath = Path.join(__dirname, 'create_index.test.log'); + +describe('ZDT upgrades - running on a fresh cluster', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + it('create the index with the correct mappings and meta', async () => { + const fooType = getFooType(); + const barType = getBarType(); + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + kibanaIndex: '.kibana', + kibanaVersion: '8.7.0', + logFilePath, + types: [fooType, barType], + settings: { + migrations: { + algorithm: 'zdt', + }, + }, + }); + + const result = await runMigrations(); + + expect(result).toEqual([ + { + destIndex: '.kibana', + elapsedMs: expect.any(Number), + status: 'patched', + }, + ]); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + + expect(mappings.properties).toEqual( + expect.objectContaining({ + foo: fooType.mappings, + bar: barType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + docVersions: { + foo: 2, + bar: 1, + }, + mappingVersions: { + foo: 2, + bar: 1, + }, + }); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; + + const expectLogsContains = (messagePrefix: string) => { + expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); + }; + + expectLogsContains('INIT -> CREATE_TARGET_INDEX'); + expectLogsContains('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); + expectLogsContains('UPDATE_ALIASES -> DONE'); + expectLogsContains('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js new file mode 100644 index 0000000000000..4772e43faa148 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + // TODO replace the line below with + // preset: '@kbn/test/jest_integration_node + // to do so, we must fix all integration tests first + // see https://github.com/elastic/kibana/pull/130255/ + preset: '@kbn/test/jest_integration', + rootDir: '../../../../../../..', + roots: ['/src/core/server/integration_tests/saved_objects/migrations/zero_downtime'], + // must override to match all test given there is no `integration_tests` subfolder + testMatch: ['**/*.test.{js,mjs,ts,tsx}'], +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts new file mode 100644 index 0000000000000..eadd24bf447ac --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; +import { LogRecord } from '@kbn/logging'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { + getKibanaMigratorTestKit, + type KibanaMigratorTestKitParams, +} from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; + +export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log'); + +describe('ZDT upgrades - mapping model version conflict', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + const baseMigratorParams: KibanaMigratorTestKitParams = { + kibanaIndex: '.kibana', + kibanaVersion: '8.7.0', + settings: { + migrations: { + algorithm: 'zdt', + }, + }, + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + + // increasing bar model version on the baseline + barType.modelVersions = { + ...barType.modelVersions, + '2': dummyModelVersion, + }; + barType.mappings.properties = { + ...barType.mappings.properties, + anotherAddedField: { type: 'boolean' }, + }; + const { runMigrations } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('fails the migration with an error', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + // increasing foo model version + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + fooType.mappings.properties = { + ...fooType.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + // we have the following versions: + // baseline : bar:2, foo: 2 + // upgrade: bar:3, foo:1 + // which should cause a migration failure. + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + logFilePath, + types: [fooType, barType], + }); + + await expect(runMigrations()).rejects.toThrowErrorMatchingInlineSnapshot( + `"Unable to complete saved object migrations for the [.kibana] index: Model version conflict: inconsistent higher/lower versions"` + ); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + + expect(mappingMeta.mappingVersions).toEqual({ + foo: 2, + bar: 2, + }); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; + + const expectLogsContains = (messagePrefix: string) => { + expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); + }; + + // + expectLogsContains('Mappings model version check result: conflict'); + expectLogsContains('INIT -> FATAL'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts new file mode 100644 index 0000000000000..5d27c32c1ee6a --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; +import { LogRecord } from '@kbn/logging'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { + getKibanaMigratorTestKit, + type KibanaMigratorTestKitParams, +} from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; + +export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); + +describe('ZDT upgrades - basic mapping update', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + const baseMigratorParams: KibanaMigratorTestKitParams = { + kibanaIndex: '.kibana', + kibanaVersion: '8.7.0', + settings: { + migrations: { + algorithm: 'zdt', + }, + }, + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + const { runMigrations } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('updates the mappings and the meta', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + // increasing the model version of the types + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + fooType.mappings.properties = { + ...fooType.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + barType.modelVersions = { + ...barType.modelVersions, + '2': dummyModelVersion, + }; + barType.mappings.properties = { + ...barType.mappings.properties, + anotherAddedField: { type: 'boolean' }, + }; + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + logFilePath, + types: [fooType, barType], + }); + + const result = await runMigrations(); + + expect(result).toEqual([ + { + destIndex: '.kibana', + elapsedMs: expect.any(Number), + status: 'patched', + }, + ]); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + + expect(mappings.properties).toEqual( + expect.objectContaining({ + foo: fooType.mappings, + bar: barType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + // doc migration not implemented yet - docVersions are not bumped. + docVersions: { + foo: 2, + bar: 1, + }, + mappingVersions: { + foo: 3, + bar: 2, + }, + }); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; + + const expectLogsContains = (messagePrefix: string) => { + expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); + }; + + expectLogsContains('INIT -> UPDATE_INDEX_MAPPINGS'); + expectLogsContains('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'); + expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS'); + expectLogsContains('UPDATE_MAPPING_MODEL_VERSIONS -> DONE'); + expectLogsContains('Migration completed'); + }); +});