From 1a5cc56bd953ec24c5ebcf520145510c155d1c85 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 27 Feb 2023 14:36:27 +0100 Subject: [PATCH 01/29] add model version compare base utils --- .../index.ts | 3 + .../src/model_version/index.ts | 1 + .../src/model_version/version_map.test.ts | 119 ++++++++++++++++++ .../src/model_version/version_map.ts | 33 +++++ 4 files changed, 156 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 722694a32f6ad..3d936d41f924a 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -45,4 +45,7 @@ export { isVirtualModelVersion, virtualVersionToModelVersion, modelVersionToVirtualVersion, + buildVersionMap, + getLatestModelVersion, + type ModelVersionMap, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 5301c0a4d219c..65b4d86dbcf8e 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -13,3 +13,4 @@ export { modelVersionToVirtualVersion, virtualVersionToModelVersion, } from './conversion'; +export { buildVersionMap, getLatestModelVersion, type ModelVersionMap } from './version_map'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts new file mode 100644 index 0000000000000..fab603285e7b1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType, SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { buildVersionMap, getLatestModelVersion } from './version_map'; + +describe('ModelVersion map utilities', () => { + const buildType = (parts: Partial = {}): SavedObjectsType => ({ + name: 'test-type', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, + }); + + const dummyModelVersion = (): SavedObjectsModelVersion => ({ + modelChange: { + type: 'expansion', + }, + }); + + describe('getLatestModelVersion', () => { + it('returns 0 when no model versions are registered', () => { + expect(getLatestModelVersion(buildType({ modelVersions: {} }))).toEqual(0); + expect(getLatestModelVersion(buildType({ modelVersions: undefined }))).toEqual(0); + }); + + it('throws if an invalid version is provided', () => { + expect(() => + getLatestModelVersion( + buildType({ + modelVersions: { + foo: dummyModelVersion(), + }, + }) + ) + ).toThrow(); + }); + + it('returns the latest registered version', () => { + expect( + getLatestModelVersion( + buildType({ + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersion(), + '3': dummyModelVersion(), + }, + }) + ) + ).toEqual(3); + }); + + it('accepts provider functions', () => { + expect( + getLatestModelVersion( + buildType({ + modelVersions: () => ({ + '1': dummyModelVersion(), + '2': dummyModelVersion(), + '3': dummyModelVersion(), + }), + }) + ) + ).toEqual(3); + }); + + it('supports unordered maps', () => { + expect( + getLatestModelVersion( + buildType({ + modelVersions: { + '3': dummyModelVersion(), + '1': dummyModelVersion(), + '2': dummyModelVersion(), + }, + }) + ) + ).toEqual(3); + }); + }); + + describe('buildVersionMap', () => { + it('returns a map with the latest version of the provided types', () => { + expect( + buildVersionMap([ + buildType({ + name: 'foo', + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersion(), + }, + }), + buildType({ + name: 'bar', + modelVersions: {}, + }), + buildType({ + name: 'dolly', + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersion(), + '3': dummyModelVersion(), + }, + }), + ]) + ).toEqual({ + foo: 2, + bar: 0, + dolly: 3, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts new file mode 100644 index 0000000000000..44d9caa0617ce --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { assertValidModelVersion } from './conversion'; + +export type ModelVersionMap = Record; + +/** + * Returns the latest registered model version number for the given type. + */ +export const getLatestModelVersion = (type: SavedObjectsType): number => { + const versionMap = + typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {}; + return Object.keys(versionMap).reduce((memo, current) => { + return Math.max(memo, assertValidModelVersion(current)); + }, 0); +}; + +/** + * Build a version map for the given types. + */ +export const buildVersionMap = (types: SavedObjectsType[]): ModelVersionMap => { + return types.reduce((versionMap, type) => { + versionMap[type.name] = getLatestModelVersion(type); + return versionMap; + }, {}); +}; From e5c341a0adf0981f17c58515433a14e7ec209e12 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 27 Feb 2023 15:51:08 +0100 Subject: [PATCH 02/29] add compare function --- .../index.ts | 5 + .../src/model_version/index.ts | 7 + .../src/model_version/version_compare.test.ts | 142 ++++++++++++++++++ .../src/model_version/version_compare.ts | 77 ++++++++++ 4 files changed, 231 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 3d936d41f924a..6c7eed2c85f97 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -48,4 +48,9 @@ export { buildVersionMap, getLatestModelVersion, type ModelVersionMap, + compareModelVersions, + type CompareModelVersionMapParams, + type CompareModelVersionStatus, + type CompareModelVersionDetails, + type CompareModelVersionResult, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 65b4d86dbcf8e..77db4ee1474bf 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -14,3 +14,10 @@ export { virtualVersionToModelVersion, } from './conversion'; export { buildVersionMap, getLatestModelVersion, type ModelVersionMap } from './version_map'; +export { + compareModelVersions, + type CompareModelVersionMapParams, + type CompareModelVersionStatus, + type CompareModelVersionDetails, + type CompareModelVersionResult, +} from './version_compare'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts new file mode 100644 index 0000000000000..eba6fe1837cce --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { compareModelVersions } from './version_compare'; + +describe('compareModelVersions', () => { + it('returns the correct value for greater app version', () => { + const result = compareModelVersions({ + appVersions: { + foo: 3, + bar: 2, + }, + indexVersions: { + foo: 2, + bar: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('greater'); + }); + + it('returns the correct value for lesser app version', () => { + const result = compareModelVersions({ + appVersions: { + foo: 1, + bar: 2, + }, + indexVersions: { + foo: 2, + bar: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('lesser'); + }); + + it('returns the correct value for equal versions', () => { + const result = compareModelVersions({ + appVersions: { + foo: 2, + bar: 2, + }, + indexVersions: { + foo: 2, + bar: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('equal'); + }); + + it('handles new types not being present in the index', () => { + const result = compareModelVersions({ + appVersions: { + foo: 2, + new: 1, + }, + indexVersions: { + foo: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('greater'); + }); + + it('handles types not being present in the app', () => { + const result = compareModelVersions({ + appVersions: { + foo: 3, + }, + indexVersions: { + foo: 2, + old: 1, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('conflict'); + }); + + it('returns the correct value for conflicts', () => { + const result = compareModelVersions({ + appVersions: { + a: 3, + b: 3, + c: 3, + }, + indexVersions: { + a: 2, + b: 3, + c: 4, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('conflict'); + }); + + it('properly lists the details', () => { + const result = compareModelVersions({ + appVersions: { + a: 3, + b: 3, + c: 3, + }, + indexVersions: { + a: 2, + b: 3, + c: 4, + }, + deletedTypes: [], + }); + + expect(result.details.lesser).toEqual(['c']); + expect(result.details.equal).toEqual(['b']); + expect(result.details.greater).toEqual(['a']); + }); + + it('ignores deleted types when comparing', () => { + const result = compareModelVersions({ + appVersions: { + a: 3, + }, + indexVersions: { + a: 2, + b: 3, + }, + deletedTypes: ['b'], + }); + + expect(result.status).toEqual('greater'); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts new file mode 100644 index 0000000000000..9b8d14b7fd862 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ModelVersionMap } from './version_map'; + +export interface CompareModelVersionMapParams { + /** The latest model version of the types registered in the application */ + appVersions: ModelVersionMap; + /** The model version stored in the index */ + indexVersions: ModelVersionMap; + /** The list of deleted types to exclude during the compare process */ + deletedTypes: string[]; +} + +/** + * The overall status of the model version comparison: + * - `greater`: app version is greater than the index version + * - `lesser`: app version is lesser than the index version + * - `equal`: app version is equal to the index version + * - `conflict`: app and index versions are incompatible (versions for some types are higher, and for other types lower) + */ +export type CompareModelVersionStatus = 'greater' | 'lesser' | 'equal' | 'conflict'; + +export interface CompareModelVersionDetails { + greater: string[]; + lesser: string[]; + equal: string[]; +} + +export interface CompareModelVersionResult { + status: CompareModelVersionStatus; + details: CompareModelVersionDetails; +} + +export const compareModelVersions = ({ + appVersions, + indexVersions, + deletedTypes, +}: CompareModelVersionMapParams): CompareModelVersionResult => { + const allTypes = [ + ...new Set([...Object.keys(appVersions), ...Object.keys(indexVersions)]), + ].filter((type) => !deletedTypes.includes(type)); + + const details: CompareModelVersionDetails = { + greater: [], + lesser: [], + equal: [], + }; + + allTypes.forEach((type) => { + const appVersion = appVersions[type] ?? 0; + const indexVersion = indexVersions[type] ?? 0; + + if (appVersion > indexVersion) { + details.greater.push(type); + } else if (appVersion < indexVersion) { + details.lesser.push(type); + } else { + details.equal.push(type); + } + }); + + const hasGreater = details.greater.length > 0; + const hasLesser = details.lesser.length > 0; + const status: CompareModelVersionStatus = + hasGreater && hasLesser ? 'conflict' : hasGreater ? 'greater' : hasLesser ? 'lesser' : 'equal'; + + return { + status, + details, + }; +}; From 087bc30959b04a80ffb934a885e7213fd2455463 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 28 Feb 2023 09:54:23 +0100 Subject: [PATCH 03/29] work in progress --- .../index.ts | 3 +- .../src/mappings/types.ts | 18 ++++- .../src/model_version/index.ts | 7 +- .../model_version_from_mappings.test.ts | 73 +++++++++++++++++ .../model_version_from_mappings.ts | 33 ++++++++ .../src/model_version/version_map.test.ts | 6 +- .../src/model_version/version_map.ts | 2 +- .../src/actions/index.ts | 6 +- .../src/zdt/actions/index.ts | 5 ++ .../src/zdt/context/create_context.ts | 2 + .../src/zdt/context/types.ts | 2 + .../src/zdt/model/model.ts | 8 ++ .../src/zdt/model/stages/init.test.ts | 36 +++++++-- .../src/zdt/model/stages/init.ts | 81 +++++++++++++++++-- .../src/zdt/next.ts | 43 ++++++++-- .../src/zdt/state/index.ts | 5 ++ .../src/zdt/state/types.ts | 43 +++++++++- .../src/zdt/test_helpers/context.ts | 1 + .../zdt/utils/check_version_compatibility.ts | 37 +++++++++ .../src/zdt/utils/get_current_index.test.ts | 48 +++++++++++ .../src/zdt/utils/get_current_index.ts | 28 +++++++ .../src/zdt/utils/index.ts | 2 + 22 files changed, 458 insertions(+), 31 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 6c7eed2c85f97..4b6afe1e76a63 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -45,7 +45,7 @@ export { isVirtualModelVersion, virtualVersionToModelVersion, modelVersionToVirtualVersion, - buildVersionMap, + getModelVersionMapForTypes, getLatestModelVersion, type ModelVersionMap, compareModelVersions, @@ -53,4 +53,5 @@ export { type CompareModelVersionStatus, type CompareModelVersionDetails, type CompareModelVersionResult, + getModelVersionsFromMappings, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index c93abc2064fb6..1eca0245f18f2 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -57,8 +57,20 @@ export interface IndexMapping { /** @internal */ export interface IndexMappingMeta { - // A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa') - // with each key being a root-level mapping property, and each value being - // the md5 hash of that mapping's value when the index was created. + /** + * A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa') + * with each key being a root-level mapping property, and each value being + * the md5 hash of that mapping's value when the index was created. + * + * @remark: Only defined for the v2 migration algorithm. + */ migrationMappingPropertyHashes?: { [k: string]: string }; + /** + * TODO: doc + */ + mappingVersions?: { [k: string]: number }; + /** + * TODO: doc + */ + docVersions?: { [k: string]: number }; } diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 77db4ee1474bf..5577378a671b7 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -13,7 +13,11 @@ export { modelVersionToVirtualVersion, virtualVersionToModelVersion, } from './conversion'; -export { buildVersionMap, getLatestModelVersion, type ModelVersionMap } from './version_map'; +export { + getModelVersionMapForTypes, + getLatestModelVersion, + type ModelVersionMap, +} from './version_map'; export { compareModelVersions, type CompareModelVersionMapParams, @@ -21,3 +25,4 @@ export { type CompareModelVersionDetails, type CompareModelVersionResult, } from './version_compare'; +export { getModelVersionsFromMappings } from './model_version_from_mappings'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts new file mode 100644 index 0000000000000..8fea10f11f6b1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMapping, IndexMappingMeta } from '../mappings'; +import { getModelVersionsFromMappings } from './model_version_from_mappings'; + +describe('getModelVersionsFromMappings', () => { + const createIndexMapping = (parts: Partial = {}): IndexMapping => ({ + properties: {}, + _meta: { + ...parts, + }, + }); + + it('retrieves the version map from docVersions', () => { + const mappings = createIndexMapping({ + docVersions: { + foo: 3, + bar: 5, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + + expect(versionMap).toEqual({ + foo: 3, + bar: 5, + }); + }); + + it('retrieves the version map from mappingVersions', () => { + const mappings = createIndexMapping({ + mappingVersions: { + foo: 2, + bar: 7, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + + expect(versionMap).toEqual({ + foo: 2, + bar: 7, + }); + }); + + it('returns undefined for docVersions if meta field is not present', () => { + const mappings = createIndexMapping({ + mappingVersions: { + foo: 3, + bar: 5, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + + expect(versionMap).toBeUndefined(); + }); + + it('returns undefined for mappingVersions if meta field is not present', () => { + const mappings = createIndexMapping({ + docVersions: { + foo: 3, + bar: 5, + }, + }); + const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + + expect(versionMap).toBeUndefined(); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts new file mode 100644 index 0000000000000..f6fb5a4d52095 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMapping } from '../mappings'; +import type { ModelVersionMap } from './version_map'; +import { assertValidModelVersion } from './conversion'; + +/** + * Build the version map from the specified source of the provided mappings. + */ +export const getModelVersionsFromMappings = ({ + mappings, + source, +}: { + mappings: IndexMapping; + source: 'mappingVersions' | 'docVersions'; +}): ModelVersionMap | undefined => { + const indexVersions = + source === 'mappingVersions' ? mappings._meta?.mappingVersions : mappings._meta?.docVersions; + if (!indexVersions) { + return undefined; + } + + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { + map[type] = assertValidModelVersion(rawVersion); + return map; + }, {}); +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts index fab603285e7b1..aafb83ab96009 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts @@ -7,7 +7,7 @@ */ import type { SavedObjectsType, SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; -import { buildVersionMap, getLatestModelVersion } from './version_map'; +import { getModelVersionMapForTypes, getLatestModelVersion } from './version_map'; describe('ModelVersion map utilities', () => { const buildType = (parts: Partial = {}): SavedObjectsType => ({ @@ -85,10 +85,10 @@ describe('ModelVersion map utilities', () => { }); }); - describe('buildVersionMap', () => { + describe('getModelVersionMapForTypes', () => { it('returns a map with the latest version of the provided types', () => { expect( - buildVersionMap([ + getModelVersionMapForTypes([ buildType({ name: 'foo', modelVersions: { diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts index 44d9caa0617ce..dd05e64dbcbef 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -25,7 +25,7 @@ export const getLatestModelVersion = (type: SavedObjectsType): number => { /** * Build a version map for the given types. */ -export const buildVersionMap = (types: SavedObjectsType[]): ModelVersionMap => { +export const getModelVersionMapForTypes = (types: SavedObjectsType[]): ModelVersionMap => { return types.reduce((versionMap, type) => { versionMap[type.name] = getLatestModelVersion(type); return versionMap; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index 2593ac7867d1e..376e232aaaf29 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -36,7 +36,11 @@ export { removeWriteBlock } from './remove_write_block'; export type { CloneIndexResponse, CloneIndexParams } from './clone_index'; export { cloneIndex } from './clone_index'; -export type { WaitForIndexStatusParams, IndexNotYellowTimeout } from './wait_for_index_status'; +export type { + WaitForIndexStatusParams, + IndexNotYellowTimeout, + IndexNotGreenTimeout, +} from './wait_for_index_status'; import { type IndexNotGreenTimeout, type IndexNotYellowTimeout, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index 92334d396adc0..9a0cd0c785ced 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -10,11 +10,14 @@ import type { IncompatibleClusterRoutingAllocation, RetryableEsClientError, WaitForTaskCompletionTimeout, + IndexNotYellowTimeout, + IndexNotGreenTimeout, IndexNotFound, } from '../../actions'; export { initAction as init, + waitForIndexStatus, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -27,6 +30,8 @@ export interface ActionErrorTypeMap { incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation; retryable_es_client_error: RetryableEsClientError; index_not_found_exception: IndexNotFound; + index_not_green_timeout: IndexNotGreenTimeout; + index_not_yellow_timeout: IndexNotYellowTimeout; } /** Type guard for narrowing the type of a left */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index cc4c7b63d3993..6d7049bbea2ad 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -8,6 +8,7 @@ import type { MigratorContext } from './types'; import type { MigrateIndexOptions } from '../migrate_index'; +import { REMOVED_TYPES } from '../../core'; export type CreateContextOps = Omit; @@ -31,5 +32,6 @@ export const createContext = ({ serializer, maxRetryAttempts: migrationConfig.retryAttempts, migrationDocLinks: docLinks.links.kibanaUpgradeSavedObjects, + deletedTypes: REMOVED_TYPES, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 2603a5b69a681..ced9d79ee92cb 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -31,4 +31,6 @@ export interface MigratorContext { readonly serializer: ISavedObjectsSerializer; /** The SO type registry to use for the migration */ readonly typeRegistry: ISavedObjectTypeRegistry; + /** List of types that are no longer registered */ + readonly deletedTypes: string[]; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 2ea48bd1a58af..22d2a952f81c4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -30,6 +30,14 @@ export const model = ( } switch (current.controlState) { + // TODO: unimplemented + case 'WAIT_FOR_YELLOW_INDEX': + case 'CREATE_TARGET_INDEX': + case 'UPDATE_INDEX_MAPPINGS': + case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': + case 'UPDATE_OR_CREATE_ALIASES': + return current; + // END TODO case 'INIT': return Stages.init(current, response as StateActionResponse<'INIT'>, context); case 'DONE': diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index 8105449c7fce8..0301302e18ed3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -24,16 +24,15 @@ describe('Action: init', () => { }); beforeEach(() => { - context = createContextMock(); + context = createContextMock({ indexPrefix: '.kibana' }); }); - test('INIT -> DONE because its not implemented yet', () => { + test('INIT -> WAIT_FOR_YELLOW_INDEX when index is found', () => { const state = createState(); const res: StateActionResponse<'INIT'> = Either.right({ - '.kibana_8.7.0_001': { + '.kibana_1': { aliases: { '.kibana': {}, - '.kibana_8.7.0': {}, }, mappings: { properties: {} }, settings: {}, @@ -42,7 +41,34 @@ describe('Action: init', () => { const newState = init(state, res, context); - expect(newState.controlState).toEqual('DONE'); + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'WAIT_FOR_YELLOW_INDEX', + currentIndex: '.kibana_1', + previousMappings: { properties: {} }, + }) + ); + }); + + test('INIT -> CREATE_TARGET_INDEX because its not implemented yet', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right({ + '.foo_1': { + aliases: { + '.some_alias': {}, + }, + mappings: { properties: {} }, + settings: {}, + }, + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'CREATE_TARGET_INDEX', + }) + ); }); test('INIT -> INIT when cluster routing allocation is incompatible', () => { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index 78dccf237afca..582fd3464b739 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -9,11 +9,15 @@ import * as Either from 'fp-ts/lib/Either'; import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; +import type { MigrationLog } from '../../../types'; import { isTypeof } from '../../actions'; -import type { State } from '../../state'; +import { getCurrentIndex, checkVersionCompatibility } from '../../utils'; import type { ModelStage } from '../types'; -export const init: ModelStage<'INIT', 'DONE' | 'FATAL'> = (state, res, context): State => { +export const init: ModelStage< + 'INIT', + 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_OR_CREATE_ALIASES' | 'FATAL' +> = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'incompatible_cluster_routing_allocation')) { @@ -24,9 +28,72 @@ export const init: ModelStage<'INIT', 'DONE' | 'FATAL'> = (state, res, context): } } - // nothing implemented yet, just going to 'DONE' - return { - ...state, - controlState: 'DONE', - }; + const logs: MigrationLog[] = [...state.logs]; + + const indices = res.right; + const currentIndex = getCurrentIndex(indices, context.indexPrefix); + + // No indices were found, likely because it is the first time Kibana boots. + // In that case, we just create the index. + if (!currentIndex) { + return { + ...state, + logs, + controlState: 'CREATE_TARGET_INDEX', + }; + } + + // Index was found. This is the standard scenario, we check the model versions + // compatibility before going further. + const currentMappings = indices[currentIndex].mappings; + const types = context.types.map((type) => context.typeRegistry.getType(type)!); + const versionCheck = checkVersionCompatibility({ + mappings: currentMappings, + types, + source: 'mappingVersions', + deletedTypes: context.deletedTypes, + }); + + switch (versionCheck.status) { + // app version is greater than the index mapping version. + // scenario of an upgrade: we need to update the mappings + case 'greater': + return { + ...state, + logs, + currentIndex, + previousMappings: currentMappings, + controlState: 'UPDATE_INDEX_MAPPINGS', + }; + // app version and index mapping version are the same. + // either application upgrade without model change, or a simple reboot on the same version. + // In that case we jump directly to alias update + case 'equal': + return { + ...state, + logs, + currentIndex, + previousMappings: currentMappings, + controlState: 'UPDATE_OR_CREATE_ALIASES', + }; + // app version is lower than the index mapping version. + // likely a rollback scenario - unsupported for the initial implementation + case 'lesser': + return { + ...state, + logs, + reason: 'Downgrading model version is currently unsupported', + controlState: 'FATAL', + }; + // conflicts: version for some types are greater, some are lower + // shouldn't occur in any normal scenario - cannot recover + case 'conflict': + default: + return { + ...state, + logs, + reason: 'Model version conflict: inconsistent higher/lower versions', + controlState: 'FATAL', + }; + } }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index 329ba194b5957..4330b5be038dd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -6,7 +6,16 @@ * Side Public License, v 1. */ -import type { AllActionStates, InitState, State } from './state'; +import type { + AllActionStates, + State, + InitState, + WaitForYellowIndexState, + CreateTargetIndexState, + UpdateIndexMappingsState, + UpdateIndexMappingsWaitForTaskState, + UpdateOrCreateAliasesState, +} from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; @@ -22,10 +31,24 @@ export type ResponseType = Awaited< ReturnType> >; +// TODO: remove when done +const NOT_IMPLEMENTED = () => Promise.resolve({} as any); + export const nextActionMap = (context: MigratorContext) => { return { INIT: (state: InitState) => Actions.init({ client: context.elasticsearchClient, indices: [context.indexPrefix] }), + WAIT_FOR_YELLOW_INDEX: (state: WaitForYellowIndexState) => + Actions.waitForIndexStatus({ + client: context.elasticsearchClient, + index: state.currentIndex, + status: 'yellow', + }), + CREATE_TARGET_INDEX: (state: CreateTargetIndexState) => NOT_IMPLEMENTED, + UPDATE_INDEX_MAPPINGS: (state: UpdateIndexMappingsState) => NOT_IMPLEMENTED, + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: (state: UpdateIndexMappingsWaitForTaskState) => + NOT_IMPLEMENTED, + UPDATE_OR_CREATE_ALIASES: (state: UpdateOrCreateAliasesState) => () => NOT_IMPLEMENTED, }; }; @@ -33,13 +56,7 @@ export const next = (context: MigratorContext) => { const map = nextActionMap(context); return (state: State) => { - const delay = any>(fn: F): (() => ReturnType) => { - return () => { - return state.retryDelay > 0 - ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) - : fn(); - }; - }; + const delay = createDelayFn(state); if (state.controlState === 'DONE' || state.controlState === 'FATAL') { // Return null if we're in one of the terminating states @@ -56,3 +73,13 @@ export const next = (context: MigratorContext) => { } }; }; + +const createDelayFn = + (state: State) => + any>(fn: F): (() => ReturnType) => { + return () => { + return state.retryDelay > 0 + ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) + : fn(); + }; + }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index bff3ea15da4c2..193e7e4368181 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -9,6 +9,11 @@ export type { BaseState, InitState, + CreateTargetIndexState, + WaitForYellowIndexState, + UpdateIndexMappingsState, + UpdateIndexMappingsWaitForTaskState, + UpdateOrCreateAliasesState, DoneState, FatalState, State, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 90a888a8a947c..7314389dc951e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrationLog } from '../../types'; import type { ControlState } from '../../state_action_machine'; @@ -15,10 +16,37 @@ export interface BaseState extends ControlState { readonly logs: MigrationLog[]; } +/** Initial state before any action is performed */ export interface InitState extends BaseState { readonly controlState: 'INIT'; } +export interface PostInitState extends BaseState { + currentIndex: string; + previousMappings: IndexMapping; +} + +export interface CreateTargetIndexState extends BaseState { + readonly controlState: 'CREATE_TARGET_INDEX'; +} + +export interface WaitForYellowIndexState extends PostInitState { + readonly controlState: 'WAIT_FOR_YELLOW_INDEX'; +} + +export interface UpdateIndexMappingsState extends PostInitState { + readonly controlState: 'UPDATE_INDEX_MAPPINGS'; +} + +export interface UpdateIndexMappingsWaitForTaskState extends PostInitState { + readonly controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'; + readonly updateTargetMappingsTaskId: string; +} + +export interface UpdateOrCreateAliasesState extends PostInitState { + readonly controlState: 'UPDATE_OR_CREATE_ALIASES'; +} + /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; @@ -31,7 +59,15 @@ export interface FatalState extends BaseState { readonly reason: string; } -export type State = InitState | DoneState | FatalState; +export type State = + | InitState + | DoneState + | FatalState + | CreateTargetIndexState + | UpdateIndexMappingsState + | UpdateIndexMappingsWaitForTaskState + | UpdateOrCreateAliasesState + | WaitForYellowIndexState; export type AllControlStates = State['controlState']; @@ -44,6 +80,11 @@ export interface ControlStateMap { INIT: InitState; FATAL: FatalState; DONE: DoneState; + CREATE_TARGET_INDEX: CreateTargetIndexState; + UPDATE_INDEX_MAPPINGS: UpdateIndexMappingsState; + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; + UPDATE_OR_CREATE_ALIASES: UpdateOrCreateAliasesState; + WAIT_FOR_YELLOW_INDEX: WaitForYellowIndexState; } /** diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index ded69aa02e7de..b7258ea0c62a1 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -32,6 +32,7 @@ export const createContextMock = ( migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects, typeRegistry, serializer: serializerMock.create(), + deletedTypes: ['deleted-type'], ...parts, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts new file mode 100644 index 0000000000000..4499ce419d34a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + getModelVersionsFromMappings, + compareModelVersions, + getModelVersionMapForTypes, + type IndexMapping, + type CompareModelVersionResult, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface CheckVersionCompatibilityOpts { + mappings: IndexMapping; + types: SavedObjectsType[]; + source: 'docVersions' | 'mappingVersions'; + deletedTypes: string[]; +} + +export const checkVersionCompatibility = ({ + mappings, + types, + source, + deletedTypes, +}: CheckVersionCompatibilityOpts): CompareModelVersionResult => { + const appVersions = getModelVersionMapForTypes(types); + const indexVersions = getModelVersionsFromMappings({ mappings, source }); + if (!indexVersions) { + throw new Error(`Cannot check version: ${source} not present in the mapping meta`); + } + return compareModelVersions({ appVersions, indexVersions, deletedTypes }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts new file mode 100644 index 0000000000000..d6db500dca633 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getCurrentIndex } from './get_current_index'; +import type { FetchIndexResponse } from '../../actions'; + +describe('getCurrentIndex', () => { + const createIndexResponse = (...indexNames: string[]): FetchIndexResponse => { + return indexNames.reduce((resp, indexName) => { + resp[indexName] = { aliases: {}, mappings: { properties: {} }, settings: {} }; + return resp; + }, {}); + }; + + it('returns the highest numbered index matching the index prefix', () => { + const resp = createIndexResponse('.kibana_1', '.kibana_2'); + expect(getCurrentIndex(resp, '.kibana')).toEqual('.kibana_2'); + }); + + it('ignores other indices', () => { + const resp = createIndexResponse('.kibana_1', '.kibana_2', '.foo_3'); + expect(getCurrentIndex(resp, '.kibana')).toEqual('.kibana_2'); + }); + + it('ignores other indices including the prefix', () => { + const resp = createIndexResponse('.kibana_1', '.kibana_2', '.kibana_task_manager_3'); + expect(getCurrentIndex(resp, '.kibana')).toEqual('.kibana_2'); + }); + + it('ignores other indices including a subpart of the prefix', () => { + const resp = createIndexResponse( + '.kibana_3', + '.kibana_task_manager_1', + '.kibana_task_manager_2' + ); + expect(getCurrentIndex(resp, '.kibana_task_manager')).toEqual('.kibana_task_manager_2'); + }); + + it('returns undefined if no indices match', () => { + const resp = createIndexResponse('.kibana_task_manager_1', '.kibana_task_manager_2'); + expect(getCurrentIndex(resp, '.kibana')).toBeUndefined(); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts new file mode 100644 index 0000000000000..59eea5c550804 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_current_index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { escapeRegExp } from 'lodash'; +import type { FetchIndexResponse } from '../../actions'; + +export const getCurrentIndex = ( + indices: FetchIndexResponse, + indexPrefix: string +): string | undefined => { + const matcher = new RegExp(`^${escapeRegExp(indexPrefix)}[_](?\\d+)$`); + + let lastCount = -1; + Object.keys(indices).forEach((indexName) => { + const match = matcher.exec(indexName); + if (match && match.groups?.counter) { + const suffix = parseInt(match.groups.counter, 10); + lastCount = Math.max(lastCount, suffix); + } + }); + + return lastCount === -1 ? undefined : `${indexPrefix}_${lastCount}`; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index f96ea531e460d..cea76874ae256 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -7,3 +7,5 @@ */ export { buildMigratorConfigs, type MigratorConfig } from './get_migrator_configs'; +export { getCurrentIndex } from './get_current_index'; +export { checkVersionCompatibility } from './check_version_compatibility'; From b647ac7317f23b0312c434e4b40e824e98436fd5 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 28 Feb 2023 10:48:15 +0100 Subject: [PATCH 04/29] update unit tests for init stage --- .../src/zdt/model/stages/init.test.mocks.ts | 19 ++ .../src/zdt/model/stages/init.test.ts | 243 +++++++++++++++--- .../src/zdt/model/stages/init.ts | 5 + .../src/zdt/test_helpers/context.ts | 1 + 4 files changed, 227 insertions(+), 41 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts new file mode 100644 index 0000000000000..5cee1a57a7375 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getCurrentIndexMock = jest.fn(); +export const checkVersionCompatibilityMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getCurrentIndex: getCurrentIndexMock, + checkVersionCompatibility: checkVersionCompatibilityMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index 0301302e18ed3..9f8da7b2d093b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -6,7 +6,9 @@ * Side Public License, v 1. */ +import { getCurrentIndexMock, checkVersionCompatibilityMock } from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; +import { FetchIndexResponse } from '../../../actions'; import { createContextMock, MockedMigratorContext } from '../../test_helpers'; import type { InitState } from '../../state'; import type { StateActionResponse } from '../types'; @@ -15,6 +17,8 @@ import { init } from './init'; describe('Action: init', () => { let context: MockedMigratorContext; + const currentIndex = '.kibana_1'; + const createState = (parts: Partial = {}): InitState => ({ controlState: 'INIT', retryDelay: 0, @@ -23,65 +27,222 @@ describe('Action: init', () => { ...parts, }); + const createResponse = (): FetchIndexResponse => ({ + [currentIndex]: { + aliases: {}, + mappings: { + properties: {}, + _meta: {}, + }, + settings: {}, + }, + }); + beforeEach(() => { - context = createContextMock({ indexPrefix: '.kibana' }); + getCurrentIndexMock.mockReset().mockReturnValue(currentIndex); + checkVersionCompatibilityMock.mockReset().mockReturnValue({ + status: 'equal', + }); + + context = createContextMock({ indexPrefix: '.kibana', types: ['foo', 'bar'] }); + context.typeRegistry.registerType({ + name: 'foo', + mappings: { properties: {} }, + namespaceType: 'single', + hidden: false, + }); + context.typeRegistry.registerType({ + name: 'bar', + mappings: { properties: {} }, + namespaceType: 'single', + hidden: false, + }); }); - test('INIT -> WAIT_FOR_YELLOW_INDEX when index is found', () => { + it('loops to INIT when cluster routing allocation is incompatible', () => { const state = createState(); - const res: StateActionResponse<'INIT'> = Either.right({ - '.kibana_1': { - aliases: { - '.kibana': {}, - }, - mappings: { properties: {} }, - settings: {}, - }, + const res: StateActionResponse<'INIT'> = Either.left({ + type: 'incompatible_cluster_routing_allocation', }); const newState = init(state, res, context); - expect(newState).toEqual( - expect.objectContaining({ - controlState: 'WAIT_FOR_YELLOW_INDEX', - currentIndex: '.kibana_1', - previousMappings: { properties: {} }, - }) - ); + expect(newState.controlState).toEqual('INIT'); + expect(newState.retryCount).toEqual(1); + expect(newState.retryDelay).toEqual(2000); + expect(newState.logs).toHaveLength(1); }); - test('INIT -> CREATE_TARGET_INDEX because its not implemented yet', () => { + it('calls getCurrentIndex with the correct parameters', () => { const state = createState(); - const res: StateActionResponse<'INIT'> = Either.right({ - '.foo_1': { - aliases: { - '.some_alias': {}, - }, - mappings: { properties: {} }, - settings: {}, - }, - }); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); - const newState = init(state, res, context); + init(state, res, context); - expect(newState).toEqual( - expect.objectContaining({ - controlState: 'CREATE_TARGET_INDEX', - }) - ); + expect(getCurrentIndexMock).toHaveBeenCalledTimes(1); + expect(getCurrentIndexMock).toHaveBeenCalledWith(fetchIndexResponse, context.indexPrefix); }); - test('INIT -> INIT when cluster routing allocation is incompatible', () => { + it('calls checkVersionCompatibility with the correct parameters', () => { const state = createState(); - const res: StateActionResponse<'INIT'> = Either.left({ - type: 'incompatible_cluster_routing_allocation', + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + init(state, res, context); + + expect(checkVersionCompatibilityMock).toHaveBeenCalledTimes(1); + expect(checkVersionCompatibilityMock).toHaveBeenCalledWith({ + mappings: fetchIndexResponse[currentIndex].mappings, + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + source: 'mappingVersions', + deletedTypes: context.deletedTypes, }); + }); - const newState = init(state, res, context); + describe('when checkVersionCompatibility returns `greater`', () => { + it('forwards to UPDATE_INDEX_MAPPINGS', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); - expect(newState.controlState).toEqual('INIT'); - expect(newState.retryCount).toEqual(1); - expect(newState.retryDelay).toEqual(2000); - expect(newState.logs).toHaveLength(1); + checkVersionCompatibilityMock.mockReturnValue({ + status: 'greater', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'UPDATE_INDEX_MAPPINGS', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'greater', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: greater`, + ]); + }); + }); + + describe('when checkVersionCompatibility returns `equal`', () => { + it('forwards to UPDATE_OR_CREATE_ALIASES', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'UPDATE_OR_CREATE_ALIASES', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: equal`, + ]); + }); + }); + + describe('when checkVersionCompatibility returns `lesser`', () => { + it('forwards to UPDATE_INDEX_MAPPINGS', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'lesser', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'FATAL', + reason: 'Downgrading model version is currently unsupported', + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'lesser', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: lesser`, + ]); + }); + }); + + describe('when checkVersionCompatibility returns `conflict`', () => { + it('forwards to UPDATE_INDEX_MAPPINGS', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'conflict', + }); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'FATAL', + reason: 'Model version conflict: inconsistent higher/lower versions', + }) + ); + }); + + it('adds a log entry about the version check', () => { + const state = createState(); + const res: StateActionResponse<'INIT'> = Either.right(createResponse()); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'conflict', + }); + + const newState = init(state, res, context); + + expect(newState.logs.map((entry) => entry.message)).toEqual([ + `Mappings model version check result: conflict`, + ]); + }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index 582fd3464b739..2173a69498655 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -54,6 +54,11 @@ export const init: ModelStage< deletedTypes: context.deletedTypes, }); + logs.push({ + level: 'info', + message: `Mappings model version check result: ${versionCheck.status}`, + }); + switch (versionCheck.status) { // app version is greater than the index mapping version. // scenario of an upgrade: we need to update the mappings diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index b7258ea0c62a1..ccb2ca7d52a8a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -17,6 +17,7 @@ import type { MigratorContext } from '../context'; export type MockedMigratorContext = Omit & { elasticsearchClient: ElasticsearchClientMock; + typeRegistry: SavedObjectTypeRegistry; }; export const createContextMock = ( From 8951569ae89d1a99558e7cc05488bea3932ef2b4 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 28 Feb 2023 12:15:49 +0100 Subject: [PATCH 05/29] build mappings for initial index creation --- .../src/actions/index.ts | 2 +- .../src/core/build_active_mappings.ts | 4 +- .../src/core/index.ts | 2 +- .../src/zdt/actions/index.ts | 3 ++ .../src/zdt/model/stages/init.test.mocks.ts | 2 + .../src/zdt/model/stages/init.test.ts | 44 ++++++++++++++++- .../src/zdt/model/stages/init.ts | 6 ++- .../src/zdt/next.ts | 20 +++++--- .../src/zdt/state/types.ts | 6 ++- .../src/zdt/utils/build_index_mappings.ts | 47 +++++++++++++++++++ .../src/zdt/utils/index.ts | 1 + 11 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index 376e232aaaf29..42792ac3ea541 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -84,7 +84,7 @@ export { cleanupUnknownAndExcluded } from './cleanup_unknown_and_excluded'; export { waitForDeleteByQueryTask } from './wait_for_delete_by_query_task'; -export type { CreateIndexParams } from './create_index'; +export type { CreateIndexParams, ClusterShardLimitExceeded } from './create_index'; export { createIndex } from './create_index'; export { checkTargetMappings } from './check_target_mappings'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts index f14de6bc72ee0..3d8648ae33274 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts @@ -27,7 +27,7 @@ import type { export function buildActiveMappings( typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties ): IndexMapping { - const mapping = defaultMapping(); + const mapping = getBaseMappings(); const mergedProperties = validateAndMerge(mapping.properties, typeDefinitions); @@ -114,7 +114,7 @@ function findChangedProp(actual: any, expected: any) { * * @returns {IndexMapping} */ -function defaultMapping(): IndexMapping { +export function getBaseMappings(): IndexMapping { return { dynamic: 'strict', properties: { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts index a113e5e5f77bc..1503fdcf19814 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export { buildActiveMappings } from './build_active_mappings'; +export { buildActiveMappings, getBaseMappings } from './build_active_mappings'; export type { LogFn } from './migration_logger'; export { excludeUnusedTypesQuery, REMOVED_TYPES } from './unused_types'; export { TransformSavedObjectDocumentError } from './transform_saved_object_document_error'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index 9a0cd0c785ced..d24bb20ba8343 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -12,12 +12,14 @@ import type { WaitForTaskCompletionTimeout, IndexNotYellowTimeout, IndexNotGreenTimeout, + ClusterShardLimitExceeded, IndexNotFound, } from '../../actions'; export { initAction as init, waitForIndexStatus, + createIndex, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -32,6 +34,7 @@ export interface ActionErrorTypeMap { index_not_found_exception: IndexNotFound; index_not_green_timeout: IndexNotGreenTimeout; index_not_yellow_timeout: IndexNotYellowTimeout; + cluster_shard_limit_exceeded: ClusterShardLimitExceeded; } /** Type guard for narrowing the type of a left */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts index 5cee1a57a7375..051a4e6341460 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -8,6 +8,7 @@ export const getCurrentIndexMock = jest.fn(); export const checkVersionCompatibilityMock = jest.fn(); +export const buildIndexMappingsMock = jest.fn(); jest.doMock('../../utils', () => { const realModule = jest.requireActual('../../utils'); @@ -15,5 +16,6 @@ jest.doMock('../../utils', () => { ...realModule, getCurrentIndex: getCurrentIndexMock, checkVersionCompatibility: checkVersionCompatibilityMock, + buildIndexMappings: buildIndexMappingsMock, }; }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index 9f8da7b2d093b..7011d086cd656 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -6,7 +6,11 @@ * Side Public License, v 1. */ -import { getCurrentIndexMock, checkVersionCompatibilityMock } from './init.test.mocks'; +import { + getCurrentIndexMock, + checkVersionCompatibilityMock, + buildIndexMappingsMock, +} from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { FetchIndexResponse } from '../../../actions'; import { createContextMock, MockedMigratorContext } from '../../test_helpers'; @@ -100,6 +104,44 @@ describe('Action: init', () => { }); }); + describe('when getCurrentIndex returns undefined', () => { + beforeEach(() => { + getCurrentIndexMock.mockReturnValue(undefined); + }); + + it('calls buildIndexMappings with the correct parameters', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + init(state, res, context); + + expect(buildIndexMappingsMock).toHaveBeenCalledTimes(1); + expect(buildIndexMappingsMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + }); + }); + + it('forwards to CREATE_TARGET_INDEX', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + const mockMappings = { properties: { someMappings: 'string' } }; + buildIndexMappingsMock.mockReturnValue(mockMappings); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'CREATE_TARGET_INDEX', + currentIndex: '.kibana_1', + indexMappings: mockMappings, + }) + ); + }); + }); + describe('when checkVersionCompatibility returns `greater`', () => { it('forwards to UPDATE_INDEX_MAPPINGS', () => { const state = createState(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index 2173a69498655..5eb2b4592d508 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -11,7 +11,7 @@ import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; import type { MigrationLog } from '../../../types'; import { isTypeof } from '../../actions'; -import { getCurrentIndex, checkVersionCompatibility } from '../../utils'; +import { getCurrentIndex, checkVersionCompatibility, buildIndexMappings } from '../../utils'; import type { ModelStage } from '../types'; export const init: ModelStage< @@ -28,6 +28,7 @@ export const init: ModelStage< } } + const types = context.types.map((type) => context.typeRegistry.getType(type)!); const logs: MigrationLog[] = [...state.logs]; const indices = res.right; @@ -40,13 +41,14 @@ export const init: ModelStage< ...state, logs, controlState: 'CREATE_TARGET_INDEX', + currentIndex: `${context.indexPrefix}_1`, + indexMappings: buildIndexMappings({ types }), }; } // Index was found. This is the standard scenario, we check the model versions // compatibility before going further. const currentMappings = indices[currentIndex].mappings; - const types = context.types.map((type) => context.typeRegistry.getType(type)!); const versionCheck = checkVersionCompatibility({ mappings: currentMappings, types, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index 4330b5be038dd..187c198c8ea68 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -37,18 +37,26 @@ const NOT_IMPLEMENTED = () => Promise.resolve({} as any); export const nextActionMap = (context: MigratorContext) => { return { INIT: (state: InitState) => - Actions.init({ client: context.elasticsearchClient, indices: [context.indexPrefix] }), - WAIT_FOR_YELLOW_INDEX: (state: WaitForYellowIndexState) => - Actions.waitForIndexStatus({ + Actions.init({ client: context.elasticsearchClient, - index: state.currentIndex, - status: 'yellow', + indices: [`${context.indexPrefix}_*`], + }), + CREATE_TARGET_INDEX: (state: CreateTargetIndexState) => + Actions.createIndex({ + client: context.elasticsearchClient, + indexName: state.currentIndex, + mappings: state.indexMappings, }), - CREATE_TARGET_INDEX: (state: CreateTargetIndexState) => NOT_IMPLEMENTED, UPDATE_INDEX_MAPPINGS: (state: UpdateIndexMappingsState) => NOT_IMPLEMENTED, UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: (state: UpdateIndexMappingsWaitForTaskState) => NOT_IMPLEMENTED, UPDATE_OR_CREATE_ALIASES: (state: UpdateOrCreateAliasesState) => () => NOT_IMPLEMENTED, + WAIT_FOR_YELLOW_INDEX: (state: WaitForYellowIndexState) => + Actions.waitForIndexStatus({ + client: context.elasticsearchClient, + index: state.currentIndex, + status: 'yellow', + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 7314389dc951e..c40e454964e54 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -22,12 +22,14 @@ export interface InitState extends BaseState { } export interface PostInitState extends BaseState { - currentIndex: string; - previousMappings: IndexMapping; + readonly currentIndex: string; + readonly previousMappings: IndexMapping; } export interface CreateTargetIndexState extends BaseState { readonly controlState: 'CREATE_TARGET_INDEX'; + readonly currentIndex: string; + readonly indexMappings: IndexMapping; } export interface WaitForYellowIndexState extends PostInitState { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts new file mode 100644 index 0000000000000..18e700a2a2eaa --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { cloneDeep } from 'lodash'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + type IndexMapping, + type IndexMappingMeta, + getModelVersionMapForTypes, +} from '@kbn/core-saved-objects-base-server-internal'; +import { getBaseMappings, buildTypesMappings } from '../../core'; + +interface BuildIndexMappingsOpts { + types: SavedObjectsType[]; +} + +export const buildIndexMappings = ({ types }: BuildIndexMappingsOpts): IndexMapping => { + const mappings: IndexMapping = cloneDeep(getBaseMappings()); + const typeMappings = buildTypesMappings(types); + + mappings.properties = { + ...mappings.properties, + ...typeMappings, + }; + + mappings._meta = buildIndexMeta({ types }); + + return mappings; +}; + +interface BuildIndexMetaOpts { + types: SavedObjectsType[]; +} + +export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta => { + const modelVersions = getModelVersionMapForTypes(types); + + return { + mappingVersions: modelVersions, + docVersions: modelVersions, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index cea76874ae256..b3bde5677c415 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -9,3 +9,4 @@ export { buildMigratorConfigs, type MigratorConfig } from './get_migrator_configs'; export { getCurrentIndex } from './get_current_index'; export { checkVersionCompatibility } from './check_version_compatibility'; +export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; From d2e9a4f9fdacb08e4301af4cf9dade3ce2987543 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 28 Feb 2023 14:01:17 +0100 Subject: [PATCH 06/29] continue the implementation --- .../src/common/constants.ts | 9 ++++ .../src/kibana_migrator.ts | 1 + .../src/model/model.ts | 2 +- .../src/zdt/actions/index.ts | 3 ++ .../src/zdt/context/create_context.ts | 2 + .../src/zdt/context/types.ts | 1 + .../src/zdt/migrate_index.ts | 1 + .../src/zdt/model/model.ts | 14 ++++- .../zdt/model/stages/create_target_index.ts | 53 +++++++++++++++++++ .../src/zdt/model/stages/index.ts | 2 + .../src/zdt/model/stages/init.test.ts | 4 +- .../src/zdt/model/stages/init.ts | 22 ++++++-- .../src/zdt/model/stages/update_aliases.ts | 46 ++++++++++++++++ .../src/zdt/next.ts | 8 ++- .../src/zdt/run_zdt_migration.ts | 1 + .../src/zdt/state/index.ts | 2 +- .../src/zdt/state/types.ts | 11 ++-- .../src/zdt/test_helpers/context.ts | 1 + .../src/zdt/utils/get_alias_actions.ts | 41 ++++++++++++++ .../src/zdt/utils/index.ts | 1 + 20 files changed, 210 insertions(+), 15 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts new file mode 100644 index 0000000000000..5dfdb05a0bca8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index 408b277444995..d63cc69d8f7f4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -145,6 +145,7 @@ export class KibanaMigrator implements IKibanaMigrator { private runMigrationZdt(): Promise { return runZeroDowntimeMigration({ + kibanaVersion: this.kibanaVersion, kibanaIndexPrefix: this.kibanaIndex, typeRegistry: this.typeRegistry, logger: this.log, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 4c2a9147eb125..b9418faa0d58b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -44,9 +44,9 @@ import { import { createBatches } from './create_batches'; import type { MigrationLog } from '../types'; import { diffMappings } from '../core/build_active_mappings'; +import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../common/constants'; export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; -const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; export const model = (currentState: State, resW: ResponseType): State => { // The action response `resW` is weakly typed, the type includes all action diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index d24bb20ba8343..a72b63605cb91 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -14,12 +14,14 @@ import type { IndexNotGreenTimeout, ClusterShardLimitExceeded, IndexNotFound, + AliasNotFound, } from '../../actions'; export { initAction as init, waitForIndexStatus, createIndex, + updateAliases, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -35,6 +37,7 @@ export interface ActionErrorTypeMap { index_not_green_timeout: IndexNotGreenTimeout; index_not_yellow_timeout: IndexNotYellowTimeout; cluster_shard_limit_exceeded: ClusterShardLimitExceeded; + alias_not_found_exception: AliasNotFound; } /** Type guard for narrowing the type of a left */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index 6d7049bbea2ad..6dd23773da4f8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -16,6 +16,7 @@ export type CreateContextOps = Omit; * Create the context object that will be used for this index migration. */ export const createContext = ({ + kibanaVersion, types, docLinks, migrationConfig, @@ -25,6 +26,7 @@ export const createContext = ({ serializer, }: CreateContextOps): MigratorContext => { return { + kibanaVersion, indexPrefix, types, elasticsearchClient, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index ced9d79ee92cb..62e090148c649 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -17,6 +17,7 @@ import type { DocLinks } from '@kbn/doc-links'; * The set of static, precomputed values and services used by the ZDT migration */ export interface MigratorContext { + readonly kibanaVersion: string; /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ readonly indexPrefix: string; /** Name of the types that are living in the index */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts index 72ee369236e16..6c3850d5dc8e9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migrate_index.ts @@ -25,6 +25,7 @@ import { model } from './model'; import { createInitialState } from './state'; export interface MigrateIndexOptions { + kibanaVersion: string; indexPrefix: string; types: string[]; /** The SO type registry to use for the migration */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 22d2a952f81c4..3b8eb6ee22721 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -32,14 +32,24 @@ export const model = ( switch (current.controlState) { // TODO: unimplemented case 'WAIT_FOR_YELLOW_INDEX': - case 'CREATE_TARGET_INDEX': case 'UPDATE_INDEX_MAPPINGS': case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': - case 'UPDATE_OR_CREATE_ALIASES': return current; // END TODO case 'INIT': return Stages.init(current, response as StateActionResponse<'INIT'>, context); + case 'CREATE_TARGET_INDEX': + return Stages.createTargetIndex( + current, + response as StateActionResponse<'CREATE_TARGET_INDEX'>, + context + ); + case 'UPDATE_ALIASES': + return Stages.updateAliases( + current, + response as StateActionResponse<'UPDATE_ALIASES'>, + context + ); case 'DONE': case 'FATAL': // The state-action machine will never call the model in the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts new file mode 100644 index 0000000000000..37c1d358bfbd0 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { throwBadResponse } from '../../../model/helpers'; +import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../../../common/constants'; +import { isTypeof } from '../../actions'; +import { getAliasActions } from '../../utils'; +import type { ModelStage } from '../types'; + +export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASES' | 'FATAL'> = ( + state, + res, + context +) => { + if (Either.isLeft(res)) { + const left = res.left; + if (isTypeof(left, 'index_not_green_timeout')) { + // cluster might just be busy so we retry the action for a set number of times. + const retryErrorMessage = `${left.message} Refer to ${context.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; + return delayRetryState(state, retryErrorMessage, context.maxRetryAttempts); + } else if (isTypeof(left, 'cluster_shard_limit_exceeded')) { + return { + ...state, + controlState: 'FATAL', + reason: `${CLUSTER_SHARD_LIMIT_EXCEEDED_REASON} See ${context.migrationDocLinks.clusterShardLimitExceeded}`, + }; + } else { + return throwBadResponse(state, left); + } + } + + const aliasActions = getAliasActions({ + currentIndex: state.currentIndex, + existingAliases: [], + indexPrefix: context.indexPrefix, + kibanaVersion: context.kibanaVersion, + }); + + return { + ...state, + controlState: 'UPDATE_ALIASES', + previousMappings: state.indexMappings, + aliases: [], + aliasActions, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index aa12c6bed1b22..ba765627e9fb5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -7,3 +7,5 @@ */ export { init } from './init'; +export { createTargetIndex } from './create_target_index'; +export { updateAliases } from './update_aliases'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index 7011d086cd656..f1106a94339f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -180,7 +180,7 @@ describe('Action: init', () => { }); describe('when checkVersionCompatibility returns `equal`', () => { - it('forwards to UPDATE_OR_CREATE_ALIASES', () => { + it('forwards to UPDATE_ALIASES', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -193,7 +193,7 @@ describe('Action: init', () => { expect(newState).toEqual( expect.objectContaining({ - controlState: 'UPDATE_OR_CREATE_ALIASES', + controlState: 'UPDATE_ALIASES', currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, }) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index 5eb2b4592d508..c8afefe6d63ef 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -11,12 +11,17 @@ import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; import type { MigrationLog } from '../../../types'; import { isTypeof } from '../../actions'; -import { getCurrentIndex, checkVersionCompatibility, buildIndexMappings } from '../../utils'; +import { + getCurrentIndex, + checkVersionCompatibility, + buildIndexMappings, + getAliasActions, +} from '../../utils'; import type { ModelStage } from '../types'; export const init: ModelStage< 'INIT', - 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_OR_CREATE_ALIASES' | 'FATAL' + 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_ALIASES' | 'FATAL' > = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; @@ -55,6 +60,13 @@ export const init: ModelStage< source: 'mappingVersions', deletedTypes: context.deletedTypes, }); + const aliases = Object.keys(indices[currentIndex].aliases); + const aliasActions = getAliasActions({ + existingAliases: aliases, + currentIndex, + indexPrefix: context.indexPrefix, + kibanaVersion: context.kibanaVersion, + }); logs.push({ level: 'info', @@ -69,6 +81,8 @@ export const init: ModelStage< ...state, logs, currentIndex, + aliases, + aliasActions, previousMappings: currentMappings, controlState: 'UPDATE_INDEX_MAPPINGS', }; @@ -80,8 +94,10 @@ export const init: ModelStage< ...state, logs, currentIndex, + aliases, + aliasActions, previousMappings: currentMappings, - controlState: 'UPDATE_OR_CREATE_ALIASES', + controlState: 'UPDATE_ALIASES', }; // app version is lower than the index mapping version. // likely a rollback scenario - unsupported for the initial implementation diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts new file mode 100644 index 0000000000000..4d91eb116871b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( + state, + res, + context +) => { + if (Either.isLeft(res)) { + const left = res.left; + if (isTypeof(left, 'alias_not_found_exception')) { + // Should never occur given a single operator is supposed to perform the migration. + // we just terminate in that case + return { + ...state, + controlState: 'FATAL', + reason: `Alias missing during alias update`, + }; + } else if (isTypeof(left, 'index_not_found_exception')) { + // Should never occur given a single operator is supposed to perform the migration. + // we just terminate in that case + return { + ...state, + controlState: 'FATAL', + reason: `Index ${left.index} missing during alias update`, + }; + } else { + throwBadResponse(state, left as never); + } + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index 187c198c8ea68..b53ffddfe6aaa 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -14,7 +14,7 @@ import type { CreateTargetIndexState, UpdateIndexMappingsState, UpdateIndexMappingsWaitForTaskState, - UpdateOrCreateAliasesState, + UpdateAliasesState, } from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; @@ -50,7 +50,11 @@ export const nextActionMap = (context: MigratorContext) => { UPDATE_INDEX_MAPPINGS: (state: UpdateIndexMappingsState) => NOT_IMPLEMENTED, UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: (state: UpdateIndexMappingsWaitForTaskState) => NOT_IMPLEMENTED, - UPDATE_OR_CREATE_ALIASES: (state: UpdateOrCreateAliasesState) => () => NOT_IMPLEMENTED, + UPDATE_ALIASES: (state: UpdateAliasesState) => + Actions.updateAliases({ + client: context.elasticsearchClient, + aliasActions: state.aliasActions, + }), WAIT_FOR_YELLOW_INDEX: (state: WaitForYellowIndexState) => Actions.waitForIndexStatus({ client: context.elasticsearchClient, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts index 8a2686e23764b..8902aaea64f52 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts @@ -22,6 +22,7 @@ import { buildMigratorConfigs } from './utils'; import { migrateIndex } from './migrate_index'; export interface RunZeroDowntimeMigrationOpts { + kibanaVersion: string; /** The kibana system index prefix. e.g `.kibana` */ kibanaIndexPrefix: string; /** The SO type registry to use for the migration */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index 193e7e4368181..626b8c76468b5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -13,7 +13,7 @@ export type { WaitForYellowIndexState, UpdateIndexMappingsState, UpdateIndexMappingsWaitForTaskState, - UpdateOrCreateAliasesState, + UpdateAliasesState, DoneState, FatalState, State, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index c40e454964e54..22df604c06f3d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -9,6 +9,7 @@ import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrationLog } from '../../types'; import type { ControlState } from '../../state_action_machine'; +import type { AliasAction } from '../../actions'; export interface BaseState extends ControlState { readonly retryCount: number; @@ -23,6 +24,8 @@ export interface InitState extends BaseState { export interface PostInitState extends BaseState { readonly currentIndex: string; + readonly aliases: string[]; + readonly aliasActions: AliasAction[]; readonly previousMappings: IndexMapping; } @@ -45,8 +48,8 @@ export interface UpdateIndexMappingsWaitForTaskState extends PostInitState { readonly updateTargetMappingsTaskId: string; } -export interface UpdateOrCreateAliasesState extends PostInitState { - readonly controlState: 'UPDATE_OR_CREATE_ALIASES'; +export interface UpdateAliasesState extends PostInitState { + readonly controlState: 'UPDATE_ALIASES'; } /** Migration completed successfully */ @@ -68,7 +71,7 @@ export type State = | CreateTargetIndexState | UpdateIndexMappingsState | UpdateIndexMappingsWaitForTaskState - | UpdateOrCreateAliasesState + | UpdateAliasesState | WaitForYellowIndexState; export type AllControlStates = State['controlState']; @@ -85,7 +88,7 @@ export interface ControlStateMap { CREATE_TARGET_INDEX: CreateTargetIndexState; UPDATE_INDEX_MAPPINGS: UpdateIndexMappingsState; UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; - UPDATE_OR_CREATE_ALIASES: UpdateOrCreateAliasesState; + UPDATE_ALIASES: UpdateAliasesState; WAIT_FOR_YELLOW_INDEX: WaitForYellowIndexState; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index ccb2ca7d52a8a..8161098cb60a6 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -26,6 +26,7 @@ export const createContextMock = ( const typeRegistry = new SavedObjectTypeRegistry(); return { + kibanaVersion: '8.7.0', indexPrefix: '.kibana', types: ['foo', 'bar'], elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts new file mode 100644 index 0000000000000..185b454987b45 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { AliasAction } from '../../actions'; + +interface GetAliasActionOpts { + indexPrefix: string; + currentIndex: string; + existingAliases: string[]; + kibanaVersion: string; +} + +/** + * Build the list of alias actions to perform, depending on the current state of the cluster. + */ +export const getAliasActions = ({ + indexPrefix, + currentIndex, + existingAliases, + kibanaVersion, +}: GetAliasActionOpts): AliasAction[] => { + const actions: AliasAction[] = []; + + const globalAlias = indexPrefix; + const versionAlias = `${indexPrefix}_${kibanaVersion}`; + const allAliases = [globalAlias, versionAlias]; + allAliases.forEach((alias) => { + if (!existingAliases.includes(alias)) { + actions.push({ + add: { index: currentIndex, alias }, + }); + } + }); + + return actions; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index b3bde5677c415..2a567790ec604 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -10,3 +10,4 @@ export { buildMigratorConfigs, type MigratorConfig } from './get_migrator_config export { getCurrentIndex } from './get_current_index'; export { checkVersionCompatibility } from './check_version_compatibility'; export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; +export { getAliasActions } from './get_alias_actions'; From e891e1573ad010ceb1e9b8dd693aa2e2a3841eaa Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 07:53:44 +0100 Subject: [PATCH 07/29] adding first integration test --- .../saved_objects/migrations/test_utils.ts | 9 ++ .../zero_downtime/create_index.test.ts | 144 ++++++++++++++++++ .../zero_downtime/jest.integration.config.js | 19 +++ 3 files changed, 172 insertions(+) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js diff --git a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts index aa78d8dbc5d90..0a84b9bc4b7e8 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts @@ -10,6 +10,7 @@ import { Env } from '@kbn/config'; import { getDocLinksMeta, getDocLinks } from '@kbn/doc-links'; import { REPO_ROOT } from '@kbn/repo-info'; import { getEnvOptions } from '@kbn/config-mocks'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; export const getDocVersion = () => { const env = Env.createDefault(REPO_ROOT, getEnvOptions()); @@ -24,3 +25,11 @@ export const getMigrationDocLink = () => { export const delay = (seconds: number) => new Promise((resolve) => setTimeout(resolve, seconds * 1000)); + +export const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test-type', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts new file mode 100644 index 0000000000000..c638330890420 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; +import { LogRecord } from '@kbn/logging'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import type { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, createType } from '../test_utils'; + +export const logFilePath = Path.join(__dirname, 'create_index.test.log'); + +describe('ZDT upgrades - running on a fresh cluster', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + it('do some shit', async () => { + const fooType = createType({ + name: 'foo', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + '2': dummyModelVersion, + }, + }); + const barType = createType({ + name: 'bar', + mappings: { + properties: { + aKeyword: { type: 'keyword' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + kibanaIndex: '.kibana', + kibanaVersion: '8.7.0', + logFilePath, + types: [fooType, barType], + settings: { + migrations: { + algorithm: 'zdt', + }, + }, + }); + + const result = await runMigrations(); + + expect(result).toEqual([ + { + destIndex: '.kibana', + elapsedMs: expect.any(Number), + status: 'patched', + }, + ]); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + + expect(mappings.properties).toEqual( + expect.objectContaining({ + foo: fooType.mappings, + bar: barType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + docVersions: { + foo: 2, + bar: 1, + }, + mappingVersions: { + foo: 2, + bar: 1, + }, + }); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; + + const expectLogsContains = (messagePrefix: string) => { + expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); + }; + + expectLogsContains('INIT -> CREATE_TARGET_INDEX'); + expectLogsContains('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); + expectLogsContains('UPDATE_ALIASES -> DONE'); + expectLogsContains('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js new file mode 100644 index 0000000000000..4772e43faa148 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/jest.integration.config.js @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + // TODO replace the line below with + // preset: '@kbn/test/jest_integration_node + // to do so, we must fix all integration tests first + // see https://github.com/elastic/kibana/pull/130255/ + preset: '@kbn/test/jest_integration', + rootDir: '../../../../../../..', + roots: ['/src/core/server/integration_tests/saved_objects/migrations/zero_downtime'], + // must override to match all test given there is no `integration_tests` subfolder + testMatch: ['**/*.test.{js,mjs,ts,tsx}'], +}; From 6fd0bb3047339bcb38bf4ec9a9209dbf600c90a0 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 09:54:22 +0100 Subject: [PATCH 08/29] implement update mapping part - untested --- .../index.ts | 2 + .../model_version/get_version_delta.test.ts | 122 ++++++++++++++++++ .../src/model_version/get_version_delta.ts | 97 ++++++++++++++ .../src/model_version/index.ts | 6 +- .../model_version_from_mappings.ts | 26 +++- .../src/zdt/actions/index.ts | 2 + .../src/zdt/model/model.ts | 7 +- .../src/zdt/model/stages/index.ts | 2 + .../src/zdt/model/stages/init.ts | 7 + .../zdt/model/stages/update_index_mappings.ts | 27 ++++ .../update_index_mappings_wait_for_task.ts | 36 ++++++ .../src/zdt/next.ts | 24 +++- .../src/zdt/state/types.ts | 15 ++- .../utils/generate_additive_mapping_diff.ts | 69 ++++++++++ .../src/zdt/utils/index.ts | 1 + 15 files changed, 426 insertions(+), 17 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 4b6afe1e76a63..f93602bd99350 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -54,4 +54,6 @@ export { type CompareModelVersionDetails, type CompareModelVersionResult, getModelVersionsFromMappings, + getModelVersionsFromMappingMeta, + getModelVersionDelta, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts new file mode 100644 index 0000000000000..521ddd2a0efc3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getModelVersionDelta } from './get_version_delta'; + +describe('getModelVersionDelta', () => { + it('generates an upward delta', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 1, + b: 1, + }, + targetVersions: { + a: 2, + b: 3, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('upward'); + expect(result.diff).toEqual([ + { + name: 'a', + current: 1, + target: 2, + }, + { + name: 'b', + current: 1, + target: 3, + }, + ]); + }); + + it('generates a downward delta', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 4, + b: 2, + }, + targetVersions: { + a: 1, + b: 1, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('downward'); + expect(result.diff).toEqual([ + { + name: 'a', + current: 4, + target: 1, + }, + { + name: 'b', + current: 2, + target: 1, + }, + ]); + }); + + it('generates a noop delta', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 4, + b: 2, + }, + targetVersions: { + a: 4, + b: 2, + }, + deletedTypes: [], + }); + + expect(result.status).toEqual('noop'); + expect(result.diff).toEqual([]); + }); + + it('ignores deleted types', () => { + const result = getModelVersionDelta({ + currentVersions: { + a: 1, + b: 3, + }, + targetVersions: { + a: 2, + }, + deletedTypes: ['b'], + }); + + expect(result.status).toEqual('upward'); + expect(result.diff).toEqual([ + { + name: 'a', + current: 1, + target: 2, + }, + ]); + }); + + it('throws if the provided version maps are in conflict', () => { + expect(() => + getModelVersionDelta({ + currentVersions: { + a: 1, + b: 2, + }, + targetVersions: { + a: 2, + b: 1, + }, + deletedTypes: [], + }) + ).toThrow(); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts new file mode 100644 index 0000000000000..63b7137a1c3d0 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ModelVersionMap } from './version_map'; +import { compareModelVersions } from './version_compare'; + +interface GetModelVersionDeltaOpts { + currentVersions: ModelVersionMap; + targetVersions: ModelVersionMap; + deletedTypes: string[]; +} + +type ModelVersionDeltaResultStatus = 'upward' | 'downward' | 'noop'; + +interface ModelVersionDeltaResult { + status: ModelVersionDeltaResultStatus; + diff: ModelVersionDeltaTypeResult[]; +} + +interface ModelVersionDeltaTypeResult { + name: string; + /** the current version the type is in */ + current: number; + /** the target version the type should go to */ + target: number; +} + +/** + * Will generate the difference to go from `appVersions` to `indexVersions`. + * + * @remarks: will throw if the version maps are in conflict + */ +export const getModelVersionDelta = ({ + currentVersions, + targetVersions, + deletedTypes, +}: GetModelVersionDeltaOpts): ModelVersionDeltaResult => { + const compared = compareModelVersions({ + indexVersions: currentVersions, + appVersions: targetVersions, + deletedTypes, + }); + + if (compared.status === 'conflict') { + throw new Error('Cannot generate model version difference: conflict between versions'); + } + + const status: ModelVersionDeltaResultStatus = + compared.status === 'lesser' ? 'downward' : compared.status === 'greater' ? 'upward' : 'noop'; + + const result: ModelVersionDeltaResult = { + status, + diff: [], + }; + + if (compared.status === 'greater') { + compared.details.greater.forEach((type) => { + result.diff.push(getTypeDelta({ type, currentVersions, targetVersions })); + }); + } else if (compared.status === 'lesser') { + compared.details.lesser.forEach((type) => { + result.diff.push(getTypeDelta({ type, currentVersions, targetVersions })); + }); + } + + return result; +}; + +const getTypeDelta = ({ + type, + currentVersions, + targetVersions, +}: { + type: string; + currentVersions: ModelVersionMap; + targetVersions: ModelVersionMap; +}): ModelVersionDeltaTypeResult => { + const currentVersion = currentVersions[type]; + const targetVersion = targetVersions[type]; + if (currentVersion === undefined || targetVersion === undefined) { + // should never occur given we've been checking consistency numerous times before getting there + // but better safe than sorry. + throw new Error( + `Consistency error: trying to generate delta with missing entry for type ${type}` + ); + } + return { + name: type, + current: currentVersion, + target: targetVersion, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 5577378a671b7..2179199921a82 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -25,4 +25,8 @@ export { type CompareModelVersionDetails, type CompareModelVersionResult, } from './version_compare'; -export { getModelVersionsFromMappings } from './model_version_from_mappings'; +export { + getModelVersionsFromMappings, + getModelVersionsFromMappingMeta, +} from './model_version_from_mappings'; +export { getModelVersionDelta } from './get_version_delta'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts index f6fb5a4d52095..8e7816a12fb53 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import type { IndexMapping } from '../mappings'; +import type { IndexMapping, IndexMappingMeta } from '../mappings'; import type { ModelVersionMap } from './version_map'; import { assertValidModelVersion } from './conversion'; @@ -20,12 +20,30 @@ export const getModelVersionsFromMappings = ({ mappings: IndexMapping; source: 'mappingVersions' | 'docVersions'; }): ModelVersionMap | undefined => { - const indexVersions = - source === 'mappingVersions' ? mappings._meta?.mappingVersions : mappings._meta?.docVersions; - if (!indexVersions) { + if (!mappings._meta) { return undefined; } + return getModelVersionsFromMappingMeta({ + meta: mappings._meta, + source, + }); +}; + +/** + * Build the version map from the specified source of the provided mappings meta. + */ +export const getModelVersionsFromMappingMeta = ({ + meta, + source, +}: { + meta: IndexMappingMeta; + source: 'mappingVersions' | 'docVersions'; +}): ModelVersionMap | undefined => { + const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; + if (!indexVersions) { + return undefined; + } return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { map[type] = assertValidModelVersion(rawVersion); return map; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index a72b63605cb91..c02c950cf488f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -22,6 +22,8 @@ export { waitForIndexStatus, createIndex, updateAliases, + updateAndPickupMappings, + waitForPickupUpdatedMappingsTask, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 3b8eb6ee22721..93e96ba0f2038 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -32,7 +32,6 @@ export const model = ( switch (current.controlState) { // TODO: unimplemented case 'WAIT_FOR_YELLOW_INDEX': - case 'UPDATE_INDEX_MAPPINGS': case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': return current; // END TODO @@ -50,6 +49,12 @@ export const model = ( response as StateActionResponse<'UPDATE_ALIASES'>, context ); + case 'UPDATE_INDEX_MAPPINGS': + return Stages.updateIndexMappings( + current, + response as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); case 'DONE': case 'FATAL': // The state-action machine will never call the model in the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index ba765627e9fb5..1715c88230498 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -9,3 +9,5 @@ export { init } from './init'; export { createTargetIndex } from './create_target_index'; export { updateAliases } from './update_aliases'; +export { updateIndexMappings } from './update_index_mappings'; +export { updateIndexMappingsWaitForTask } from './update_index_mappings_wait_for_task'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index c8afefe6d63ef..de8b05e01c13e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -16,6 +16,7 @@ import { checkVersionCompatibility, buildIndexMappings, getAliasActions, + generateAdditiveMappingDiff, } from '../../utils'; import type { ModelStage } from '../types'; @@ -77,6 +78,11 @@ export const init: ModelStage< // app version is greater than the index mapping version. // scenario of an upgrade: we need to update the mappings case 'greater': + const additiveMappingChanges = generateAdditiveMappingDiff({ + types, + meta: currentMappings._meta ?? {}, + deletedTypes: context.deletedTypes, + }); return { ...state, logs, @@ -84,6 +90,7 @@ export const init: ModelStage< aliases, aliasActions, previousMappings: currentMappings, + additiveMappingChanges, controlState: 'UPDATE_INDEX_MAPPINGS', }; // app version and index mapping version are the same. diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts new file mode 100644 index 0000000000000..ffcd67aed2b78 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateIndexMappings: ModelStage< + 'UPDATE_INDEX_MAPPINGS', + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' | 'FATAL' +> = (state, res, context) => { + if (Either.isRight(res)) { + const right = res.right; + return { + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: right.taskId, + }; + } + + throwBadResponse(state, res as never); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts new file mode 100644 index 0000000000000..02c90b82d0e8d --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const updateIndexMappingsWaitForTask: ModelStage< + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + 'DONE' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + const left = res.left; + if (isTypeof(left, 'wait_for_task_completion_timeout')) { + // After waiting for the specified timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticsearch task succeeds or fails. + return delayRetryState(state, left.message, Number.MAX_SAFE_INTEGER); + } else { + throwBadResponse(state, left); + } + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index b53ffddfe6aaa..b0bcfee17bff4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -32,32 +32,42 @@ export type ResponseType = Awaited< >; // TODO: remove when done -const NOT_IMPLEMENTED = () => Promise.resolve({} as any); +// const NOT_IMPLEMENTED = () => Promise.resolve({} as any); export const nextActionMap = (context: MigratorContext) => { + const client = context.elasticsearchClient; return { INIT: (state: InitState) => Actions.init({ - client: context.elasticsearchClient, + client, indices: [`${context.indexPrefix}_*`], }), CREATE_TARGET_INDEX: (state: CreateTargetIndexState) => Actions.createIndex({ - client: context.elasticsearchClient, + client, indexName: state.currentIndex, mappings: state.indexMappings, }), - UPDATE_INDEX_MAPPINGS: (state: UpdateIndexMappingsState) => NOT_IMPLEMENTED, + UPDATE_INDEX_MAPPINGS: (state: UpdateIndexMappingsState) => + Actions.updateAndPickupMappings({ + client, + index: state.currentIndex, + mappings: { properties: state.additiveMappingChanges }, + }), UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: (state: UpdateIndexMappingsWaitForTaskState) => - NOT_IMPLEMENTED, + Actions.waitForPickupUpdatedMappingsTask({ + client, + taskId: state.updateTargetMappingsTaskId, + timeout: '60s', + }), UPDATE_ALIASES: (state: UpdateAliasesState) => Actions.updateAliases({ - client: context.elasticsearchClient, + client, aliasActions: state.aliasActions, }), WAIT_FOR_YELLOW_INDEX: (state: WaitForYellowIndexState) => Actions.waitForIndexStatus({ - client: context.elasticsearchClient, + client, index: state.currentIndex, status: 'yellow', }), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 22df604c06f3d..b462f6dbed85a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrationLog } from '../../types'; import type { ControlState } from '../../state_action_machine'; @@ -17,6 +18,11 @@ export interface BaseState extends ControlState { readonly logs: MigrationLog[]; } +/** + * The state after + */ +export interface StateWithIndexPresent {} + /** Initial state before any action is performed */ export interface InitState extends BaseState { readonly controlState: 'INIT'; @@ -35,12 +41,9 @@ export interface CreateTargetIndexState extends BaseState { readonly indexMappings: IndexMapping; } -export interface WaitForYellowIndexState extends PostInitState { - readonly controlState: 'WAIT_FOR_YELLOW_INDEX'; -} - export interface UpdateIndexMappingsState extends PostInitState { readonly controlState: 'UPDATE_INDEX_MAPPINGS'; + readonly additiveMappingChanges: SavedObjectsMappingProperties; } export interface UpdateIndexMappingsWaitForTaskState extends PostInitState { @@ -52,6 +55,10 @@ export interface UpdateAliasesState extends PostInitState { readonly controlState: 'UPDATE_ALIASES'; } +export interface WaitForYellowIndexState extends PostInitState { + readonly controlState: 'WAIT_FOR_YELLOW_INDEX'; +} + /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts new file mode 100644 index 0000000000000..f23b1e84a87ea --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + SavedObjectsType, + SavedObjectsMappingProperties, +} from '@kbn/core-saved-objects-server'; +import { + IndexMappingMeta, + getModelVersionsFromMappingMeta, + getModelVersionMapForTypes, + getModelVersionDelta, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface GenerateAdditiveMappingsDiffOpts { + types: SavedObjectsType[]; + meta: IndexMappingMeta; + deletedTypes: string[]; +} + +/** + * Generates the additive mapping diff we will need to update the index mapping with. + * + * @param types The types to generate the diff for + * @param meta The meta field of the index we're migrating + * @param deletedTypes The list of deleted types to ignore during diff/comparison + */ +export const generateAdditiveMappingDiff = ({ + types, + meta, + deletedTypes, +}: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { + const typeVersions = getModelVersionMapForTypes(types); + const mappingVersion = getModelVersionsFromMappingMeta({ meta, source: 'mappingVersions' }); + if (!mappingVersion) { + // should never occur given we checked previously in the flow but better safe than sorry. + throw new Error( + 'Cannot generate additive mapping diff: mappingVersions not present on index meta' + ); + } + + const delta = getModelVersionDelta({ + currentVersions: mappingVersion, + targetVersions: typeVersions, + deletedTypes, + }); + const typeMap = types.reduce>((map, type) => { + map[type.name] = type; + return map; + }, {}); + + // TODO: later we will want to generate the proper diff from `SavedObjectsModelExpansionChange.addedMappings` + // for this first implementation this is acceptable given we only allow compatible mapping changes anyway. + // we may want to implement the proper logic before this get used by real (non-test) type owners. + + const changedTypes = delta.diff.map((diff) => diff.name); + + const addedMappings: SavedObjectsMappingProperties = {}; + changedTypes.forEach((type) => { + addedMappings[type] = typeMap[type].mappings; + }); + + return addedMappings; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index 2a567790ec604..ebc22e623f600 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -11,3 +11,4 @@ export { getCurrentIndex } from './get_current_index'; export { checkVersionCompatibility } from './check_version_compatibility'; export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; export { getAliasActions } from './get_alias_actions'; +export { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; From 78efa7dc3c3ac058ff37bb4c8be8262e282a4d7b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 09:56:09 +0100 Subject: [PATCH 09/29] remove dead code --- .../src/zdt/state/types.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index b462f6dbed85a..7dcb350f182f4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -18,11 +18,6 @@ export interface BaseState extends ControlState { readonly logs: MigrationLog[]; } -/** - * The state after - */ -export interface StateWithIndexPresent {} - /** Initial state before any action is performed */ export interface InitState extends BaseState { readonly controlState: 'INIT'; From 4dc5d870033f21e839140274fd5a4d1a0d7199e6 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 11:09:41 +0100 Subject: [PATCH 10/29] adding mapping test --- .../src/zdt/model/model.ts | 7 +- .../zero_downtime/base_types.fixtures.ts | 47 ++++++ .../zero_downtime/create_index.test.ts | 39 +---- .../zero_downtime/update_mappings.test.ts | 153 ++++++++++++++++++ 4 files changed, 211 insertions(+), 35 deletions(-) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 93e96ba0f2038..f631c39864173 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -32,7 +32,6 @@ export const model = ( switch (current.controlState) { // TODO: unimplemented case 'WAIT_FOR_YELLOW_INDEX': - case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': return current; // END TODO case 'INIT': @@ -55,6 +54,12 @@ export const model = ( response as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, context ); + case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': + return Stages.updateIndexMappingsWaitForTask( + current, + response as StateActionResponse<'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'>, + context + ); case 'DONE': case 'FATAL': // The state-action machine will never call the model in the terminating states diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts new file mode 100644 index 0000000000000..9bfac3ac5fd49 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { createType } from '../test_utils'; + +export const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +export const getFooType = () => { + return createType({ + name: 'foo', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + '2': dummyModelVersion, + }, + }); +}; + +export const getBarType = () => { + return createType({ + name: 'bar', + mappings: { + properties: { + aKeyword: { type: 'keyword' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts index c638330890420..e3f45bb561cf6 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -11,9 +11,9 @@ import fs from 'fs/promises'; import JSON5 from 'json5'; import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import type { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; -import { delay, createType } from '../test_utils'; +import { delay } from '../test_utils'; +import { getFooType, getBarType } from './base_types.fixtures'; export const logFilePath = Path.join(__dirname, 'create_index.test.log'); @@ -32,12 +32,6 @@ describe('ZDT upgrades - running on a fresh cluster', () => { return await startES(); }; - const dummyModelVersion: SavedObjectsModelVersion = { - modelChange: { - type: 'expansion', - }, - }; - beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); esServer = await startElasticsearch(); @@ -48,32 +42,9 @@ describe('ZDT upgrades - running on a fresh cluster', () => { await delay(10); }); - it('do some shit', async () => { - const fooType = createType({ - name: 'foo', - mappings: { - properties: { - someField: { type: 'text' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - '2': dummyModelVersion, - }, - }); - const barType = createType({ - name: 'bar', - mappings: { - properties: { - aKeyword: { type: 'keyword' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - }, - }); + it('create the index with the correct mappings and meta', async () => { + const fooType = getFooType(); + const barType = getBarType(); const { runMigrations, client } = await getKibanaMigratorTestKit({ kibanaIndex: '.kibana', diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts new file mode 100644 index 0000000000000..32184af639ab2 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; +import { LogRecord } from '@kbn/logging'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { + getKibanaMigratorTestKit, + type KibanaMigratorTestKitParams, +} from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; + +export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); + +describe('ZDT upgrades - basic mapping update', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + const baseMigratorParams: KibanaMigratorTestKitParams = { + kibanaIndex: '.kibana', + kibanaVersion: '8.7.0', + settings: { + migrations: { + algorithm: 'zdt', + }, + }, + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + const { runMigrations } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('do some shit', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + // increasing the model version of the types + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + fooType.mappings.properties = { + ...fooType.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + barType.modelVersions = { + ...barType.modelVersions, + '2': dummyModelVersion, + }; + barType.mappings.properties = { + ...barType.mappings.properties, + anotherAddedField: { type: 'boolean' }, + }; + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + logFilePath, + types: [fooType, barType], + }); + + const result = await runMigrations(); + + expect(result).toEqual([ + { + destIndex: '.kibana', + elapsedMs: expect.any(Number), + status: 'patched', + }, + ]); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + + expect(mappings.properties).toEqual( + expect.objectContaining({ + foo: fooType.mappings, + bar: barType.mappings, + }) + ); + + expect(mappingMeta).toEqual({ + docVersions: { + foo: 2, + bar: 1, + }, + mappingVersions: { + foo: 2, + bar: 1, + }, + }); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; + + const expectLogsContains = (messagePrefix: string) => { + expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); + }; + + expectLogsContains('INIT -> UPDATE_INDEX_MAPPINGS'); + expectLogsContains('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'); + expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> DONE'); + expectLogsContains('Migration completed'); + }); +}); From 7822670c8d05c69a4cef820528d055e2f4bd8333 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 13:54:36 +0100 Subject: [PATCH 11/29] update the meta too --- .../src/actions/index.ts | 2 +- .../src/zdt/actions/index.ts | 3 +++ .../src/zdt/context/create_context.ts | 6 +++-- .../src/zdt/context/types.ts | 3 +++ .../src/zdt/model/model.ts | 6 +++++ .../src/zdt/model/stages/index.ts | 1 + .../src/zdt/model/stages/init.ts | 27 +++++++++++-------- .../update_index_mappings_wait_for_task.ts | 11 ++++++-- .../stages/update_mapping_model_version.ts | 25 +++++++++++++++++ .../src/zdt/next.ts | 13 ++++++--- .../src/zdt/state/index.ts | 1 + .../src/zdt/state/types.ts | 13 ++++++++- .../src/zdt/test_helpers/context.ts | 4 +++ .../zero_downtime/update_mappings.test.ts | 9 ++++--- 14 files changed, 100 insertions(+), 24 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index ff7a304e4b7f8..7aec839dca066 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -95,7 +95,7 @@ export type { } from './update_and_pickup_mappings'; export { updateAndPickupMappings } from './update_and_pickup_mappings'; -export { updateMappings } from './update_mappings'; +export { updateMappings, type IncompatibleMappingException } from './update_mappings'; import type { UnknownDocsFound } from './check_for_unknown_docs'; import type { IncompatibleClusterRoutingAllocation } from './initialize_action'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index c02c950cf488f..bb135c115ce92 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -15,6 +15,7 @@ import type { ClusterShardLimitExceeded, IndexNotFound, AliasNotFound, + IncompatibleMappingException, } from '../../actions'; export { @@ -22,6 +23,7 @@ export { waitForIndexStatus, createIndex, updateAliases, + updateMappings, updateAndPickupMappings, waitForPickupUpdatedMappingsTask, type InitActionParams, @@ -40,6 +42,7 @@ export interface ActionErrorTypeMap { index_not_yellow_timeout: IndexNotYellowTimeout; cluster_shard_limit_exceeded: ClusterShardLimitExceeded; alias_not_found_exception: AliasNotFound; + incompatible_mapping_exception: IncompatibleMappingException; } /** Type guard for narrowing the type of a left */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index 6dd23773da4f8..7a660ea470443 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -6,9 +6,10 @@ * Side Public License, v 1. */ -import type { MigratorContext } from './types'; -import type { MigrateIndexOptions } from '../migrate_index'; +import { getModelVersionMapForTypes } from '@kbn/core-saved-objects-base-server-internal'; import { REMOVED_TYPES } from '../../core'; +import type { MigrateIndexOptions } from '../migrate_index'; +import type { MigratorContext } from './types'; export type CreateContextOps = Omit; @@ -29,6 +30,7 @@ export const createContext = ({ kibanaVersion, indexPrefix, types, + typeModelVersions: getModelVersionMapForTypes(types.map((type) => typeRegistry.getType(type)!)), elasticsearchClient, typeRegistry, serializer, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 62e090148c649..68adf0b62b458 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -11,6 +11,7 @@ import type { ISavedObjectTypeRegistry, ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; +import { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; /** @@ -22,6 +23,8 @@ export interface MigratorContext { readonly indexPrefix: string; /** Name of the types that are living in the index */ readonly types: string[]; + /** Model versions for the registered types */ + readonly typeModelVersions: ModelVersionMap; /** The client to use for communications with ES */ readonly elasticsearchClient: ElasticsearchClient; /** The maximum number of retries to attempt for a failing action */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index f631c39864173..5f6404c56ffe7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -60,6 +60,12 @@ export const model = ( response as StateActionResponse<'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'>, context ); + case 'UPDATE_MAPPING_MODEL_VERSIONS': + return Stages.updateMappingModelVersion( + current, + response as StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'>, + context + ); case 'DONE': case 'FATAL': // The state-action machine will never call the model in the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index 1715c88230498..0322f92eb35aa 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -11,3 +11,4 @@ export { createTargetIndex } from './create_target_index'; export { updateAliases } from './update_aliases'; export { updateIndexMappings } from './update_index_mappings'; export { updateIndexMappingsWaitForTask } from './update_index_mappings_wait_for_task'; +export { updateMappingModelVersion } from './update_mapping_model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index de8b05e01c13e..ce3480db5ef37 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { cloneDeep } from 'lodash'; import * as Either from 'fp-ts/lib/Either'; import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; @@ -61,6 +62,12 @@ export const init: ModelStage< source: 'mappingVersions', deletedTypes: context.deletedTypes, }); + + logs.push({ + level: 'info', + message: `Mappings model version check result: ${versionCheck.status}`, + }); + const aliases = Object.keys(indices[currentIndex].aliases); const aliasActions = getAliasActions({ existingAliases: aliases, @@ -68,11 +75,7 @@ export const init: ModelStage< indexPrefix: context.indexPrefix, kibanaVersion: context.kibanaVersion, }); - - logs.push({ - level: 'info', - message: `Mappings model version check result: ${versionCheck.status}`, - }); + const currentIndexMeta = cloneDeep(currentMappings._meta!); // cloning as we may be mutating it. switch (versionCheck.status) { // app version is greater than the index mapping version. @@ -85,13 +88,14 @@ export const init: ModelStage< }); return { ...state, + controlState: 'UPDATE_INDEX_MAPPINGS', logs, currentIndex, + currentIndexMeta, aliases, aliasActions, previousMappings: currentMappings, additiveMappingChanges, - controlState: 'UPDATE_INDEX_MAPPINGS', }; // app version and index mapping version are the same. // either application upgrade without model change, or a simple reboot on the same version. @@ -99,21 +103,22 @@ export const init: ModelStage< case 'equal': return { ...state, + controlState: 'UPDATE_ALIASES', logs, currentIndex, + currentIndexMeta, aliases, aliasActions, previousMappings: currentMappings, - controlState: 'UPDATE_ALIASES', }; // app version is lower than the index mapping version. // likely a rollback scenario - unsupported for the initial implementation case 'lesser': return { ...state, - logs, - reason: 'Downgrading model version is currently unsupported', controlState: 'FATAL', + reason: 'Downgrading model version is currently unsupported', + logs, }; // conflicts: version for some types are greater, some are lower // shouldn't occur in any normal scenario - cannot recover @@ -121,9 +126,9 @@ export const init: ModelStage< default: return { ...state, - logs, - reason: 'Model version conflict: inconsistent higher/lower versions', controlState: 'FATAL', + reason: 'Model version conflict: inconsistent higher/lower versions', + logs, }; } }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts index 02c90b82d0e8d..9856cb0c5a1e5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import {} from 'lodash'; import * as Either from 'fp-ts/lib/Either'; import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; @@ -14,7 +15,7 @@ import type { ModelStage } from '../types'; export const updateIndexMappingsWaitForTask: ModelStage< 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', - 'DONE' | 'FATAL' + 'UPDATE_MAPPING_MODEL_VERSIONS' | 'FATAL' > = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; @@ -31,6 +32,12 @@ export const updateIndexMappingsWaitForTask: ModelStage< return { ...state, - controlState: 'DONE', + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + currentIndexMeta: { + ...state.currentIndexMeta, + mappingVersions: { + ...context.typeModelVersions, + }, + }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts new file mode 100644 index 0000000000000..8b4df56fc83a7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateMappingModelVersion: ModelStage< + 'UPDATE_MAPPING_MODEL_VERSIONS', + 'DONE' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index b0bcfee17bff4..3c78973732cbf 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -14,6 +14,7 @@ import type { CreateTargetIndexState, UpdateIndexMappingsState, UpdateIndexMappingsWaitForTaskState, + UpdateMappingModelVersionState, UpdateAliasesState, } from './state'; import type { MigratorContext } from './context'; @@ -31,9 +32,6 @@ export type ResponseType = Awaited< ReturnType> >; -// TODO: remove when done -// const NOT_IMPLEMENTED = () => Promise.resolve({} as any); - export const nextActionMap = (context: MigratorContext) => { const client = context.elasticsearchClient; return { @@ -60,6 +58,15 @@ export const nextActionMap = (context: MigratorContext) => { taskId: state.updateTargetMappingsTaskId, timeout: '60s', }), + UPDATE_MAPPING_MODEL_VERSIONS: (state: UpdateMappingModelVersionState) => + Actions.updateMappings({ + client, + index: state.currentIndex, + mappings: { + properties: {}, + _meta: state.currentIndexMeta, + }, + }), UPDATE_ALIASES: (state: UpdateAliasesState) => Actions.updateAliases({ client, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index 626b8c76468b5..da425445effe5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -13,6 +13,7 @@ export type { WaitForYellowIndexState, UpdateIndexMappingsState, UpdateIndexMappingsWaitForTaskState, + UpdateMappingModelVersionState, UpdateAliasesState, DoneState, FatalState, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 7dcb350f182f4..2c9d856cbebf6 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -7,7 +7,7 @@ */ import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; -import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; +import type { IndexMapping, IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrationLog } from '../../types'; import type { ControlState } from '../../state_action_machine'; import type { AliasAction } from '../../actions'; @@ -28,6 +28,11 @@ export interface PostInitState extends BaseState { readonly aliases: string[]; readonly aliasActions: AliasAction[]; readonly previousMappings: IndexMapping; + /** + * The current _meta field of the index. + * All operations updating this field will update in the state accordingly. + */ + readonly currentIndexMeta: IndexMappingMeta; } export interface CreateTargetIndexState extends BaseState { @@ -46,6 +51,10 @@ export interface UpdateIndexMappingsWaitForTaskState extends PostInitState { readonly updateTargetMappingsTaskId: string; } +export interface UpdateMappingModelVersionState extends PostInitState { + readonly controlState: 'UPDATE_MAPPING_MODEL_VERSIONS'; +} + export interface UpdateAliasesState extends PostInitState { readonly controlState: 'UPDATE_ALIASES'; } @@ -73,6 +82,7 @@ export type State = | CreateTargetIndexState | UpdateIndexMappingsState | UpdateIndexMappingsWaitForTaskState + | UpdateMappingModelVersionState | UpdateAliasesState | WaitForYellowIndexState; @@ -90,6 +100,7 @@ export interface ControlStateMap { CREATE_TARGET_INDEX: CreateTargetIndexState; UPDATE_INDEX_MAPPINGS: UpdateIndexMappingsState; UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; + UPDATE_MAPPING_MODEL_VERSIONS: UpdateMappingModelVersionState; UPDATE_ALIASES: UpdateAliasesState; WAIT_FOR_YELLOW_INDEX: WaitForYellowIndexState; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index 8161098cb60a6..faf9f9c89c9f5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -29,6 +29,10 @@ export const createContextMock = ( kibanaVersion: '8.7.0', indexPrefix: '.kibana', types: ['foo', 'bar'], + typeModelVersions: { + foo: 1, + bar: 2, + }, elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(), maxRetryAttempts: 15, migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects, diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts index 32184af639ab2..1914a4e5f5f7f 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -65,7 +65,7 @@ describe('ZDT upgrades - basic mapping update', () => { await runMigrations(); }; - it('do some shit', async () => { + it('updates the mappings and the meta', async () => { await createBaseline(); const fooType = getFooType(); @@ -130,8 +130,8 @@ describe('ZDT upgrades - basic mapping update', () => { bar: 1, }, mappingVersions: { - foo: 2, - bar: 1, + foo: 3, + bar: 2, }, }); @@ -147,7 +147,8 @@ describe('ZDT upgrades - basic mapping update', () => { expectLogsContains('INIT -> UPDATE_INDEX_MAPPINGS'); expectLogsContains('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'); - expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> DONE'); + expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS'); + expectLogsContains('UPDATE_MAPPING_MODEL_VERSIONS -> DONE'); expectLogsContains('Migration completed'); }); }); From f0e394d852fb192bddb38ba7e6eba7648a87e2ea Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 14:01:53 +0100 Subject: [PATCH 12/29] fix tests --- .../src/zdt/model/stages/init.test.mocks.ts | 2 ++ .../src/zdt/model/stages/init.test.ts | 25 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts index 051a4e6341460..8f773fe951171 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -9,6 +9,7 @@ export const getCurrentIndexMock = jest.fn(); export const checkVersionCompatibilityMock = jest.fn(); export const buildIndexMappingsMock = jest.fn(); +export const generateAdditiveMappingDiffMock = jest.fn(); jest.doMock('../../utils', () => { const realModule = jest.requireActual('../../utils'); @@ -17,5 +18,6 @@ jest.doMock('../../utils', () => { getCurrentIndex: getCurrentIndexMock, checkVersionCompatibility: checkVersionCompatibilityMock, buildIndexMappings: buildIndexMappingsMock, + generateAdditiveMappingDiff: generateAdditiveMappingDiffMock, }; }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index f1106a94339f0..b66dc24cad77a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -10,6 +10,7 @@ import { getCurrentIndexMock, checkVersionCompatibilityMock, buildIndexMappingsMock, + generateAdditiveMappingDiffMock, } from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { FetchIndexResponse } from '../../../actions'; @@ -36,7 +37,7 @@ describe('Action: init', () => { aliases: {}, mappings: { properties: {}, - _meta: {}, + _meta: { mappingVersions: { foo: 1, bar: 1 } }, }, settings: {}, }, @@ -47,6 +48,7 @@ describe('Action: init', () => { checkVersionCompatibilityMock.mockReset().mockReturnValue({ status: 'equal', }); + generateAdditiveMappingDiffMock.mockReset().mockReturnValue({}); context = createContextMock({ indexPrefix: '.kibana', types: ['foo', 'bar'] }); context.typeRegistry.registerType({ @@ -143,6 +145,25 @@ describe('Action: init', () => { }); describe('when checkVersionCompatibility returns `greater`', () => { + it('calls generateAdditiveMappingDiff with the correct parameters', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'greater', + }); + + init(state, res, context); + + expect(generateAdditiveMappingDiffMock).toHaveBeenCalledTimes(1); + expect(generateAdditiveMappingDiffMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + meta: fetchIndexResponse[currentIndex].mappings._meta, + deletedTypes: context.deletedTypes, + }); + }); + it('forwards to UPDATE_INDEX_MAPPINGS', () => { const state = createState(); const fetchIndexResponse = createResponse(); @@ -151,6 +172,7 @@ describe('Action: init', () => { checkVersionCompatibilityMock.mockReturnValue({ status: 'greater', }); + generateAdditiveMappingDiffMock.mockReturnValue({ someToken: {} }); const newState = init(state, res, context); @@ -159,6 +181,7 @@ describe('Action: init', () => { controlState: 'UPDATE_INDEX_MAPPINGS', currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, + additiveMappingChanges: { someToken: {} }, }) ); }); From c5b51e8cb2ce2dfa1ba47773325580252cee9694 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 14:03:27 +0100 Subject: [PATCH 13/29] remove unused stage --- .../src/zdt/model/model.ts | 4 ---- .../src/zdt/next.ts | 7 ------- .../src/zdt/state/types.ts | 8 +------- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 5f6404c56ffe7..62971c3a614aa 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -30,10 +30,6 @@ export const model = ( } switch (current.controlState) { - // TODO: unimplemented - case 'WAIT_FOR_YELLOW_INDEX': - return current; - // END TODO case 'INIT': return Stages.init(current, response as StateActionResponse<'INIT'>, context); case 'CREATE_TARGET_INDEX': diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index 3c78973732cbf..38308fecabc88 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -10,7 +10,6 @@ import type { AllActionStates, State, InitState, - WaitForYellowIndexState, CreateTargetIndexState, UpdateIndexMappingsState, UpdateIndexMappingsWaitForTaskState, @@ -72,12 +71,6 @@ export const nextActionMap = (context: MigratorContext) => { client, aliasActions: state.aliasActions, }), - WAIT_FOR_YELLOW_INDEX: (state: WaitForYellowIndexState) => - Actions.waitForIndexStatus({ - client, - index: state.currentIndex, - status: 'yellow', - }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 2c9d856cbebf6..0b389d227d408 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -59,10 +59,6 @@ export interface UpdateAliasesState extends PostInitState { readonly controlState: 'UPDATE_ALIASES'; } -export interface WaitForYellowIndexState extends PostInitState { - readonly controlState: 'WAIT_FOR_YELLOW_INDEX'; -} - /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; @@ -83,8 +79,7 @@ export type State = | UpdateIndexMappingsState | UpdateIndexMappingsWaitForTaskState | UpdateMappingModelVersionState - | UpdateAliasesState - | WaitForYellowIndexState; + | UpdateAliasesState; export type AllControlStates = State['controlState']; @@ -102,7 +97,6 @@ export interface ControlStateMap { UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; UPDATE_MAPPING_MODEL_VERSIONS: UpdateMappingModelVersionState; UPDATE_ALIASES: UpdateAliasesState; - WAIT_FOR_YELLOW_INDEX: WaitForYellowIndexState; } /** From affe31378345359875fab3aef829b23e32db8e77 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 14:16:02 +0100 Subject: [PATCH 14/29] fix ts errors --- .../src/zdt/model/stages/create_target_index.ts | 4 ++++ .../src/zdt/state/index.ts | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts index 37c1d358bfbd0..bb0697a70cf51 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { cloneDeep } from 'lodash'; import * as Either from 'fp-ts/lib/Either'; import { delayRetryState } from '../../../model/retry_state'; import { throwBadResponse } from '../../../model/helpers'; @@ -43,10 +44,13 @@ export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASE kibanaVersion: context.kibanaVersion, }); + const currentIndexMeta = cloneDeep(state.indexMappings._meta!); + return { ...state, controlState: 'UPDATE_ALIASES', previousMappings: state.indexMappings, + currentIndexMeta, aliases: [], aliasActions, }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index da425445effe5..0f7d28507bb4a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -10,7 +10,6 @@ export type { BaseState, InitState, CreateTargetIndexState, - WaitForYellowIndexState, UpdateIndexMappingsState, UpdateIndexMappingsWaitForTaskState, UpdateMappingModelVersionState, From 8a05ba2238547343add21fce17f08fb10f93d97c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 14:34:18 +0100 Subject: [PATCH 15/29] update doc --- .../src/mappings/types.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index 1eca0245f18f2..0267f2ce27c1a 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -62,15 +62,19 @@ export interface IndexMappingMeta { * with each key being a root-level mapping property, and each value being * the md5 hash of that mapping's value when the index was created. * - * @remark: Only defined for the v2 migration algorithm. + * @remark: Only defined for indices using the v2 migration algorithm. */ migrationMappingPropertyHashes?: { [k: string]: string }; /** - * TODO: doc + * The current model versions of the mapping of the index. + * + * @remark: Only defined for indices using the zdt migration algorithm. */ mappingVersions?: { [k: string]: number }; /** - * TODO: doc + * The current model versions of the documents of the index. + * + * @remark: Only defined for indices using the zdt migration algorithm. */ docVersions?: { [k: string]: number }; } From 0d0d51e2abed8fa9d8b53f21bfd99d81d313ae09 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 14:37:59 +0100 Subject: [PATCH 16/29] more doc --- .../src/model_version/get_version_delta.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts index 63b7137a1c3d0..cbc0e2bcba670 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts @@ -23,8 +23,9 @@ interface ModelVersionDeltaResult { } interface ModelVersionDeltaTypeResult { + /** the name of the type */ name: string; - /** the current version the type is in */ + /** the current version the type is at */ current: number; /** the target version the type should go to */ target: number; From ecb453527aafdace5180c17a43c2e5b6a07c8f36 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 1 Mar 2023 15:22:47 +0100 Subject: [PATCH 17/29] self review / nits --- .../src/model_version/get_version_delta.ts | 2 +- .../src/zdt/context/types.ts | 3 ++- .../src/zdt/model/stages/init.ts | 3 ++- .../src/zdt/state/types.ts | 15 ++++++++++++++- .../src/zdt/utils/build_index_mappings.ts | 10 ++++++++++ .../zero_downtime/update_mappings.test.ts | 1 + 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts index cbc0e2bcba670..f39c52b47f9f7 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts @@ -32,7 +32,7 @@ interface ModelVersionDeltaTypeResult { } /** - * Will generate the difference to go from `appVersions` to `indexVersions`. + * Will generate the difference to go from `currentVersions` to `targetVersions`. * * @remarks: will throw if the version maps are in conflict */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 68adf0b62b458..5b6d4b2fe27e9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -11,13 +11,14 @@ import type { ISavedObjectTypeRegistry, ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; /** * The set of static, precomputed values and services used by the ZDT migration */ export interface MigratorContext { + /** The current Kibana version */ readonly kibanaVersion: string; /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ readonly indexPrefix: string; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index ce3480db5ef37..da138730364f4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -75,7 +75,8 @@ export const init: ModelStage< indexPrefix: context.indexPrefix, kibanaVersion: context.kibanaVersion, }); - const currentIndexMeta = cloneDeep(currentMappings._meta!); // cloning as we may be mutating it. + // cloning as we may be mutating it in later stages. + const currentIndexMeta = cloneDeep(currentMappings._meta!); switch (versionCheck.status) { // app version is greater than the index mapping version. diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index 0b389d227d408..d43c6e49dd5e5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -24,12 +24,25 @@ export interface InitState extends BaseState { } export interface PostInitState extends BaseState { + /** + * The index we're currently migrating. + */ readonly currentIndex: string; + /** + * The aliases that are already present for the current index. + */ readonly aliases: string[]; + /** + * The alias actions to perform to update the aliases. + */ readonly aliasActions: AliasAction[]; + /** + * The *previous* mappings (and _meta), as they were when we resolved the index + * information. This shouldn't be updated once populated. + */ readonly previousMappings: IndexMapping; /** - * The current _meta field of the index. + * The *current* _meta field of the index. * All operations updating this field will update in the state accordingly. */ readonly currentIndexMeta: IndexMappingMeta; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts index 18e700a2a2eaa..dee04aef7d5f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -19,6 +19,11 @@ interface BuildIndexMappingsOpts { types: SavedObjectsType[]; } +/** + * Build the mappings to use when creating a new index. + * + * @param types The list of all registered SO types. + */ export const buildIndexMappings = ({ types }: BuildIndexMappingsOpts): IndexMapping => { const mappings: IndexMapping = cloneDeep(getBaseMappings()); const typeMappings = buildTypesMappings(types); @@ -37,6 +42,11 @@ interface BuildIndexMetaOpts { types: SavedObjectsType[]; } +/** + * Build the mapping _meta field to used when creating a new index. + * + * @param types The list of all registered SO types. + */ export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta => { const modelVersions = getModelVersionMapForTypes(types); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts index 1914a4e5f5f7f..5d27c32c1ee6a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -125,6 +125,7 @@ describe('ZDT upgrades - basic mapping update', () => { ); expect(mappingMeta).toEqual({ + // doc migration not implemented yet - docVersions are not bumped. docVersions: { foo: 2, bar: 1, From c4240583aa88578e5dfa3b4f357c7637c399978b Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 2 Mar 2023 16:17:16 +0100 Subject: [PATCH 18/29] review nits --- .../src/zdt/model/stages/init.test.ts | 4 ++-- .../src/zdt/utils/build_index_mappings.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index b66dc24cad77a..b0295c938e483 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -240,7 +240,7 @@ describe('Action: init', () => { }); describe('when checkVersionCompatibility returns `lesser`', () => { - it('forwards to UPDATE_INDEX_MAPPINGS', () => { + it('forwards to FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -276,7 +276,7 @@ describe('Action: init', () => { }); describe('when checkVersionCompatibility returns `conflict`', () => { - it('forwards to UPDATE_INDEX_MAPPINGS', () => { + it('forwards to FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts index dee04aef7d5f0..6221221ab993c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -43,7 +43,7 @@ interface BuildIndexMetaOpts { } /** - * Build the mapping _meta field to used when creating a new index. + * Build the mapping _meta field to use when creating a new index. * * @param types The list of all registered SO types. */ From 4c1fce672921277c60946bc2a6c814e24e5f1a00 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 2 Mar 2023 17:38:03 +0100 Subject: [PATCH 19/29] add some unit tests --- .../zdt/model/stages/update_aliases.test.ts | 74 +++++++++++++++++++ .../stages/update_index_mappings.test.ts | 53 +++++++++++++ .../src/zdt/test_helpers/index.ts | 1 + .../src/zdt/test_helpers/state.ts | 21 ++++++ 4 files changed, 149 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts new file mode 100644 index 0000000000000..1c28b6e43badf --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateAliasesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateAliases } from './update_aliases'; + +describe('Action: updateAliases', () => { + let context: MockedMigratorContext; + + const createState = (parts: Partial = {}): UpdateAliasesState => ({ + ...createPostInitState(), + controlState: 'UPDATE_ALIASES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_ALIASES -> FATAL in case of alias_not_found_exception', () => { + const state = createState(); + const res: StateActionResponse<'UPDATE_ALIASES'> = Either.left({ + type: 'alias_not_found_exception', + }); + + const newState = updateAliases(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: `Alias missing during alias update`, + }); + }); + + it('UPDATE_ALIASES -> FATAL in case of index_not_found_exception', () => { + const state = createState(); + const res: StateActionResponse<'UPDATE_ALIASES'> = Either.left({ + type: 'index_not_found_exception', + index: '.test', + }); + + const newState = updateAliases(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: `Index .test missing during alias update`, + }); + }); + + it('UPDATE_ALIASES -> DONE if successful', () => { + const state = createState(); + const res: StateActionResponse<'UPDATE_ALIASES'> = Either.right('update_aliases_succeeded'); + + const newState = updateAliases(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts new file mode 100644 index 0000000000000..ae5c391a85ae4 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { UpdateIndexMappingsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateIndexMappings } from './update_index_mappings'; + +describe('Action: updateIndexMappings', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateIndexMappingsState => ({ + ...createPostInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: {}, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ + taskId: '42', + }); + + const newState = updateIndexMappings( + state, + res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: '42', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts index 8b0c329317c03..8387dc0d20bda 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts @@ -7,3 +7,4 @@ */ export { createContextMock, type MockedMigratorContext } from './context'; +export { createPostInitState } from './state'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts new file mode 100644 index 0000000000000..bd95881abbba4 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { PostInitState } from '../state/types'; + +export const createPostInitState = (): PostInitState => ({ + controlState: 'INIT', + retryDelay: 0, + retryCount: 0, + logs: [], + currentIndex: '.kibana_1', + aliases: ['.kibana'], + aliasActions: [], + previousMappings: { properties: {} }, + currentIndexMeta: {}, +}); From 6d0b3f0de4fa3d631a728d03ef6d29552bb0e590 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 08:42:28 +0100 Subject: [PATCH 20/29] extract createDelayFn --- .../src/common/utils/delay.ts | 25 +++++++++++++++++++ .../src/common/utils/index.ts | 1 + .../src/next.ts | 9 ++----- .../src/zdt/next.ts | 11 +------- 4 files changed, 29 insertions(+), 17 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts new file mode 100644 index 0000000000000..8a8a87ec65def --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface RetryableState { + retryCount: number; + retryDelay: number; +} + +/** + * HOC wrapping the function with a delay. + */ +export const createDelayFn = + (state: RetryableState) => + any>(fn: F): (() => ReturnType) => { + return () => { + return state.retryDelay > 0 + ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) + : fn(); + }; + }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts index 962f40b87db02..3801e180ed024 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/index.ts @@ -7,3 +7,4 @@ */ export { logActionResponse, logStateTransition, type LogAwareState } from './logs'; +export { createDelayFn, type RetryableState } from './delay'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts index 8cebce9995900..11c675d88e39f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts @@ -48,6 +48,7 @@ import type { WaitForMigrationCompletionState, WaitForYellowSourceState, } from './state'; +import { createDelayFn } from './common/utils'; import type { TransformRawDocs } from './types'; import * as Actions from './actions'; import { REMOVED_TYPES } from './core'; @@ -247,13 +248,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra export const next = (client: ElasticsearchClient, transformRawDocs: TransformRawDocs) => { const map = nextActionMap(client, transformRawDocs); return (state: State) => { - const delay = any>(fn: F): (() => ReturnType) => { - return () => { - return state.retryDelay > 0 - ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) - : fn(); - }; - }; + const delay = createDelayFn(state); if (state.controlState === 'DONE' || state.controlState === 'FATAL') { // Return null if we're in one of the terminating states diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index 38308fecabc88..a85e9bfde6b56 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -18,6 +18,7 @@ import type { } from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; +import { createDelayFn } from '../common/utils'; export type ActionMap = ReturnType; @@ -95,13 +96,3 @@ export const next = (context: MigratorContext) => { } }; }; - -const createDelayFn = - (state: State) => - any>(fn: F): (() => ReturnType) => { - return () => { - return state.retryDelay > 0 - ? new Promise((resolve) => setTimeout(resolve, state.retryDelay)).then(fn) - : fn(); - }; - }; From 7bfb9261f8344d2d3abc8cb2c279296b0c00c480 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 08:45:04 +0100 Subject: [PATCH 21/29] tsdoc --- .../src/zdt/run_zdt_migration.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts index 8902aaea64f52..44a067944bd05 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/run_zdt_migration.ts @@ -22,8 +22,9 @@ import { buildMigratorConfigs } from './utils'; import { migrateIndex } from './migrate_index'; export interface RunZeroDowntimeMigrationOpts { + /** The current Kibana version */ kibanaVersion: string; - /** The kibana system index prefix. e.g `.kibana` */ + /** The Kibana system index prefix. e.g `.kibana` or `.kibana_task_manager` */ kibanaIndexPrefix: string; /** The SO type registry to use for the migration */ typeRegistry: ISavedObjectTypeRegistry; From d61541953d5cee91e8b9328aa9d8d3e7e6ca524d Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 08:59:53 +0100 Subject: [PATCH 22/29] add more unit tests on stages --- .../src/zdt/model/model.test.mocks.ts | 2 +- .../src/zdt/model/model.test.ts | 38 +++++++++---- ...pdate_index_mappings_wait_for_task.test.ts | 55 +++++++++++++++++++ .../update_mapping_model_version.test.ts | 55 +++++++++++++++++++ 4 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts index 70faab1fdc8d5..b59f1ef7b7aa4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.mocks.ts @@ -11,7 +11,7 @@ const realStages = jest.requireActual('./stages'); export const StageMocks = Object.keys(realStages).reduce((mocks, key) => { mocks[key] = jest.fn().mockImplementation((state: unknown) => state); return mocks; -}, {} as Record); +}, {} as Record>); jest.doMock('./stages', () => { return StageMocks; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts index b6c91e702bd2b..e9343d719b301 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts @@ -10,8 +10,9 @@ import { StageMocks } from './model.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { createContextMock, MockedMigratorContext } from '../test_helpers'; import type { RetryableEsClientError } from '../../actions'; -import type { State, BaseState, FatalState } from '../state'; +import type { State, BaseState, FatalState, AllActionStates } from '../state'; import type { StateActionResponse } from './types'; +import type { ResponseType } from '../next'; import { model } from './model'; describe('model', () => { @@ -113,22 +114,39 @@ describe('model', () => { }); describe('dispatching to correct stage', () => { - test('dispatching INIT state', () => { - const state: State = { + const createStubState = (controlState: AllActionStates): State => + ({ ...baseState, - controlState: 'INIT', - }; - const res: StateActionResponse<'INIT'> = Either.right({ + controlState, + } as unknown as State); + + const createStubResponse = (): ResponseType => + Either.right({ '.kibana_7.11.0_001': { aliases: {}, mappings: { properties: {} }, settings: {}, }, - }); - model(state, res, context); + }) as StateActionResponse<'INIT'>; + + const stageMapping: Record = { + INIT: StageMocks.init, + CREATE_TARGET_INDEX: StageMocks.createTargetIndex, + UPDATE_INDEX_MAPPINGS: StageMocks.updateIndexMappings, + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: StageMocks.updateIndexMappingsWaitForTask, + UPDATE_MAPPING_MODEL_VERSIONS: StageMocks.updateMappingModelVersion, + UPDATE_ALIASES: StageMocks.updateAliases, + }; - expect(StageMocks.init).toHaveBeenCalledTimes(1); - expect(StageMocks.init).toHaveBeenCalledWith(state, res, context); + Object.entries(stageMapping).forEach(([stage, handler]) => { + test(`dispatch ${stage} state`, () => { + const state = createStubState(stage as AllActionStates); + const res = createStubResponse(); + model(state, res, context); + + expect(handler).toHaveBeenCalledTimes(1); + expect(handler).toHaveBeenCalledWith(state, res, context); + }); }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts new file mode 100644 index 0000000000000..dfd033af53fc7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { UpdateIndexMappingsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateIndexMappings } from './update_index_mappings'; + +// TODO: all + +describe('Action: updateIndexMappings', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateIndexMappingsState => ({ + ...createPostInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: {}, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ + taskId: '42', + }); + + const newState = updateIndexMappings( + state, + res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: '42', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts new file mode 100644 index 0000000000000..dfd033af53fc7 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { UpdateIndexMappingsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateIndexMappings } from './update_index_mappings'; + +// TODO: all + +describe('Action: updateIndexMappings', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateIndexMappingsState => ({ + ...createPostInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: {}, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ + taskId: '42', + }); + + const newState = updateIndexMappings( + state, + res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', + updateTargetMappingsTaskId: '42', + }); + }); +}); From ae2235113a63b5c12ed24554cba264bf0f2c0594 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 09:23:31 +0100 Subject: [PATCH 23/29] last stage unit tests --- .../stages/create_target_index.test.mocks.ts | 17 +++ .../model/stages/create_target_index.test.ts | 107 ++++++++++++++++++ .../src/zdt/model/stages/init.test.ts | 2 +- .../zdt/model/stages/update_aliases.test.ts | 2 +- .../stages/update_index_mappings.test.ts | 2 +- ...pdate_index_mappings_wait_for_task.test.ts | 4 +- .../update_mapping_model_version.test.ts | 4 +- 7 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts new file mode 100644 index 0000000000000..b85e06bde59bc --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getAliasActionsMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getAliasActions: getAliasActionsMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts new file mode 100644 index 0000000000000..cd15594d32b29 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getAliasActionsMock } from './create_target_index.test.mocks'; +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CreateTargetIndexState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { createTargetIndex } from './create_target_index'; + +describe('Stage: createTargetIndex', () => { + let context: MockedMigratorContext; + + const createState = (parts: Partial = {}): CreateTargetIndexState => ({ + ...createPostInitState(), + controlState: 'CREATE_TARGET_INDEX', + indexMappings: { properties: { foo: { type: 'text' } }, _meta: {} }, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + getAliasActionsMock.mockReset().mockReturnValue([]); + }); + + describe('In case of left return', () => { + it('CREATE_TARGET_INDEX -> CREATE_TARGET_INDEX in case of index_not_green_timeout exception', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.left({ + type: 'index_not_green_timeout', + message: 'index not green', + }); + + const result = createTargetIndex(state, res, context); + + expect(result).toEqual({ + ...state, + controlState: 'CREATE_TARGET_INDEX', + retryCount: 1, + retryDelay: 2000, + logs: expect.any(Array), + }); + }); + + it('CREATE_TARGET_INDEX -> FATAL in case of cluster_shard_limit_exceeded exception', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.left({ + type: 'cluster_shard_limit_exceeded', + }); + + const result = createTargetIndex(state, res, context); + + expect(result).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.stringContaining('[cluster_shard_limit_exceeded]'), + }); + }); + }); + + describe('In case of right return', () => { + it('calls getAliasActions with the correct parameters', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + createTargetIndex(state, res, context); + + expect(getAliasActionsMock).toHaveBeenCalledTimes(1); + expect(getAliasActionsMock).toHaveBeenCalledWith({ + currentIndex: state.currentIndex, + existingAliases: [], + indexPrefix: context.indexPrefix, + kibanaVersion: context.kibanaVersion, + }); + }); + + it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + const aliasActions = [{ add: { index: '.kibana_1', alias: '.kibana' } }]; + getAliasActionsMock.mockReturnValue(aliasActions); + + const newState = createTargetIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_ALIASES', + previousMappings: state.indexMappings, + currentIndexMeta: state.indexMappings._meta, + aliases: [], + aliasActions, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index b0295c938e483..ea6f4424404ef 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -19,7 +19,7 @@ import type { InitState } from '../../state'; import type { StateActionResponse } from '../types'; import { init } from './init'; -describe('Action: init', () => { +describe('Stage: init', () => { let context: MockedMigratorContext; const currentIndex = '.kibana_1'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts index 1c28b6e43badf..4fac3d02db044 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts @@ -16,7 +16,7 @@ import type { UpdateAliasesState } from '../../state'; import type { StateActionResponse } from '../types'; import { updateAliases } from './update_aliases'; -describe('Action: updateAliases', () => { +describe('Stage: updateAliases', () => { let context: MockedMigratorContext; const createState = (parts: Partial = {}): UpdateAliasesState => ({ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts index ae5c391a85ae4..971482d3262b7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings.test.ts @@ -17,7 +17,7 @@ import type { UpdateIndexMappingsState } from '../../state'; import type { StateActionResponse } from '../types'; import { updateIndexMappings } from './update_index_mappings'; -describe('Action: updateIndexMappings', () => { +describe('Stage: updateIndexMappings', () => { let context: MockedMigratorContext; const createState = ( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts index dfd033af53fc7..971482d3262b7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.test.ts @@ -17,9 +17,7 @@ import type { UpdateIndexMappingsState } from '../../state'; import type { StateActionResponse } from '../types'; import { updateIndexMappings } from './update_index_mappings'; -// TODO: all - -describe('Action: updateIndexMappings', () => { +describe('Stage: updateIndexMappings', () => { let context: MockedMigratorContext; const createState = ( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts index dfd033af53fc7..971482d3262b7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -17,9 +17,7 @@ import type { UpdateIndexMappingsState } from '../../state'; import type { StateActionResponse } from '../types'; import { updateIndexMappings } from './update_index_mappings'; -// TODO: all - -describe('Action: updateIndexMappings', () => { +describe('Stage: updateIndexMappings', () => { let context: MockedMigratorContext; const createState = ( From a70b1f311ca10bd3a5bcaa3a82c046c78151932d Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 09:25:38 +0100 Subject: [PATCH 24/29] just some doc --- .../src/zdt/state/create_initial_state.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts index ceb68b42b4a72..f5132d2ad2739 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/create_initial_state.ts @@ -9,6 +9,9 @@ import type { InitState, State } from './types'; import type { MigratorContext } from '../context'; +/** + * Create the initial state to be used for the ZDT migrator. + */ export const createInitialState = (context: MigratorContext): State => { const initialState: InitState = { controlState: 'INIT', From 0cd232eca7fa0c1f640e6d958d5aebb8c2b79c40 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 10:02:15 +0100 Subject: [PATCH 25/29] some utils unit tests --- .../src/zdt/test_helpers/index.ts | 1 + .../src/zdt/test_helpers/saved_object_type.ts | 17 ++++ .../zdt/utils/build_index_mappings.test.ts | 83 ++++++++++++++++++ .../src/zdt/utils/get_alias_actions.test.ts | 86 +++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts index 8387dc0d20bda..5658828fc2e0c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts @@ -8,3 +8,4 @@ export { createContextMock, type MockedMigratorContext } from './context'; export { createPostInitState } from './state'; +export { createType } from './saved_object_type'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts new file mode 100644 index 0000000000000..5f1f2ee8676f1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object_type.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; + +export const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test-type', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts new file mode 100644 index 0000000000000..f4536cf1c75b0 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; +import { createType } from '../test_helpers'; + +const getTestTypes = () => { + const foo = createType({ + name: 'foo', + switchToModelVersionAt: '8.7.0', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { fooField: { type: 'text' } } }, + }); + const bar = createType({ + name: 'bar', + switchToModelVersionAt: '8.7.0', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { barField: { type: 'text' } } }, + }); + const dolly = createType({ + name: 'dolly', + switchToModelVersionAt: '8.7.0', + modelVersions: () => ({ + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + 3: { modelChange: { type: 'expansion' } }, + }), + mappings: { properties: { dollyField: { type: 'text' } } }, + }); + + return { foo, bar, dolly }; +}; + +describe('buildIndexMappings', () => { + it('builds the mappings used when creating a new index', () => { + const { foo, bar, dolly } = getTestTypes(); + const mappings = buildIndexMappings({ + types: [foo, bar, dolly], + }); + + expect(mappings).toEqual({ + dynamic: 'strict', + properties: expect.objectContaining({ + foo: foo.mappings, + bar: bar.mappings, + dolly: dolly.mappings, + }), + _meta: buildIndexMeta({ types: [foo, bar, dolly] }), + }); + }); +}); + +describe('buildIndexMeta', () => { + it('builds the _meta field value of the mapping', () => { + const { foo, bar, dolly } = getTestTypes(); + const meta = buildIndexMeta({ + types: [foo, bar, dolly], + }); + + expect(meta).toEqual({ + mappingVersions: { + foo: 2, + bar: 1, + dolly: 3, + }, + docVersions: { + foo: 2, + bar: 1, + dolly: 3, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts new file mode 100644 index 0000000000000..42d7bcf27d11a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/get_alias_actions.test.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getAliasActions } from './get_alias_actions'; + +describe('getAliasActions', () => { + it('creates the global and version aliases', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: [], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([ + { add: { alias: '.kibana', index: '.kibana_1' } }, + { add: { alias: '.kibana_8.7.0', index: '.kibana_1' } }, + ]); + }); + + it('does not create the version alias when already present', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana_8.7.0'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([{ add: { alias: '.kibana', index: '.kibana_1' } }]); + }); + + it('does not create the global alias when already present', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([{ add: { alias: '.kibana_8.7.0', index: '.kibana_1' } }]); + }); + + it('creates nothing when both aliases are present', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana', '.kibana_8.7.0'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([]); + }); + + it('ignores other aliases', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana', + currentIndex: '.kibana_1', + existingAliases: ['.kibana_8.6.0', '.kibana_old'], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([ + { add: { alias: '.kibana', index: '.kibana_1' } }, + { add: { alias: '.kibana_8.7.0', index: '.kibana_1' } }, + ]); + }); + + it('accepts other prefixes', () => { + const actions = getAliasActions({ + indexPrefix: '.kibana_task_manager', + currentIndex: '.kibana_task_manager_2', + existingAliases: [], + kibanaVersion: '8.7.0', + }); + + expect(actions).toEqual([ + { add: { alias: '.kibana_task_manager', index: '.kibana_task_manager_2' } }, + { add: { alias: '.kibana_task_manager_8.7.0', index: '.kibana_task_manager_2' } }, + ]); + }); +}); From 19a82b1e4efca59959edd648c67408f2b1b4b5d4 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 13:30:44 +0100 Subject: [PATCH 26/29] more unit tests --- .../check_version_compatibility.test.mocks.ts | 21 +++ .../utils/check_version_compatibility.test.ts | 110 ++++++++++++++ .../generate_additive_mapping_diff.test.ts | 139 ++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts new file mode 100644 index 0000000000000..a0e03552ed946 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.mocks.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getModelVersionsFromMappingsMock = jest.fn(); +export const compareModelVersionsMock = jest.fn(); +export const getModelVersionMapForTypesMock = jest.fn(); + +jest.doMock('@kbn/core-saved-objects-base-server-internal', () => { + const actual = jest.requireActual('@kbn/core-saved-objects-base-server-internal'); + return { + ...actual, + getModelVersionsFromMappings: getModelVersionsFromMappingsMock, + compareModelVersions: compareModelVersionsMock, + getModelVersionMapForTypes: getModelVersionMapForTypesMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts new file mode 100644 index 0000000000000..6ad12656229fc --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + compareModelVersionsMock, + getModelVersionsFromMappingsMock, + getModelVersionMapForTypesMock, +} from './check_version_compatibility.test.mocks'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import type { + IndexMapping, + ModelVersionMap, + CompareModelVersionResult, +} from '@kbn/core-saved-objects-base-server-internal'; +import { checkVersionCompatibility } from './check_version_compatibility'; +import { createType } from '../test_helpers'; + +describe('checkVersionCompatibility', () => { + const deletedTypes = ['some-deleted-type']; + + let types: SavedObjectsType[]; + let mappings: IndexMapping; + + beforeEach(() => { + compareModelVersionsMock.mockReset().mockReturnValue({}); + getModelVersionsFromMappingsMock.mockReset().mockReturnValue({}); + getModelVersionMapForTypesMock.mockReset().mockReturnValue({ status: 'equal' }); + + types = [createType({ name: 'foo' }), createType({ name: 'bar' })]; + + mappings = { + properties: { foo: { type: 'boolean' } }, + }; + }); + + it('calls getModelVersionMapForTypes with the correct parameters', () => { + checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(getModelVersionMapForTypesMock).toHaveBeenCalledTimes(1); + expect(getModelVersionMapForTypesMock).toHaveBeenCalledWith(types); + }); + + it('calls getModelVersionsFromMappings with the correct parameters', () => { + checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(getModelVersionsFromMappingsMock).toHaveBeenCalledTimes(1); + expect(getModelVersionsFromMappingsMock).toHaveBeenCalledWith({ + mappings, + source: 'mappingVersions', + }); + }); + + it('calls compareModelVersions with the correct parameters', () => { + const appVersions: ModelVersionMap = { foo: 2, bar: 2 }; + const indexVersions: ModelVersionMap = { foo: 1, bar: 1 }; + + getModelVersionMapForTypesMock.mockReturnValue(appVersions); + getModelVersionsFromMappingsMock.mockReturnValue(indexVersions); + + checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(compareModelVersionsMock).toHaveBeenCalledTimes(1); + expect(compareModelVersionsMock).toHaveBeenCalledWith({ + appVersions, + indexVersions, + deletedTypes, + }); + }); + + it('returns the result of the compareModelVersions call', () => { + const expected: CompareModelVersionResult = { + status: 'lesser', + details: { + greater: [], + lesser: [], + equal: [], + }, + }; + compareModelVersionsMock.mockReturnValue(expected); + + const result = checkVersionCompatibility({ + types, + mappings, + source: 'mappingVersions', + deletedTypes, + }); + + expect(result).toEqual(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts new file mode 100644 index 0000000000000..ce0cba20f427c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.test.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; +import { createType } from '../test_helpers'; + +describe('generateAdditiveMappingDiff', () => { + const deletedTypes = ['deletedType']; + + const getTypes = () => { + const foo = createType({ + name: 'foo', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { fooProp: { type: 'text' } } }, + }); + const bar = createType({ + name: 'bar', + modelVersions: { + 1: { modelChange: { type: 'expansion' } }, + 2: { modelChange: { type: 'expansion' } }, + }, + mappings: { properties: { barProp: { type: 'text' } } }, + }); + + return { foo, bar }; + }; + + it('aggregates the mappings of the types with versions higher than in the index', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + }, + }; + + const addedMappings = generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }); + + expect(addedMappings).toEqual({ + foo: foo.mappings, + bar: bar.mappings, + }); + }); + + it('ignores mapping from types already up to date', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 2, + }, + }; + + const addedMappings = generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }); + + expect(addedMappings).toEqual({ + foo: foo.mappings, + }); + }); + + it('ignores deleted types', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + deletedType: 42, + }, + }; + + const addedMappings = generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }); + + expect(addedMappings).toEqual({ + foo: foo.mappings, + bar: bar.mappings, + }); + }); + + it('throws an error in case of version conflict', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 3, + }, + }; + + expect(() => + generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }) + ).toThrowErrorMatchingInlineSnapshot( + `"Cannot generate model version difference: conflict between versions"` + ); + }); + + it('throws an error if mappingVersions is not present on the index meta', () => { + const { foo, bar } = getTypes(); + const types = [foo, bar]; + const meta: IndexMappingMeta = {}; + + expect(() => + generateAdditiveMappingDiff({ + types, + meta, + deletedTypes, + }) + ).toThrowErrorMatchingInlineSnapshot( + `"Cannot generate additive mapping diff: mappingVersions not present on index meta"` + ); + }); +}); From 5bb6732f4983e13a8f18ab509e7b73c51364164c Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 15:38:20 +0100 Subject: [PATCH 27/29] add integration test on mapping version mismatch --- .../mapping_version_conflict.test.ts | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts new file mode 100644 index 0000000000000..eadd24bf447ac --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; +import { LogRecord } from '@kbn/logging'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { + getKibanaMigratorTestKit, + type KibanaMigratorTestKitParams, +} from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; + +export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log'); + +describe('ZDT upgrades - mapping model version conflict', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + const baseMigratorParams: KibanaMigratorTestKitParams = { + kibanaIndex: '.kibana', + kibanaVersion: '8.7.0', + settings: { + migrations: { + algorithm: 'zdt', + }, + }, + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + + // increasing bar model version on the baseline + barType.modelVersions = { + ...barType.modelVersions, + '2': dummyModelVersion, + }; + barType.mappings.properties = { + ...barType.mappings.properties, + anotherAddedField: { type: 'boolean' }, + }; + const { runMigrations } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('fails the migration with an error', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + // increasing foo model version + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + fooType.mappings.properties = { + ...fooType.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + // we have the following versions: + // baseline : bar:2, foo: 2 + // upgrade: bar:3, foo:1 + // which should cause a migration failure. + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...baseMigratorParams, + logFilePath, + types: [fooType, barType], + }); + + await expect(runMigrations()).rejects.toThrowErrorMatchingInlineSnapshot( + `"Unable to complete saved object migrations for the [.kibana] index: Model version conflict: inconsistent higher/lower versions"` + ); + + const indices = await client.indices.get({ index: '.kibana*' }); + + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const aliases = Object.keys(index.aliases ?? {}).sort(); + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + + expect(mappingMeta.mappingVersions).toEqual({ + foo: 2, + bar: 2, + }); + + const logFileContent = await fs.readFile(logFilePath, 'utf-8'); + const records = logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; + + const expectLogsContains = (messagePrefix: string) => { + expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); + }; + + // + expectLogsContains('Mappings model version check result: conflict'); + expectLogsContains('INIT -> FATAL'); + }); +}); From c03377098bb03e1b74be05e41b08f1adc386fbeb Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Mon, 6 Mar 2023 16:32:13 +0100 Subject: [PATCH 28/29] more utility unit tests --- .../src/common/utils/delay.test.ts | 41 ++++++++++ .../src/common/utils/logs.test.ts | 80 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts new file mode 100644 index 0000000000000..3701621a96a12 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/delay.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createDelayFn } from './delay'; + +const nextTick = () => new Promise((resolve) => resolve()); + +describe('createDelayFn', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('adds a delay effect to the provided function', async () => { + const handler = jest.fn(); + + const wrapped = createDelayFn({ retryDelay: 2000, retryCount: 0 })(handler); + + wrapped(); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(500); + await nextTick(); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(1500); + await nextTick(); + + expect(handler).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts new file mode 100644 index 0000000000000..5615f18e0828d --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { omit } from 'lodash'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { logStateTransition, type LogAwareState } from './logs'; + +describe('logStateTransition', () => { + let logger: MockedLogger; + + const messagePrefix = '[PREFIX] '; + + beforeEach(() => { + logger = loggerMock.create(); + }); + + it('logs the offset of messages between the old and the new state', () => { + const previous: LogAwareState = { + controlState: 'PREVIOUS', + logs: [], + }; + const next: LogAwareState = { + controlState: 'NEXT', + logs: [ + ...previous.logs, + { level: 'info', message: 'info message' }, + { level: 'warning', message: 'warning message' }, + ], + }; + + logStateTransition(logger, messagePrefix, previous, next, 500); + + expect(omit(loggerMock.collect(logger), 'debug')).toEqual({ + error: [], + fatal: [], + info: [['[PREFIX] info message'], ['[PREFIX] PREVIOUS -> NEXT. took: 500ms.']], + log: [], + trace: [], + warn: [['[PREFIX] warning message']], + }); + }); + + it('logs a debug message with the ', () => { + const previous: LogAwareState = { + controlState: 'PREVIOUS', + logs: [], + }; + const next: LogAwareState = { + controlState: 'NEXT', + logs: [ + ...previous.logs, + { level: 'info', message: 'info message' }, + { level: 'warning', message: 'warning message' }, + ], + }; + + logStateTransition(logger, messagePrefix, previous, next, 500); + + expect(loggerMock.collect(logger).debug).toEqual([ + [ + '[PREFIX] PREVIOUS -> NEXT. took: 500ms.', + { + kibana: { + migrations: { + duration: 500, + state: expect.objectContaining({ + controlState: 'NEXT', + }), + }, + }, + }, + ], + ]); + }); +}); From 1d244dbe1fce4f8a515050c2686c4923ee4eaf3d Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 8 Mar 2023 10:24:12 +0100 Subject: [PATCH 29/29] review nits --- .../src/common/utils/logs.test.ts | 2 +- .../src/zdt/model/model.test.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts index 5615f18e0828d..e3bb8451b0ae9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/utils/logs.test.ts @@ -45,7 +45,7 @@ describe('logStateTransition', () => { }); }); - it('logs a debug message with the ', () => { + it('logs a debug message with the correct meta', () => { const previous: LogAwareState = { controlState: 'PREVIOUS', logs: [], diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts index e9343d719b301..c0316b954e5f3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts @@ -12,7 +12,6 @@ import { createContextMock, MockedMigratorContext } from '../test_helpers'; import type { RetryableEsClientError } from '../../actions'; import type { State, BaseState, FatalState, AllActionStates } from '../state'; import type { StateActionResponse } from './types'; -import type { ResponseType } from '../next'; import { model } from './model'; describe('model', () => { @@ -120,14 +119,14 @@ describe('model', () => { controlState, } as unknown as State); - const createStubResponse = (): ResponseType => + const createStubResponse = () => Either.right({ '.kibana_7.11.0_001': { aliases: {}, mappings: { properties: {} }, settings: {}, }, - }) as StateActionResponse<'INIT'>; + }); const stageMapping: Record = { INIT: StageMocks.init,