From 1bca7a346f3daab1733c7310e1d88d2edfc47b5f Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Thu, 27 Apr 2023 11:23:46 +0200 Subject: [PATCH] start adapting the version utils --- .../index.ts | 10 +- .../src/mappings/index.ts | 2 + .../src/mappings/types.ts | 17 +- .../src/model_version/conversion.test.ts | 17 ++ .../src/model_version/conversion.ts | 8 + .../model_version/get_version_delta.test.ts | 58 ++--- .../src/model_version/get_version_delta.ts | 18 +- .../src/model_version/index.ts | 10 +- .../model_version_from_mappings.test.ts | 34 +-- .../model_version_from_mappings.ts | 18 +- .../src/model_version/version_compare.test.ts | 84 +++---- .../src/model_version/version_compare.ts | 22 +- .../src/model_version/version_map.test.ts | 147 +++++++++++- .../src/model_version/version_map.ts | 39 +++- .../src/zdt/context/create_context.ts | 4 +- .../src/zdt/context/types.ts | 6 +- .../zdt/utils/check_version_compatibility.ts | 12 +- .../utils/generate_additive_mapping_diff.ts | 8 +- .../utils/outdated_documents_query.test.ts | 217 ++++++++++++------ .../src/zdt/utils/outdated_documents_query.ts | 34 +-- 20 files changed, 527 insertions(+), 238 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 981783bb05fd5..759ede0401feb 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -49,13 +49,17 @@ export { modelVersionToVirtualVersion, getModelVersionMapForTypes, getLatestModelVersion, + getCurrentVirtualVersion, + getLatestMigrationVersion, + getVirtualVersionMap, type ModelVersionMap, - compareModelVersions, + type VirtualVersionMap, + compareVirtualVersions, type CompareModelVersionMapParams, type CompareModelVersionStatus, type CompareModelVersionDetails, type CompareModelVersionResult, - getModelVersionsFromMappings, - getModelVersionsFromMappingMeta, + getVirtualVersionsFromMappings, + getVirtualVersionsFromMappingMeta, getModelVersionDelta, } from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts index ee51fa9aaf4bb..779ff28fb077c 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts @@ -9,6 +9,8 @@ export { getTypes, getProperty, getRootProperties, getRootPropertiesObjects } from './lib'; export type { SavedObjectsTypeMappingDefinitions, + V2AlgoIndexMappingMeta, + ZdtAlgoIndexMappingMeta, IndexMappingMeta, IndexMapping, IndexTypesMap, diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index c756f0534db67..a8c74646de1b1 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -59,7 +59,7 @@ export interface IndexMapping { export type IndexTypesMap = Record; /** @internal */ -export interface IndexMappingMeta { +export interface V2AlgoIndexMappingMeta { /** * A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa') * with each key being a root-level mapping property, and each value being @@ -74,18 +74,22 @@ export interface IndexMappingMeta { * @remark: Only defined for indices using the v2 migration algorithm. */ indexTypesMap?: IndexTypesMap; +} + +/** @internal */ +export interface ZdtAlgoIndexMappingMeta { /** - * The current model versions of the mapping of the index. + * The current virtual version of the mapping of the index. * * @remark: Only defined for indices using the zdt migration algorithm. */ - mappingVersions?: { [k: string]: number }; + mappingVersions?: { [k: string]: string }; /** - * The current model versions of the documents of the index. + * The current virtual versions of the documents of the index. * * @remark: Only defined for indices using the zdt migration algorithm. */ - docVersions?: { [k: string]: number }; + docVersions?: { [k: string]: string }; /** * Info about the current state of the migration. * Should only be present if a migration is in progress or was interrupted. @@ -95,6 +99,9 @@ export interface IndexMappingMeta { migrationState?: IndexMappingMigrationStateMeta; } +/** @internal */ +export type IndexMappingMeta = V2AlgoIndexMappingMeta & ZdtAlgoIndexMappingMeta; + /** @internal */ export interface IndexMappingMigrationStateMeta { /** diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts index b65bdbce41b7a..3c89894058f57 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts @@ -11,6 +11,7 @@ import { virtualVersionToModelVersion, modelVersionToVirtualVersion, assertValidModelVersion, + assertValidVirtualVersion, } from './conversion'; describe('isVirtualModelVersion', () => { @@ -103,3 +104,19 @@ describe('assertValidModelVersion', () => { expect(assertValidModelVersion('3')).toEqual(3); }); }); + +describe('assertValidVirtualVersion', () => { + it('throws if the provided value is not a valid semver', () => { + expect(() => assertValidVirtualVersion('foooo')).toThrowErrorMatchingInlineSnapshot( + `"Virtual versions must be valid semver versions"` + ); + expect(() => assertValidVirtualVersion('1.2.3.4.5.6.7')).toThrowErrorMatchingInlineSnapshot( + `"Virtual versions must be valid semver versions"` + ); + }); + + it('returns the virtual version', () => { + expect(assertValidVirtualVersion('7.17.5')).toEqual('7.17.5'); + expect(assertValidVirtualVersion('10.3.0')).toEqual('10.3.0'); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts index c3765ca3c9be9..b10b85084eca5 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts @@ -89,6 +89,14 @@ export const assertValidModelVersion = (modelVersion: string | number): number = return modelVersion; }; +export const assertValidVirtualVersion = (virtualVersion: string): string => { + const semver = Semver.parse(virtualVersion); + if (!semver) { + throw new Error('Virtual versions must be valid semver versions'); + } + return virtualVersion; +}; + const _isVirtualModelVersion = (semver: Semver.SemVer): boolean => { return semver.major === modelVersionVirtualMajor && semver.patch === 0; }; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts index 521ddd2a0efc3..4c1bbeaf7f940 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts @@ -12,12 +12,12 @@ describe('getModelVersionDelta', () => { it('generates an upward delta', () => { const result = getModelVersionDelta({ currentVersions: { - a: 1, - b: 1, + a: '10.1.0', + b: '10.1.0', }, targetVersions: { - a: 2, - b: 3, + a: '10.2.0', + b: '10.3.0', }, deletedTypes: [], }); @@ -26,13 +26,13 @@ describe('getModelVersionDelta', () => { expect(result.diff).toEqual([ { name: 'a', - current: 1, - target: 2, + current: '10.1.0', + target: '10.2.0', }, { name: 'b', - current: 1, - target: 3, + current: '10.1.0', + target: '10.3.0', }, ]); }); @@ -40,12 +40,12 @@ describe('getModelVersionDelta', () => { it('generates a downward delta', () => { const result = getModelVersionDelta({ currentVersions: { - a: 4, - b: 2, + a: '10.4.0', + b: '10.2.0', }, targetVersions: { - a: 1, - b: 1, + a: '10.1.0', + b: '7.17.2', }, deletedTypes: [], }); @@ -54,13 +54,13 @@ describe('getModelVersionDelta', () => { expect(result.diff).toEqual([ { name: 'a', - current: 4, - target: 1, + current: '10.4.0', + target: '10.1.0', }, { name: 'b', - current: 2, - target: 1, + current: '10.2.0', + target: '7.17.2', }, ]); }); @@ -68,12 +68,12 @@ describe('getModelVersionDelta', () => { it('generates a noop delta', () => { const result = getModelVersionDelta({ currentVersions: { - a: 4, - b: 2, + a: '10.4.0', + b: '8.9.2', }, targetVersions: { - a: 4, - b: 2, + a: '10.4.0', + b: '8.9.2', }, deletedTypes: [], }); @@ -85,11 +85,11 @@ describe('getModelVersionDelta', () => { it('ignores deleted types', () => { const result = getModelVersionDelta({ currentVersions: { - a: 1, - b: 3, + a: '10.1.0', + b: '10.3.0', }, targetVersions: { - a: 2, + a: '10.2.0', }, deletedTypes: ['b'], }); @@ -98,8 +98,8 @@ describe('getModelVersionDelta', () => { expect(result.diff).toEqual([ { name: 'a', - current: 1, - target: 2, + current: '10.1.0', + target: '10.2.0', }, ]); }); @@ -108,12 +108,12 @@ describe('getModelVersionDelta', () => { expect(() => getModelVersionDelta({ currentVersions: { - a: 1, - b: 2, + a: '10.1.0', + b: '10.2.0', }, targetVersions: { - a: 2, - b: 1, + a: '10.2.0', + b: '10.1.0', }, deletedTypes: [], }) diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts index f39c52b47f9f7..2426bc4a5770f 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts @@ -6,12 +6,12 @@ * Side Public License, v 1. */ -import type { ModelVersionMap } from './version_map'; -import { compareModelVersions } from './version_compare'; +import type { VirtualVersionMap, VirtualVersion } from './version_map'; +import { compareVirtualVersions } from './version_compare'; interface GetModelVersionDeltaOpts { - currentVersions: ModelVersionMap; - targetVersions: ModelVersionMap; + currentVersions: VirtualVersionMap; + targetVersions: VirtualVersionMap; deletedTypes: string[]; } @@ -26,9 +26,9 @@ interface ModelVersionDeltaTypeResult { /** the name of the type */ name: string; /** the current version the type is at */ - current: number; + current: VirtualVersion; /** the target version the type should go to */ - target: number; + target: VirtualVersion; } /** @@ -41,7 +41,7 @@ export const getModelVersionDelta = ({ targetVersions, deletedTypes, }: GetModelVersionDeltaOpts): ModelVersionDeltaResult => { - const compared = compareModelVersions({ + const compared = compareVirtualVersions({ indexVersions: currentVersions, appVersions: targetVersions, deletedTypes, @@ -78,8 +78,8 @@ const getTypeDelta = ({ targetVersions, }: { type: string; - currentVersions: ModelVersionMap; - targetVersions: ModelVersionMap; + currentVersions: VirtualVersionMap; + targetVersions: VirtualVersionMap; }): ModelVersionDeltaTypeResult => { const currentVersion = currentVersions[type]; const targetVersion = targetVersions[type]; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 2179199921a82..7279470d26b8f 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -16,17 +16,21 @@ export { export { getModelVersionMapForTypes, getLatestModelVersion, + getCurrentVirtualVersion, + getVirtualVersionMap, + getLatestMigrationVersion, type ModelVersionMap, + type VirtualVersionMap, } from './version_map'; export { - compareModelVersions, + compareVirtualVersions, type CompareModelVersionMapParams, type CompareModelVersionStatus, type CompareModelVersionDetails, type CompareModelVersionResult, } from './version_compare'; export { - getModelVersionsFromMappings, - getModelVersionsFromMappingMeta, + getVirtualVersionsFromMappings, + getVirtualVersionsFromMappingMeta, } from './model_version_from_mappings'; export { getModelVersionDelta } from './get_version_delta'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts index 8fea10f11f6b1..ec8d36a53fa9d 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts @@ -7,7 +7,7 @@ */ import type { IndexMapping, IndexMappingMeta } from '../mappings'; -import { getModelVersionsFromMappings } from './model_version_from_mappings'; +import { getVirtualVersionsFromMappings } from './model_version_from_mappings'; describe('getModelVersionsFromMappings', () => { const createIndexMapping = (parts: Partial = {}): IndexMapping => ({ @@ -20,41 +20,41 @@ describe('getModelVersionsFromMappings', () => { it('retrieves the version map from docVersions', () => { const mappings = createIndexMapping({ docVersions: { - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '8.16.2', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'docVersions' }); expect(versionMap).toEqual({ - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '8.16.2', }); }); it('retrieves the version map from mappingVersions', () => { const mappings = createIndexMapping({ mappingVersions: { - foo: 2, - bar: 7, + foo: '10.2.0', + bar: '7.17.0', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'mappingVersions' }); expect(versionMap).toEqual({ - foo: 2, - bar: 7, + foo: '10.2.0', + bar: '7.17.0', }); }); it('returns undefined for docVersions if meta field is not present', () => { const mappings = createIndexMapping({ mappingVersions: { - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '10.5.0', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'docVersions' }); expect(versionMap).toBeUndefined(); }); @@ -62,11 +62,11 @@ describe('getModelVersionsFromMappings', () => { it('returns undefined for mappingVersions if meta field is not present', () => { const mappings = createIndexMapping({ docVersions: { - foo: 3, - bar: 5, + foo: '10.3.0', + bar: '10.5.0', }, }); - const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' }); + const versionMap = getVirtualVersionsFromMappings({ mappings, source: 'mappingVersions' }); expect(versionMap).toBeUndefined(); }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts index 01fc57d46462b..8dc28969d8476 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -7,8 +7,8 @@ */ import type { IndexMapping, IndexMappingMeta } from '../mappings'; -import type { ModelVersionMap } from './version_map'; -import { assertValidModelVersion } from './conversion'; +import type { VirtualVersionMap } from './version_map'; +import { assertValidVirtualVersion } from './conversion'; export interface GetModelVersionsFromMappingsOpts { mappings: IndexMapping; @@ -20,16 +20,16 @@ export interface GetModelVersionsFromMappingsOpts { /** * Build the version map from the specified source of the provided mappings. */ -export const getModelVersionsFromMappings = ({ +export const getVirtualVersionsFromMappings = ({ mappings, source, knownTypes, -}: GetModelVersionsFromMappingsOpts): ModelVersionMap | undefined => { +}: GetModelVersionsFromMappingsOpts): VirtualVersionMap | undefined => { if (!mappings._meta) { return undefined; } - return getModelVersionsFromMappingMeta({ + return getVirtualVersionsFromMappingMeta({ meta: mappings._meta, source, knownTypes, @@ -46,20 +46,20 @@ export interface GetModelVersionsFromMappingMetaOpts { /** * Build the version map from the specified source of the provided mappings meta. */ -export const getModelVersionsFromMappingMeta = ({ +export const getVirtualVersionsFromMappingMeta = ({ meta, source, knownTypes, -}: GetModelVersionsFromMappingMetaOpts): ModelVersionMap | undefined => { +}: GetModelVersionsFromMappingMetaOpts): VirtualVersionMap | undefined => { const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; if (!indexVersions) { return undefined; } const typeSet = knownTypes ? new Set(knownTypes) : undefined; - return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { if (!typeSet || typeSet.has(type)) { - map[type] = assertValidModelVersion(rawVersion); + map[type] = assertValidVirtualVersion(rawVersion); } return map; }, {}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts index eba6fe1837cce..64feb07455eb3 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.test.ts @@ -6,18 +6,18 @@ * Side Public License, v 1. */ -import { compareModelVersions } from './version_compare'; +import { compareVirtualVersions } from './version_compare'; describe('compareModelVersions', () => { it('returns the correct value for greater app version', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 3, - bar: 2, + foo: '10.3.0', + bar: '10.2.0', }, indexVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, deletedTypes: [], }); @@ -26,14 +26,14 @@ describe('compareModelVersions', () => { }); it('returns the correct value for lesser app version', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 1, - bar: 2, + foo: '10.1.0', + bar: '10.2.0', }, indexVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, deletedTypes: [], }); @@ -42,14 +42,14 @@ describe('compareModelVersions', () => { }); it('returns the correct value for equal versions', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, indexVersions: { - foo: 2, - bar: 2, + foo: '10.2.0', + bar: '10.2.0', }, deletedTypes: [], }); @@ -58,13 +58,13 @@ describe('compareModelVersions', () => { }); it('handles new types not being present in the index', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 2, - new: 1, + foo: '10.2.0', + bar: '10.1.0', }, indexVersions: { - foo: 2, + foo: '10.2.0', }, deletedTypes: [], }); @@ -73,13 +73,13 @@ describe('compareModelVersions', () => { }); it('handles types not being present in the app', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - foo: 3, + foo: '10.3.0', }, indexVersions: { - foo: 2, - old: 1, + foo: '10.2.0', + old: '10.1.0', }, deletedTypes: [], }); @@ -88,16 +88,16 @@ describe('compareModelVersions', () => { }); it('returns the correct value for conflicts', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - a: 3, - b: 3, - c: 3, + a: '10.3.0', + b: '10.3.0', + c: '10.3.0', }, indexVersions: { - a: 2, - b: 3, - c: 4, + a: '10.2.0', + b: '10.3.0', + c: '10.4.0', }, deletedTypes: [], }); @@ -106,16 +106,16 @@ describe('compareModelVersions', () => { }); it('properly lists the details', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - a: 3, - b: 3, - c: 3, + a: '10.3.0', + b: '10.3.0', + c: '10.3.0', }, indexVersions: { - a: 2, - b: 3, - c: 4, + a: '10.2.0', + b: '10.3.0', + c: '10.4.0', }, deletedTypes: [], }); @@ -126,13 +126,13 @@ describe('compareModelVersions', () => { }); it('ignores deleted types when comparing', () => { - const result = compareModelVersions({ + const result = compareVirtualVersions({ appVersions: { - a: 3, + a: '10.3.0', }, indexVersions: { - a: 2, - b: 3, + a: '10.2.0', + b: '10.3.0', }, deletedTypes: ['b'], }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts index 9b8d14b7fd862..6d250167cb4e2 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_compare.ts @@ -6,13 +6,14 @@ * Side Public License, v 1. */ -import type { ModelVersionMap } from './version_map'; +import Semver from 'semver'; +import type { VirtualVersionMap } from './version_map'; export interface CompareModelVersionMapParams { /** The latest model version of the types registered in the application */ - appVersions: ModelVersionMap; + appVersions: VirtualVersionMap; /** The model version stored in the index */ - indexVersions: ModelVersionMap; + indexVersions: VirtualVersionMap; /** The list of deleted types to exclude during the compare process */ deletedTypes: string[]; } @@ -37,7 +38,7 @@ export interface CompareModelVersionResult { details: CompareModelVersionDetails; } -export const compareModelVersions = ({ +export const compareVirtualVersions = ({ appVersions, indexVersions, deletedTypes, @@ -53,14 +54,19 @@ export const compareModelVersions = ({ }; allTypes.forEach((type) => { - const appVersion = appVersions[type] ?? 0; - const indexVersion = indexVersions[type] ?? 0; + const appVersion = appVersions[type] ?? '0.0.0'; + const indexVersion = indexVersions[type] ?? '0.0.0'; - if (appVersion > indexVersion) { + const comparison = Semver.compare(appVersion, indexVersion); + + if (comparison > 0) { + // app version greater than index version details.greater.push(type); - } else if (appVersion < indexVersion) { + } else if (comparison < 0) { + // app version lower than index version details.lesser.push(type); } else { + // // app version equal to index version details.equal.push(type); } }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts index aafb83ab96009..1849809c516f9 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts @@ -7,7 +7,13 @@ */ import type { SavedObjectsType, SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; -import { getModelVersionMapForTypes, getLatestModelVersion } from './version_map'; +import { + getModelVersionMapForTypes, + getLatestModelVersion, + getLatestMigrationVersion, + getCurrentVirtualVersion, + getVirtualVersionMap, +} from './version_map'; describe('ModelVersion map utilities', () => { const buildType = (parts: Partial = {}): SavedObjectsType => ({ @@ -24,6 +30,8 @@ describe('ModelVersion map utilities', () => { }, }); + const dummyMigration = jest.fn(); + describe('getLatestModelVersion', () => { it('returns 0 when no model versions are registered', () => { expect(getLatestModelVersion(buildType({ modelVersions: {} }))).toEqual(0); @@ -116,4 +124,141 @@ describe('ModelVersion map utilities', () => { }); }); }); + + describe('getLatestMigrationVersion', () => { + it('returns 0.0.0 when no migrations are registered', () => { + expect(getLatestMigrationVersion(buildType({ migrations: {} }))).toEqual('0.0.0'); + expect(getLatestMigrationVersion(buildType({ migrations: undefined }))).toEqual('0.0.0'); + }); + + it('throws if an invalid version is provided', () => { + expect(() => + getLatestMigrationVersion( + buildType({ + migrations: { + foo: dummyMigration, + '8.6.0': dummyMigration, + }, + }) + ) + ).toThrowError(); + }); + + it('returns the latest registered version', () => { + expect( + getLatestMigrationVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + }) + ) + ).toEqual('8.6.0'); + }); + + it('accepts provider functions', () => { + expect( + getLatestMigrationVersion( + buildType({ + migrations: () => ({ + '7.17.2': dummyMigration, + '8.4.0': dummyMigration, + }), + }) + ) + ).toEqual('8.4.0'); + }); + + it('supports unordered maps', () => { + expect( + getLatestMigrationVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.7.0': dummyMigration, + '8.2.0': dummyMigration, + }, + }) + ) + ).toEqual('8.7.0'); + }); + }); + + describe('getCurrentVirtualVersion', () => { + it('returns the latest registered migration if switchToModelVersionAt is unset', () => { + expect( + getCurrentVirtualVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }) + ) + ).toEqual('8.6.0'); + }); + + it('returns the virtual version of the latest model version if switchToModelVersionAt is set', () => { + expect( + getCurrentVirtualVersion( + buildType({ + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }) + ) + ).toEqual('10.1.0'); + }); + }); + + describe('getVirtualVersionMap', () => { + it('returns the virtual version for each of the provided types', () => { + expect( + getVirtualVersionMap([ + buildType({ + name: 'foo', + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }), + buildType({ + name: 'bar', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersion(), + }, + }), + buildType({ + name: 'dolly', + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + }), + ]) + ).toEqual({ + foo: '10.1.0', + bar: '8.6.0', + dolly: '10.0.0', + }); + }); + }); }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts index dd05e64dbcbef..e4d2a86077065 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -6,10 +6,13 @@ * Side Public License, v 1. */ +import Semver from 'semver'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { assertValidModelVersion } from './conversion'; +import { assertValidModelVersion, modelVersionToVirtualVersion } from './conversion'; export type ModelVersionMap = Record; +export type VirtualVersion = string; +export type VirtualVersionMap = Record; /** * Returns the latest registered model version number for the given type. @@ -22,6 +25,14 @@ export const getLatestModelVersion = (type: SavedObjectsType): number => { }, 0); }; +export const getLatestMigrationVersion = (type: SavedObjectsType): string => { + const migrationMap = + typeof type.migrations === 'function' ? type.migrations() : type.migrations ?? {}; + return Object.keys(migrationMap).reduce((memo, current) => { + return Semver.gt(memo, current) ? memo : current; + }, '0.0.0'); +}; + /** * Build a version map for the given types. */ @@ -31,3 +42,29 @@ export const getModelVersionMapForTypes = (types: SavedObjectsType[]): ModelVers return versionMap; }, {}); }; + +/** + * Returns the current virtual version for the given type. + * If will either be the latest model version if the type + * already switched to using them (switchToModelVersionAt is set), + * or the latest migration version for the type otherwise. + */ +export const getCurrentVirtualVersion = (type: SavedObjectsType): string => { + if (type.switchToModelVersionAt) { + const modelVersion = getLatestModelVersion(type); + return modelVersionToVirtualVersion(modelVersion); + } else { + return getLatestMigrationVersion(type); + } +}; + +/** + * Returns a map of virtual model version for the given types. + * See {@link getCurrentVirtualVersion} + */ +export const getVirtualVersionMap = (types: SavedObjectsType[]): VirtualVersionMap => { + return types.reduce((versionMap, type) => { + versionMap[type.name] = getCurrentVirtualVersion(type); + return versionMap; + }, {}); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index c31d4bc799b3b..82234823d973a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { getModelVersionMapForTypes } from '@kbn/core-saved-objects-base-server-internal'; +import { getVirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; import { REMOVED_TYPES } from '../../core'; import type { MigrateIndexOptions } from '../migrate_index'; import type { MigratorContext } from './types'; @@ -33,7 +33,7 @@ export const createContext = ({ kibanaVersion, indexPrefix, types, - typeModelVersions: getModelVersionMapForTypes(types.map((type) => typeRegistry.getType(type)!)), + typeVirtualVersions: getVirtualVersionMap(types.map((type) => typeRegistry.getType(type)!)), elasticsearchClient, typeRegistry, serializer, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 95ca7282daf57..58887418eb35a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -12,7 +12,7 @@ import type { ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; import type { - ModelVersionMap, + VirtualVersionMap, SavedObjectsMigrationConfigType, } from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; @@ -30,8 +30,8 @@ export interface MigratorContext { readonly indexPrefix: string; /** Name of the types that are living in the index */ readonly types: string[]; - /** Model versions for the registered types */ - readonly typeModelVersions: ModelVersionMap; + /** Virtual versions for the registered types */ + readonly typeVirtualVersions: VirtualVersionMap; /** The client to use for communications with ES */ readonly elasticsearchClient: ElasticsearchClient; /** The maximum number of retries to attempt for a failing action */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts index c231645fb7993..651875e92c438 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -8,9 +8,9 @@ import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; import { - getModelVersionsFromMappings, - compareModelVersions, - getModelVersionMapForTypes, + getVirtualVersionsFromMappings, + compareVirtualVersions, + getVirtualVersionMap, type IndexMapping, type CompareModelVersionResult, } from '@kbn/core-saved-objects-base-server-internal'; @@ -28,8 +28,8 @@ export const checkVersionCompatibility = ({ source, deletedTypes, }: CheckVersionCompatibilityOpts): CompareModelVersionResult => { - const appVersions = getModelVersionMapForTypes(types); - const indexVersions = getModelVersionsFromMappings({ + const appVersions = getVirtualVersionMap(types); + const indexVersions = getVirtualVersionsFromMappings({ mappings, source, knownTypes: types.map((type) => type.name), @@ -37,5 +37,5 @@ export const checkVersionCompatibility = ({ if (!indexVersions) { throw new Error(`Cannot check version: ${source} not present in the mapping meta`); } - return compareModelVersions({ appVersions, indexVersions, deletedTypes }); + return compareVirtualVersions({ appVersions, indexVersions, deletedTypes }); }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts index 400e01e999797..0ef7437b2d6c3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -12,8 +12,8 @@ import type { } from '@kbn/core-saved-objects-server'; import { IndexMappingMeta, - getModelVersionsFromMappingMeta, - getModelVersionMapForTypes, + getVirtualVersionsFromMappingMeta, + getVirtualVersionMap, getModelVersionDelta, } from '@kbn/core-saved-objects-base-server-internal'; @@ -35,8 +35,8 @@ export const generateAdditiveMappingDiff = ({ meta, deletedTypes, }: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { - const typeVersions = getModelVersionMapForTypes(types); - const mappingVersion = getModelVersionsFromMappingMeta({ + const typeVersions = getVirtualVersionMap(types); + const mappingVersion = getVirtualVersionsFromMappingMeta({ meta, source: 'mappingVersions', knownTypes: types.map((type) => type.name), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts index f39016b7d86a1..fb72061735eff 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts @@ -16,10 +16,13 @@ const dummyModelVersion: SavedObjectsModelVersion = { }, }; +const dummyMigration = jest.fn(); + describe('getOutdatedDocumentsQuery', () => { - it('generates the correct query', () => { + it('generates the correct query for types using model versions', () => { const fooType = createType({ name: 'foo', + switchToModelVersionAt: '8.8.0', modelVersions: { 1: dummyModelVersion, 2: dummyModelVersion, @@ -27,6 +30,7 @@ describe('getOutdatedDocumentsQuery', () => { }); const barType = createType({ name: 'bar', + switchToModelVersionAt: '8.8.0', modelVersions: { 1: dummyModelVersion, 2: dummyModelVersion, @@ -50,40 +54,11 @@ describe('getOutdatedDocumentsQuery', () => { "type": "foo", }, }, + ], + "must_not": Array [ Object { - "bool": Object { - "should": Array [ - Object { - "bool": Object { - "must": Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - "must_not": Object { - "term": Object { - "migrationVersion.foo": "10.2.0", - }, - }, - }, - }, - Object { - "bool": Object { - "must_not": Array [ - Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - Object { - "term": Object { - "typeMigrationVersion": "10.2.0", - }, - }, - ], - }, - }, - ], + "term": Object { + "typeMigrationVersion": "10.2.0", }, }, ], @@ -97,40 +72,148 @@ describe('getOutdatedDocumentsQuery', () => { "type": "bar", }, }, + ], + "must_not": Array [ Object { - "bool": Object { - "should": Array [ - Object { - "bool": Object { - "must": Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - "must_not": Object { - "term": Object { - "migrationVersion.bar": "10.3.0", - }, - }, - }, - }, - Object { - "bool": Object { - "must_not": Array [ - Object { - "exists": Object { - "field": "migrationVersion", - }, - }, - Object { - "term": Object { - "typeMigrationVersion": "10.3.0", - }, - }, - ], - }, - }, - ], + "term": Object { + "typeMigrationVersion": "10.3.0", + }, + }, + ], + }, + }, + ], + }, + } + `); + }); + + it('generates the correct query for types still using old migrations', () => { + const fooType = createType({ + name: 'foo', + migrations: { + '7.17.2': dummyMigration, + '8.5.0': dummyMigration, + }, + }); + const barType = createType({ + name: 'bar', + migrations: () => ({ + '7.15.5': dummyMigration, + '8.7.2': dummyMigration, + }), + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "8.5.0", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "8.7.2", + }, + }, + ], + }, + }, + ], + }, + } + `); + }); + + it('generates the correct query for mixed types', () => { + const fooType = createType({ + name: 'foo', + migrations: { + '7.17.2': dummyMigration, + '8.5.0': dummyMigration, + }, + switchToModelVersionAt: '8.8.0', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + }, + }); + const barType = createType({ + name: 'bar', + migrations: () => ({ + '7.15.5': dummyMigration, + '8.7.2': dummyMigration, + }), + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "10.2.0", + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + ], + "must_not": Array [ + Object { + "term": Object { + "typeMigrationVersion": "8.7.2", }, }, ], diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts index e15a2e447b554..40495654e6146 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts @@ -8,10 +8,7 @@ import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; -import { - getModelVersionMapForTypes, - modelVersionToVirtualVersion, -} from '@kbn/core-saved-objects-base-server-internal'; +import { getVirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; interface GetOutdatedDocumentsQueryOps { types: SavedObjectsType[]; @@ -23,36 +20,15 @@ export const getOutdatedDocumentsQuery = ({ // Note: in theory, we could check the difference of model version with the index's // and narrow the search filter only on the type that have different versions. // however, it feels safer to just search for all outdated document, just in case. - const modelVersions = getModelVersionMapForTypes(types); + const virtualVersions = getVirtualVersionMap(types); return { bool: { should: types.map((type) => { - const virtualVersion = modelVersionToVirtualVersion(modelVersions[type.name]); + const virtualVersion = virtualVersions[type.name]; return { bool: { - must: [ - { term: { type: type.name } }, - { - bool: { - should: [ - { - bool: { - must: { exists: { field: 'migrationVersion' } }, - must_not: { term: { [`migrationVersion.${type.name}`]: virtualVersion } }, - }, - }, - { - bool: { - must_not: [ - { exists: { field: 'migrationVersion' } }, - { term: { typeMigrationVersion: virtualVersion } }, - ], - }, - }, - ], - }, - }, - ], + must: [{ term: { type: type.name } }], + must_not: [{ term: { typeMigrationVersion: virtualVersion } }], }, }; }),