From dc8ab4de576e110cf7b38c69ac9efc95cb0b9172 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Wed, 15 Feb 2023 09:36:18 +0100 Subject: [PATCH] register model version migration to the document migrator (#150842) ## Summary Part of https://github.com/elastic/kibana/issues/150301 - Add logic to converts model version transformations to the format used by the document migrator - Use in when preparing migration data for the document migrator - Improve the migration validation logic to take model versions into account (and clean it) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../index.ts | 7 + .../src/model_version/constants.ts | 12 + .../src/model_version/conversion.test.ts | 105 ++++++ .../src/model_version/conversion.ts | 94 +++++ .../src/model_version/index.ts | 15 + .../build_active_migrations.test.mocks.ts | 34 ++ .../build_active_migrations.test.ts | 322 ++++++++++++++++++ .../build_active_migrations.ts | 98 ++++-- .../document_migrator.test.mock.ts | 12 +- .../document_migrator.test.ts | 175 +++------- .../document_migrator/document_migrator.ts | 17 +- .../document_migrator/model_version.test.ts | 187 ++++++++++ .../src/document_migrator/model_version.ts | 84 +++++ .../src/document_migrator/types.ts | 4 +- .../src/document_migrator/utils.ts | 2 +- .../validate_migration.test.ts | 272 +++++++++++++++ .../document_migrator/validate_migrations.ts | 240 ++++++++----- .../tsconfig.json | 1 + .../src/model_version/transformations.ts | 4 +- 19 files changed, 1417 insertions(+), 268 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/constants.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts create mode 100644 packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.mocks.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index b5a85a13121f1..722694a32f6ad 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -39,3 +39,10 @@ export type { MigrationStatus, } from './src/migration'; export { parseObjectKey, getObjectKey, getIndexForType } from './src/utils'; +export { + modelVersionVirtualMajor, + assertValidModelVersion, + isVirtualModelVersion, + virtualVersionToModelVersion, + modelVersionToVirtualVersion, +} from './src/model_version'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/constants.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/constants.ts new file mode 100644 index 0000000000000..f3f8ace3a142b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/constants.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * The major version that is used to represent model versions. + */ +export const modelVersionVirtualMajor = 10; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts new file mode 100644 index 0000000000000..b65bdbce41b7a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + isVirtualModelVersion, + virtualVersionToModelVersion, + modelVersionToVirtualVersion, + assertValidModelVersion, +} from './conversion'; + +describe('isVirtualModelVersion', () => { + it('returns true when the version is a virtual model version', () => { + expect(isVirtualModelVersion('10.0.0')).toEqual(true); + expect(isVirtualModelVersion('10.7.0')).toEqual(true); + expect(isVirtualModelVersion('10.12.0')).toEqual(true); + }); + + it('returns false when the version is not a virtual model version', () => { + expect(isVirtualModelVersion('9.2.0')).toEqual(false); + expect(isVirtualModelVersion('10.7.1')).toEqual(false); + expect(isVirtualModelVersion('11.2.0')).toEqual(false); + }); + + it('throws when the version is not a valid semver', () => { + expect(() => isVirtualModelVersion('9.-2.0')).toThrowErrorMatchingInlineSnapshot( + `"Invalid semver: 9.-2.0"` + ); + expect(() => isVirtualModelVersion('12.3.5.6.7')).toThrowErrorMatchingInlineSnapshot( + `"Invalid semver: 12.3.5.6.7"` + ); + expect(() => isVirtualModelVersion('dolly')).toThrowErrorMatchingInlineSnapshot( + `"Invalid semver: dolly"` + ); + }); +}); + +describe('virtualVersionToModelVersion', () => { + it('converts the given virtual version to its model version', () => { + expect(virtualVersionToModelVersion('10.0.0')).toEqual(0); + expect(virtualVersionToModelVersion('10.7.0')).toEqual(7); + expect(virtualVersionToModelVersion('10.12.0')).toEqual(12); + }); + + it('throws when the version is not a virtual model version', () => { + expect(() => virtualVersionToModelVersion('9.2.0')).toThrowErrorMatchingInlineSnapshot( + `"Version is not a virtual model version"` + ); + expect(() => virtualVersionToModelVersion('11.3.0')).toThrowErrorMatchingInlineSnapshot( + `"Version is not a virtual model version"` + ); + expect(() => virtualVersionToModelVersion('10.3.42')).toThrowErrorMatchingInlineSnapshot( + `"Version is not a virtual model version"` + ); + }); + + it('throws when the version is not a valid semver', () => { + expect(() => virtualVersionToModelVersion('9.-2.0')).toThrowErrorMatchingInlineSnapshot( + `"Invalid semver: 9.-2.0"` + ); + expect(() => virtualVersionToModelVersion('12.3.5.6.7')).toThrowErrorMatchingInlineSnapshot( + `"Invalid semver: 12.3.5.6.7"` + ); + expect(() => virtualVersionToModelVersion('dolly')).toThrowErrorMatchingInlineSnapshot( + `"Invalid semver: dolly"` + ); + }); +}); + +describe('modelVersionToVirtualVersion', () => { + it('converts the given model version to its virtual version', () => { + expect(modelVersionToVirtualVersion(0)).toEqual('10.0.0'); + expect(modelVersionToVirtualVersion(7)).toEqual('10.7.0'); + expect(modelVersionToVirtualVersion(12)).toEqual('10.12.0'); + }); +}); + +describe('assertValidModelVersion', () => { + it('throws if the provided value is not an integer', () => { + expect(() => assertValidModelVersion(9.4)).toThrowErrorMatchingInlineSnapshot( + `"Model version must be an integer"` + ); + expect(() => assertValidModelVersion('7.6')).toThrowErrorMatchingInlineSnapshot( + `"Model version must be an integer"` + ); + }); + + it('throws if the provided value is a negative integer', () => { + expect(() => assertValidModelVersion(-4)).toThrowErrorMatchingInlineSnapshot( + `"Model version cannot be negative"` + ); + expect(() => assertValidModelVersion('-3')).toThrowErrorMatchingInlineSnapshot( + `"Model version cannot be negative"` + ); + }); + + it('returns the model version as a number', () => { + expect(assertValidModelVersion(4)).toEqual(4); + expect(assertValidModelVersion('3')).toEqual(3); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts new file mode 100644 index 0000000000000..c3765ca3c9be9 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Semver from 'semver'; +import { modelVersionVirtualMajor } from './constants'; + +/** + * Returns the virtual version associated with the given model version + * + * @example + * ``` + * modelVersionToVirtualVersion(5); // "10.5.0"; + * modelVersionToVirtualVersion("3"); // "10.3.0"; + * ``` + */ +export const modelVersionToVirtualVersion = (modelVersion: number | string) => { + const validatedModelVersion = assertValidModelVersion(modelVersion); + return `${modelVersionVirtualMajor}.${validatedModelVersion}.0`; +}; + +/** + * Return true if the given semver version is a virtual model version. + * Virtual model versions are version which major is the {@link modelVersionVirtualMajor} + * + * @example + * ``` + * isVirtualModelVersion("10.3.0"); // true + * isVirtualModelVersion("9.7.0); // false + * isVirtualModelVersion("10.3.1); // false + * ``` + */ +export const isVirtualModelVersion = (version: string): boolean => { + const semver = Semver.parse(version); + if (!semver) { + throw new Error(`Invalid semver: ${version}`); + } + return _isVirtualModelVersion(semver); +}; + +/** + * Converts a virtual model version to its model version. + * + * @example + * ``` + * virtualVersionToModelVersion('10.3.0'); // 3 + * virtualVersionToModelVersion('9.3.0'); // throw + * ``` + */ +export const virtualVersionToModelVersion = (virtualVersion: string): number => { + const semver = Semver.parse(virtualVersion); + if (!semver) { + throw new Error(`Invalid semver: ${virtualVersion}`); + } + if (!_isVirtualModelVersion(semver)) { + throw new Error(`Version is not a virtual model version`); + } + return semver.minor; +}; + +/** + * Asserts the provided number or string is a valid model version, and returns it. + * + * A valid model version is a positive integer. + * + * @example + * ``` + * assertValidModelVersion("7"); // 7 + * assertValidModelVersion(4); // 4 + * assertValidModelVersion("foo"); // throw + * assertValidModelVersion("9.7"); // throw + * assertValidModelVersion("-3"); // throw + * ``` + */ +export const assertValidModelVersion = (modelVersion: string | number): number => { + if (typeof modelVersion === 'string') { + modelVersion = parseFloat(modelVersion); + } + if (!Number.isInteger(modelVersion)) { + throw new Error('Model version must be an integer'); + } + if (modelVersion < 0) { + throw new Error('Model version cannot be negative'); + } + return modelVersion; +}; + +const _isVirtualModelVersion = (semver: Semver.SemVer): boolean => { + return semver.major === modelVersionVirtualMajor && semver.patch === 0; +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts new file mode 100644 index 0000000000000..5301c0a4d219c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { modelVersionVirtualMajor } from './constants'; +export { + assertValidModelVersion, + isVirtualModelVersion, + modelVersionToVirtualVersion, + virtualVersionToModelVersion, +} from './conversion'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.mocks.ts new file mode 100644 index 0000000000000..6bf80d0e4b1b4 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.mocks.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getReferenceTransformsMock = jest.fn(); +export const getConversionTransformsMock = jest.fn(); + +jest.doMock('./internal_transforms', () => ({ + getReferenceTransforms: getReferenceTransformsMock, + getConversionTransforms: getConversionTransformsMock, +})); + +export const getModelVersionTransformsMock = jest.fn(); + +jest.doMock('./model_version', () => ({ + getModelVersionTransforms: getModelVersionTransformsMock, +})); + +export const validateTypeMigrationsMock = jest.fn(); + +jest.doMock('./validate_migrations', () => ({ + validateTypeMigrations: validateTypeMigrationsMock, +})); + +export const resetAllMocks = () => { + getReferenceTransformsMock.mockReset().mockReturnValue([]); + getConversionTransformsMock.mockReset().mockReturnValue([]); + getModelVersionTransformsMock.mockReset().mockReturnValue([]); + validateTypeMigrationsMock.mockReset(); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.ts new file mode 100644 index 0000000000000..0385271ea262d --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.test.ts @@ -0,0 +1,322 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + getConversionTransformsMock, + getModelVersionTransformsMock, + getReferenceTransformsMock, + resetAllMocks, + validateTypeMigrationsMock, +} from './build_active_migrations.test.mocks'; + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { buildActiveMigrations } from './build_active_migrations'; +import { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-internal'; +import { Transform, TransformType } from './types'; + +const kibanaVersion = '3.2.3'; + +describe('buildActiveMigrations', () => { + let log: MockedLogger; + let typeRegistry: SavedObjectTypeRegistry; + + const buildMigrations = () => { + return buildActiveMigrations({ typeRegistry, kibanaVersion, log }); + }; + + const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, + }); + + const transform = (type: TransformType, version: string): Transform => ({ + version, + transformType: type, + transform: jest.fn(), + }); + + const expectTransform = (type: TransformType, version: string): Transform => ({ + version, + transformType: type, + transform: expect.any(Function), + }); + + const addType = (parts: Partial) => { + typeRegistry.registerType(createType(parts)); + }; + + beforeEach(() => { + resetAllMocks(); + + log = loggerMock.create(); + typeRegistry = new SavedObjectTypeRegistry(); + }); + + describe('validation', () => { + it('calls validateMigrationsMapObject with the correct parameters', () => { + addType({ + name: 'foo', + migrations: { + '7.12.0': jest.fn(), + '7.16.0': jest.fn(), + }, + }); + + addType({ + name: 'bar', + migrations: () => ({ + '7.114.0': jest.fn(), + '8.3.0': jest.fn(), + }), + }); + + buildMigrations(); + + expect(validateTypeMigrationsMock).toHaveBeenCalledTimes(2); + expect(validateTypeMigrationsMock).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + type: expect.objectContaining({ name: 'foo' }), + kibanaVersion, + }) + ); + expect(validateTypeMigrationsMock).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + type: expect.objectContaining({ name: 'bar' }), + kibanaVersion, + }) + ); + }); + + it('throws if validateMigrationsMapObject throws', () => { + validateTypeMigrationsMock.mockImplementation(() => { + throw new Error('woups'); + }); + + addType({ + name: 'foo', + migrations: { + '7.12.0': jest.fn(), + '7.16.0': jest.fn(), + }, + }); + + expect(() => buildMigrations()).toThrowErrorMatchingInlineSnapshot(`"woups"`); + }); + }); + + describe('type migrations', () => { + it('returns the migrations registered by the type as transforms', () => { + addType({ + name: 'foo', + migrations: { + '7.12.0': jest.fn(), + '7.16.0': jest.fn(), + '8.3.0': jest.fn(), + }, + }); + + const migrations = buildMigrations(); + + expect(Object.keys(migrations).sort()).toEqual(['foo']); + expect(migrations.foo.transforms).toEqual([ + expectTransform(TransformType.Migrate, '7.12.0'), + expectTransform(TransformType.Migrate, '7.16.0'), + expectTransform(TransformType.Migrate, '8.3.0'), + ]); + }); + }); + + describe('model version transforms', () => { + it('calls getModelVersionTransforms with the correct parameters', () => { + const foo = createType({ name: 'foo' }); + const bar = createType({ name: 'bar' }); + + addType(foo); + addType(bar); + + buildMigrations(); + + expect(getModelVersionTransformsMock).toHaveBeenCalledTimes(2); + expect(getModelVersionTransformsMock).toHaveBeenNthCalledWith(1, { + log, + typeDefinition: foo, + }); + expect(getModelVersionTransformsMock).toHaveBeenNthCalledWith(2, { + log, + typeDefinition: bar, + }); + }); + + it('adds the transform from getModelVersionTransforms to each type', () => { + const foo = createType({ name: 'foo' }); + const bar = createType({ name: 'bar' }); + + addType(foo); + addType(bar); + + getModelVersionTransformsMock.mockImplementation( + ({ typeDefinition }: { typeDefinition: SavedObjectsType }) => { + if (typeDefinition.name === 'foo') { + return [transform(TransformType.Migrate, '7.12.0')]; + } else { + return [transform(TransformType.Migrate, '8.3.0')]; + } + } + ); + + const migrations = buildMigrations(); + + expect(Object.keys(migrations).sort()).toEqual(['bar', 'foo']); + expect(migrations.foo.transforms).toEqual([expectTransform(TransformType.Migrate, '7.12.0')]); + expect(migrations.bar.transforms).toEqual([expectTransform(TransformType.Migrate, '8.3.0')]); + }); + }); + + describe('internal transforms', () => { + it('calls getReferenceTransforms with the correct parameters', () => { + const foo = createType({ name: 'foo' }); + const bar = createType({ name: 'bar' }); + + addType(foo); + addType(bar); + + buildMigrations(); + + expect(getReferenceTransformsMock).toHaveBeenCalledTimes(1); + expect(getReferenceTransformsMock).toHaveBeenCalledWith(typeRegistry); + }); + + it('adds the transform from getReferenceTransforms to each type', () => { + const foo = createType({ name: 'foo' }); + const bar = createType({ name: 'bar' }); + + addType(foo); + addType(bar); + + getReferenceTransformsMock.mockReturnValue([ + transform(TransformType.Reference, '7.12.0'), + transform(TransformType.Reference, '7.17.0'), + ]); + + const migrations = buildMigrations(); + expect(Object.keys(migrations).sort()).toEqual(['bar', 'foo']); + expect(migrations.foo.transforms).toEqual([ + expectTransform(TransformType.Reference, '7.12.0'), + expectTransform(TransformType.Reference, '7.17.0'), + ]); + expect(migrations.bar.transforms).toEqual([ + expectTransform(TransformType.Reference, '7.12.0'), + expectTransform(TransformType.Reference, '7.17.0'), + ]); + }); + + it('calls getConversionTransforms with the correct parameters', () => { + const foo = createType({ name: 'foo' }); + const bar = createType({ name: 'bar' }); + + addType(foo); + addType(bar); + + buildMigrations(); + + expect(getConversionTransformsMock).toHaveBeenCalledTimes(2); + expect(getConversionTransformsMock).toHaveBeenNthCalledWith(1, foo); + expect(getConversionTransformsMock).toHaveBeenNthCalledWith(2, bar); + }); + + it('adds the transform from getConversionTransforms to each type', () => { + const foo = createType({ name: 'foo' }); + const bar = createType({ name: 'bar' }); + + addType(foo); + addType(bar); + + getConversionTransformsMock.mockImplementation((type: SavedObjectsType) => { + if (type.name === 'foo') { + return [transform(TransformType.Convert, '7.12.0')]; + } else { + return [transform(TransformType.Convert, '8.7.0')]; + } + }); + + const migrations = buildMigrations(); + expect(Object.keys(migrations).sort()).toEqual(['bar', 'foo']); + expect(migrations.foo.transforms).toEqual([expectTransform(TransformType.Convert, '7.12.0')]); + expect(migrations.bar.transforms).toEqual([expectTransform(TransformType.Convert, '8.7.0')]); + }); + }); + + describe('ordering', () => { + it('sort the migrations correctly', () => { + addType({ + name: 'foo', + migrations: { + '7.12.0': jest.fn(), + '7.16.0': jest.fn(), + }, + }); + + addType({ + name: 'bar', + migrations: { + '7.17.0': jest.fn(), + '8.2.1': jest.fn(), + }, + }); + + getModelVersionTransformsMock.mockImplementation( + ({ typeDefinition }: { typeDefinition: SavedObjectsType }) => { + if (typeDefinition.name === 'foo') { + return [transform(TransformType.Migrate, '7.18.2')]; + } else { + return [transform(TransformType.Migrate, '8.4.2')]; + } + } + ); + + getReferenceTransformsMock.mockReturnValue([ + transform(TransformType.Reference, '7.12.0'), + transform(TransformType.Reference, '7.17.3'), + ]); + + getConversionTransformsMock.mockImplementation((type: SavedObjectsType) => { + if (type.name === 'foo') { + return [transform(TransformType.Convert, '7.14.0')]; + } else { + return [transform(TransformType.Convert, '8.7.0')]; + } + }); + + const migrations = buildMigrations(); + + expect(Object.keys(migrations).sort()).toEqual(['bar', 'foo']); + expect(migrations.foo.transforms).toEqual([ + expectTransform(TransformType.Reference, '7.12.0'), + expectTransform(TransformType.Migrate, '7.12.0'), + expectTransform(TransformType.Convert, '7.14.0'), + expectTransform(TransformType.Migrate, '7.16.0'), + expectTransform(TransformType.Reference, '7.17.3'), + expectTransform(TransformType.Migrate, '7.18.2'), + ]); + expect(migrations.bar.transforms).toEqual([ + expectTransform(TransformType.Reference, '7.12.0'), + expectTransform(TransformType.Migrate, '7.17.0'), + expectTransform(TransformType.Reference, '7.17.3'), + expectTransform(TransformType.Migrate, '8.2.1'), + expectTransform(TransformType.Migrate, '8.4.2'), + expectTransform(TransformType.Convert, '8.7.0'), + ]); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.ts index 51d951a2a9a00..b14e2d237c8dd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.ts @@ -8,11 +8,12 @@ import _ from 'lodash'; import type { Logger } from '@kbn/logging'; -import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server'; -import { type ActiveMigrations, type Transform, TransformType } from './types'; +import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { type ActiveMigrations, type Transform, type TypeTransforms, TransformType } from './types'; import { getReferenceTransforms, getConversionTransforms } from './internal_transforms'; -import { validateMigrationsMapObject } from './validate_migrations'; -import { transformComparator, wrapWithTry } from './utils'; +import { validateTypeMigrations } from './validate_migrations'; +import { transformComparator, convertMigrationFunction } from './utils'; +import { getModelVersionTransforms } from './model_version'; /** * Converts migrations from a format that is convenient for callers to a format that @@ -20,45 +21,76 @@ import { transformComparator, wrapWithTry } from './utils'; * From: { type: { version: fn } } * To: { type: { latestVersion?: Record; transforms: [{ version: string, transform: fn }] } } */ -export function buildActiveMigrations( - typeRegistry: ISavedObjectTypeRegistry, - kibanaVersion: string, - log: Logger -): ActiveMigrations { +export function buildActiveMigrations({ + typeRegistry, + kibanaVersion, + convertVersion, + log, +}: { + typeRegistry: ISavedObjectTypeRegistry; + kibanaVersion: string; + convertVersion?: string; + log: Logger; +}): ActiveMigrations { const referenceTransforms = getReferenceTransforms(typeRegistry); return typeRegistry.getAllTypes().reduce((migrations, type) => { - const migrationsMap = - typeof type.migrations === 'function' ? type.migrations() : type.migrations; - validateMigrationsMapObject(type.name, kibanaVersion, migrationsMap); + validateTypeMigrations({ type, kibanaVersion, convertVersion }); - const migrationTransforms = Object.entries(migrationsMap ?? {}).map( - ([version, transform]) => ({ - version, - transform: wrapWithTry(version, type, transform, log), - transformType: TransformType.Migrate, - }) - ); - const conversionTransforms = getConversionTransforms(type); - const transforms = [ - ...referenceTransforms, - ...conversionTransforms, - ...migrationTransforms, - ].sort(transformComparator); + const typeTransforms = buildTypeTransforms({ + type, + log, + kibanaVersion, + referenceTransforms, + }); - if (!transforms.length) { + if (!typeTransforms.transforms.length) { return migrations; } return { ...migrations, - [type.name]: { - latestVersion: _.chain(transforms) - .groupBy('transformType') - .mapValues((items) => _.last(items)?.version) - .value() as Record, - transforms, - }, + [type.name]: typeTransforms, }; }, {} as ActiveMigrations); } + +const buildTypeTransforms = ({ + type, + log, + referenceTransforms, +}: { + type: SavedObjectsType; + kibanaVersion: string; + log: Logger; + referenceTransforms: Transform[]; +}): TypeTransforms => { + const migrationsMap = + typeof type.migrations === 'function' ? type.migrations() : type.migrations ?? {}; + + const migrationTransforms = Object.entries(migrationsMap ?? {}).map( + ([version, transform]) => ({ + version, + transform: convertMigrationFunction(version, type, transform, log), + transformType: TransformType.Migrate, + }) + ); + + const modelVersionTransforms = getModelVersionTransforms({ log, typeDefinition: type }); + + const conversionTransforms = getConversionTransforms(type); + const transforms = [ + ...referenceTransforms, + ...conversionTransforms, + ...migrationTransforms, + ...modelVersionTransforms, + ].sort(transformComparator); + + return { + latestVersion: _.chain(transforms) + .groupBy('transformType') + .mapValues((items) => _.last(items)?.version) + .value() as Record, + transforms, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.mock.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.mock.ts index df7914a55876f..34137133907b2 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.mock.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.mock.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -const mockGetConvertedObjectId = jest.fn().mockReturnValue('uuidv5'); +export const mockGetConvertedObjectId = jest.fn().mockReturnValue('uuidv5'); jest.mock('@kbn/core-saved-objects-utils-server', () => { const actual = jest.requireActual('@kbn/core-saved-objects-utils-server'); @@ -19,4 +19,12 @@ jest.mock('@kbn/core-saved-objects-utils-server', () => { }; }); -export { mockGetConvertedObjectId }; +export const validateTypeMigrationsMock = jest.fn(); + +jest.doMock('./validate_migrations', () => { + const actual = jest.requireActual('./validate_migrations'); + return { + ...actual, + validateTypeMigrations: validateTypeMigrationsMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts index 2d7fb11587840..677c9806e462b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts @@ -6,7 +6,10 @@ * Side Public License, v 1. */ -import { mockGetConvertedObjectId } from './document_migrator.test.mock'; +import { + mockGetConvertedObjectId, + validateTypeMigrationsMock, +} from './document_migrator.test.mock'; import { set } from '@kbn/safer-lodash-set'; import _ from 'lodash'; import type { SavedObjectUnsanitizedDoc, SavedObjectsType } from '@kbn/core-saved-objects-server'; @@ -48,6 +51,7 @@ const createRegistry = (...types: Array>) => { beforeEach(() => { mockGetConvertedObjectId.mockClear(); + validateTypeMigrationsMock.mockReset(); }); describe('DocumentMigrator', () => { @@ -60,69 +64,47 @@ describe('DocumentMigrator', () => { } describe('validation', () => { - const createDefinition = (migrations: any) => ({ - kibanaVersion: '3.2.3', - convertVersion: '8.0.0', - typeRegistry: createRegistry({ - name: 'foo', - migrations: migrations as any, - }), - log: mockLogger, - }); - - describe('#prepareMigrations', () => { - it('validates individual migration definitions', () => { - const invalidMigrator = new DocumentMigrator(createDefinition(() => 123)); - const voidMigrator = new DocumentMigrator(createDefinition(() => {})); - const emptyObjectMigrator = new DocumentMigrator(createDefinition(() => ({}))); - - expect(invalidMigrator.prepareMigrations).toThrow( - /Migrations map for type foo should be an object/i - ); - expect(voidMigrator.prepareMigrations).not.toThrow(); - expect(emptyObjectMigrator.prepareMigrations).not.toThrow(); - }); - - it('validates individual migrations are valid semvers', () => { - const withInvalidVersion = { - bar: (doc: any) => doc, - '1.2.3': (doc: any) => doc, + describe('during #prepareMigrations', () => { + it('calls validateMigrationsMapObject with the correct parameters', () => { + const ops = { + ...testOpts(), + typeRegistry: createRegistry( + { + name: 'foo', + migrations: { + '1.2.3': setAttr('attributes.name', 'Chris'), + }, + }, + { + name: 'bar', + migrations: { + '1.2.3': setAttr('attributes.name', 'Chris'), + }, + } + ), }; - const migrationFn = new DocumentMigrator(createDefinition(() => withInvalidVersion)); - const migrationObj = new DocumentMigrator(createDefinition(withInvalidVersion)); - - expect(migrationFn.prepareMigrations).toThrow(/Expected all properties to be semvers/i); - expect(migrationObj.prepareMigrations).toThrow(/Expected all properties to be semvers/i); - }); - it('validates individual migrations are not greater than the current Kibana version', () => { - const withGreaterVersion = { - '3.2.4': (doc: any) => doc, - }; - const migrationFn = new DocumentMigrator(createDefinition(() => withGreaterVersion)); - const migrationObj = new DocumentMigrator(createDefinition(withGreaterVersion)); + const migrator = new DocumentMigrator(ops); - const expectedError = `Invalid migration for type foo. Property '3.2.4' cannot be greater than the current Kibana version '3.2.3'.`; - expect(migrationFn.prepareMigrations).toThrowError(expectedError); - expect(migrationObj.prepareMigrations).toThrowError(expectedError); - }); + expect(validateTypeMigrationsMock).not.toHaveBeenCalled(); - it('validates the migration function', () => { - const invalidVersionFunction = { '1.2.3': 23 as any }; - const migrationFn = new DocumentMigrator(createDefinition(() => invalidVersionFunction)); - const migrationObj = new DocumentMigrator(createDefinition(invalidVersionFunction)); + migrator.prepareMigrations(); - expect(migrationFn.prepareMigrations).toThrow(/expected a function, but got 23/i); - expect(migrationObj.prepareMigrations).toThrow(/expected a function, but got 23/i); - }); - it('validates definitions with migrations: Function | Objects', () => { - const validMigrationMap = { - '1.2.3': () => {}, - }; - const migrationFn = new DocumentMigrator(createDefinition(() => validMigrationMap)); - const migrationObj = new DocumentMigrator(createDefinition(validMigrationMap)); - expect(migrationFn.prepareMigrations).not.toThrow(); - expect(migrationObj.prepareMigrations).not.toThrow(); + expect(validateTypeMigrationsMock).toHaveBeenCalledTimes(3); + expect(validateTypeMigrationsMock).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + type: expect.objectContaining({ name: 'foo' }), + kibanaVersion, + }) + ); + expect(validateTypeMigrationsMock).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + type: expect.objectContaining({ name: 'bar' }), + kibanaVersion, + }) + ); }); }); @@ -155,81 +137,6 @@ describe('DocumentMigrator', () => { }) ).toThrow(/Migrations are not ready. Make sure prepareMigrations is called first./i); }); - - it(`validates convertToMultiNamespaceTypeVersion can only be used with namespaceType 'multiple' or 'multiple-isolated'`, () => { - const invalidDefinition = { - kibanaVersion: '3.2.3', - typeRegistry: createRegistry({ - name: 'foo', - convertToMultiNamespaceTypeVersion: 'bar', - }), - log: mockLogger, - }; - expect(() => new DocumentMigrator(invalidDefinition)).toThrow( - `Invalid convertToMultiNamespaceTypeVersion for type foo. Expected namespaceType to be 'multiple' or 'multiple-isolated', but got 'single'.` - ); - }); - - it(`validates convertToMultiNamespaceTypeVersion must be a semver`, () => { - const invalidDefinition = { - kibanaVersion: '3.2.3', - typeRegistry: createRegistry({ - name: 'foo', - convertToMultiNamespaceTypeVersion: 'bar', - namespaceType: 'multiple', - }), - log: mockLogger, - }; - expect(() => new DocumentMigrator(invalidDefinition)).toThrow( - `Invalid convertToMultiNamespaceTypeVersion for type foo. Expected value to be a semver, but got 'bar'.` - ); - }); - - it('validates convertToMultiNamespaceTypeVersion matches the convertVersion, if specified', () => { - const invalidDefinition = { - kibanaVersion: '3.2.3', - typeRegistry: createRegistry({ - name: 'foo', - convertToMultiNamespaceTypeVersion: '3.2.4', - namespaceType: 'multiple', - }), - convertVersion: '3.2.3', - log: mockLogger, - }; - expect(() => new DocumentMigrator(invalidDefinition)).toThrowError( - `Invalid convertToMultiNamespaceTypeVersion for type foo. Value '3.2.4' cannot be any other than '3.2.3'.` - ); - }); - - it('validates convertToMultiNamespaceTypeVersion is not greater than the current Kibana version', () => { - const invalidDefinition = { - kibanaVersion: '3.2.3', - typeRegistry: createRegistry({ - name: 'foo', - convertToMultiNamespaceTypeVersion: '3.2.4', - namespaceType: 'multiple', - }), - log: mockLogger, - }; - expect(() => new DocumentMigrator(invalidDefinition)).toThrowError( - `Invalid convertToMultiNamespaceTypeVersion for type foo. Value '3.2.4' cannot be greater than the current Kibana version '3.2.3'.` - ); - }); - - it('validates convertToMultiNamespaceTypeVersion is not used on a patch version', () => { - const invalidDefinition = { - kibanaVersion: '3.2.3', - typeRegistry: createRegistry({ - name: 'foo', - convertToMultiNamespaceTypeVersion: '3.1.1', - namespaceType: 'multiple', - }), - log: mockLogger, - }; - expect(() => new DocumentMigrator(invalidDefinition)).toThrowError( - `Invalid convertToMultiNamespaceTypeVersion for type foo. Value '3.1.1' cannot be used on a patch version (must be like 'x.y.0').` - ); - }); }); describe('migration', () => { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.ts index 7dc2e0dd885d5..ada644dcfc9de 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.ts @@ -54,7 +54,6 @@ import type { import type { ActiveMigrations, TransformResult } from './types'; import { maxVersion } from './utils'; import { buildActiveMigrations } from './build_active_migrations'; -import { validateMigrationDefinition } from './validate_migrations'; export type MigrateFn = (doc: SavedObjectUnsanitizedDoc) => SavedObjectUnsanitizedDoc; export type MigrateAndConvertFn = (doc: SavedObjectUnsanitizedDoc) => SavedObjectUnsanitizedDoc[]; @@ -89,7 +88,7 @@ export interface VersionedTransformer { * A concrete implementation of the VersionedTransformer interface. */ export class DocumentMigrator implements VersionedTransformer { - private documentMigratorOptions: Omit; + private documentMigratorOptions: DocumentMigratorOptions; private migrations?: ActiveMigrations; private transformDoc?: ApplyTransformsFn; @@ -103,9 +102,8 @@ export class DocumentMigrator implements VersionedTransformer { * @prop {Logger} log - The migration logger * @memberof DocumentMigrator */ - constructor({ typeRegistry, kibanaVersion, convertVersion, log }: DocumentMigratorOptions) { - validateMigrationDefinition(typeRegistry, kibanaVersion, convertVersion); - this.documentMigratorOptions = { typeRegistry, kibanaVersion, log }; + constructor(documentMigratorOptions: DocumentMigratorOptions) { + this.documentMigratorOptions = documentMigratorOptions; } /** @@ -138,8 +136,13 @@ export class DocumentMigrator implements VersionedTransformer { */ public prepareMigrations = () => { - const { typeRegistry, kibanaVersion, log } = this.documentMigratorOptions; - this.migrations = buildActiveMigrations(typeRegistry, kibanaVersion, log); + const { typeRegistry, kibanaVersion, log, convertVersion } = this.documentMigratorOptions; + this.migrations = buildActiveMigrations({ + typeRegistry, + kibanaVersion, + log, + convertVersion, + }); this.transformDoc = buildDocumentTransform({ kibanaVersion, migrations: this.migrations, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.test.ts new file mode 100644 index 0000000000000..c32d18463083c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.test.ts @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import type { + SavedObjectsType, + SavedObjectsModelVersion, + SavedObjectModelTransformationFn, + SavedObjectUnsanitizedDoc, +} from '@kbn/core-saved-objects-server'; +import { Transform, TransformType } from './types'; +import { getModelVersionTransforms, convertModelVersionTransformFn } from './model_version'; + +describe('getModelVersionTransforms', () => { + let log: MockedLogger; + + const expectTransform = (type: TransformType, version: string): Transform => ({ + version, + transformType: type, + transform: expect.any(Function), + }); + + const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, + }); + + beforeEach(() => { + log = loggerMock.create(); + }); + + it('generate transforms for model version having a transformation', () => { + const typeDefinition = createType({ + name: 'foo', + modelVersions: { + '1': { + modelChange: { + type: 'expansion', + transformation: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { foo: { type: 'keyword' } }, + }, + }, + '3': { + modelChange: { + type: 'expansion', + transformation: { up: jest.fn(), down: jest.fn() }, + }, + }, + }, + }); + + const transforms = getModelVersionTransforms({ log, typeDefinition }); + + expect(transforms).toEqual([ + expectTransform(TransformType.Migrate, '10.1.0'), + expectTransform(TransformType.Migrate, '10.3.0'), + ]); + }); + + it('accepts provider functions', () => { + const typeDefinition = createType({ + name: 'foo', + modelVersions: () => ({ + '1': { + modelChange: { + type: 'expansion', + transformation: { up: jest.fn(), down: jest.fn() }, + }, + }, + '2': { + modelChange: { + type: 'expansion', + addedMappings: { foo: { type: 'keyword' } }, + }, + }, + '3': { + modelChange: { + type: 'expansion', + transformation: { up: jest.fn(), down: jest.fn() }, + }, + }, + }), + }); + + const transforms = getModelVersionTransforms({ log, typeDefinition }); + + expect(transforms).toEqual([ + expectTransform(TransformType.Migrate, '10.1.0'), + expectTransform(TransformType.Migrate, '10.3.0'), + ]); + }); +}); + +describe('convertModelVersionTransformFn', () => { + let log: MockedLogger; + let i = 0; + + beforeEach(() => { + i = 0; + log = loggerMock.create(); + }); + + const createDoc = (): SavedObjectUnsanitizedDoc => { + return { type: 'foo', id: `foo-${i++}`, attributes: {} }; + }; + + const createModelTransformFn = (): jest.MockedFunction => { + return jest.fn().mockImplementation((doc: unknown) => ({ + document: doc, + })); + }; + + it('generates a transform function calling the model transform', () => { + const upTransform = createModelTransformFn(); + const downTransform = createModelTransformFn(); + + const definition: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + transformation: { up: upTransform, down: downTransform }, + }, + }; + + const transform = convertModelVersionTransformFn({ + log, + modelVersion: 1, + virtualVersion: '10.1.0', + definition, + }); + + expect(upTransform).not.toHaveBeenCalled(); + expect(downTransform).not.toHaveBeenCalled(); + + const doc = createDoc(); + const context = { log, modelVersion: 1 }; + + transform(doc); + + expect(upTransform).toHaveBeenCalledTimes(1); + expect(downTransform).not.toHaveBeenCalled(); + expect(upTransform).toHaveBeenCalledWith(doc, context); + }); + + it('returns the document from the model transform', () => { + const upTransform = createModelTransformFn(); + + const resultDoc = createDoc(); + upTransform.mockImplementation((doc) => { + return { document: resultDoc }; + }); + + const definition: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + transformation: { up: upTransform, down: jest.fn() }, + }, + }; + + const transform = convertModelVersionTransformFn({ + log, + modelVersion: 1, + virtualVersion: '10.1.0', + definition, + }); + + const doc = createDoc(); + + const result = transform(doc); + expect(result).toEqual({ + transformedDoc: resultDoc, + additionalDocs: [], + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.ts new file mode 100644 index 0000000000000..cf6f14e2dc83a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/model_version.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Logger } from '@kbn/logging'; +import type { + SavedObjectsType, + SavedObjectsModelVersion, + SavedObjectModelTransformationContext, + SavedObjectUnsanitizedDoc, +} from '@kbn/core-saved-objects-server'; +import { + modelVersionToVirtualVersion, + assertValidModelVersion, +} from '@kbn/core-saved-objects-base-server-internal'; +import { TransformSavedObjectDocumentError } from '../core'; +import { type Transform, type TransformFn, TransformType } from './types'; + +export const getModelVersionTransforms = ({ + typeDefinition, + log, +}: { + typeDefinition: SavedObjectsType; + log: Logger; +}): Transform[] => { + const modelVersionMap = + typeof typeDefinition.modelVersions === 'function' + ? typeDefinition.modelVersions() + : typeDefinition.modelVersions ?? {}; + + const transforms = Object.entries(modelVersionMap) + .filter(([_, definition]) => !!definition.modelChange.transformation) + .map(([rawModelVersion, definition]) => { + const modelVersion = assertValidModelVersion(rawModelVersion); + const virtualVersion = modelVersionToVirtualVersion(modelVersion); + return { + version: virtualVersion, + transform: convertModelVersionTransformFn({ + log, + modelVersion, + virtualVersion, + definition, + }), + transformType: TransformType.Migrate, + }; + }); + + return transforms; +}; + +export const convertModelVersionTransformFn = ({ + virtualVersion, + modelVersion, + definition, + log, +}: { + virtualVersion: string; + modelVersion: number; + definition: SavedObjectsModelVersion; + log: Logger; +}): TransformFn => { + if (!definition.modelChange.transformation) { + throw new Error('cannot convert model change without a transform'); + } + const context: SavedObjectModelTransformationContext = { + log, + modelVersion, + }; + const modelTransformFn = definition.modelChange.transformation.up; + + return function convertedTransform(doc: SavedObjectUnsanitizedDoc) { + try { + const result = modelTransformFn(doc, context); + return { transformedDoc: result.document, additionalDocs: [] }; + } catch (error) { + log.error(error); + throw new TransformSavedObjectDocumentError(error, virtualVersion); + } + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/types.ts index f22a010bdb0e8..93e47aa84e673 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/types.ts @@ -12,13 +12,13 @@ import type { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server'; * Map containing all the info to convert types */ export interface ActiveMigrations { - [type: string]: TypeConversion; + [type: string]: TypeTransforms; } /** * Structure containing all the required info to perform a type's conversion */ -export interface TypeConversion { +export interface TypeTransforms { /** Derived from the related transforms */ latestVersion: Record; /** List of transforms registered for the type **/ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/utils.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/utils.ts index 3c2b8d83652a9..09aba35da06a1 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/utils.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/utils.ts @@ -21,7 +21,7 @@ import { type Transform, type TransformFn, TransformType } from './types'; * If a specific transform function fails, this tacks on a bit of information * about the document and transform that caused the failure. */ -export function wrapWithTry( +export function convertMigrationFunction( version: string, type: SavedObjectsType, migrationFn: SavedObjectMigrationFn, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts new file mode 100644 index 0000000000000..ef6139572baf6 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migration.test.ts @@ -0,0 +1,272 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsType, SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { validateTypeMigrations } from './validate_migrations'; + +describe('validateTypeMigrations', () => { + const defaultKibanaVersion = '3.2.3'; + const defaultConvertVersion = '8.0.0'; + + const someModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + transformation: { up: jest.fn(), down: jest.fn() }, + }, + }; + + const createType = (parts: Partial): SavedObjectsType => ({ + name: 'test', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, + ...parts, + }); + + const validate = ({ + type, + kibanaVersion = defaultKibanaVersion, + convertVersion = defaultConvertVersion, + }: { + type: SavedObjectsType; + convertVersion?: string; + kibanaVersion?: string; + }) => validateTypeMigrations({ type, kibanaVersion, convertVersion }); + + describe('migrations', () => { + it('validates individual migrations are valid semvers', () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: '3.1.1', + namespaceType: 'multiple', + migrations: { + bar: jest.fn(), + '1.2.3': jest.fn(), + }, + }); + + expect(() => validate({ type })).toThrow(/Expected all properties to be semvers/i); + }); + + it('validates individual migrations are not greater than the current Kibana version', () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: '3.1.1', + namespaceType: 'multiple', + migrations: { + '3.2.4': (doc) => doc, + }, + }); + + expect(() => validate({ type })).toThrowError( + `Invalid migration for type foo. Property '3.2.4' cannot be greater than the current Kibana version '3.2.3'.` + ); + }); + + it('validates the migration function', () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: '3.1.1', + namespaceType: 'multiple', + migrations: { '1.2.3': 23 as any }, + }); + + expect(() => validate({ type })).toThrow(/expected a function, but got 23/i); + }); + + describe('when switchToModelVersionAt is specified', () => { + it('throws if a migration is specified for a version superior to switchToModelVersionAt', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '8.9.0', + migrations: { + '8.10.0': jest.fn(), + }, + }); + + expect(() => + validate({ type, kibanaVersion: '8.10.0' }) + ).toThrowErrorMatchingInlineSnapshot( + `"Migration for type foo for version 8.10.0 registered after switchToModelVersionAt (8.9.0)"` + ); + }); + it('throws if a migration is specified for a version equal to switchToModelVersionAt', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '8.9.0', + migrations: { + '8.9.0': jest.fn(), + }, + }); + + expect(() => + validate({ type, kibanaVersion: '8.10.0' }) + ).toThrowErrorMatchingInlineSnapshot( + `"Migration for type foo for version 8.9.0 registered after switchToModelVersionAt (8.9.0)"` + ); + }); + + it('does not throw if a migration is specified for a version inferior to switchToModelVersionAt', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '8.9.0', + migrations: { + '8.7.0': jest.fn(), + }, + }); + + expect(() => validate({ type, kibanaVersion: '8.10.0' })).not.toThrow(); + }); + }); + }); + + describe('switchToModelVersionAt', () => { + it('throws if the specified version is not a valid semver', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: 'foo', + }); + + expect(() => validate({ type })).toThrowErrorMatchingInlineSnapshot( + `"Type foo: invalid version specified for switchToModelVersionAt: foo"` + ); + }); + + it('throws if the specified version defines a patch version > 0', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '8.9.3', + }); + + expect(() => validate({ type })).toThrowErrorMatchingInlineSnapshot( + `"Type foo: can't use a patch version for switchToModelVersionAt"` + ); + }); + }); + + describe('modelVersions', () => { + it('throws if used without specifying switchToModelVersionAt', () => { + const type = createType({ + name: 'foo', + modelVersions: { + '1': someModelVersion, + }, + }); + + expect(() => validate({ type, kibanaVersion: '3.2.3' })).toThrowErrorMatchingInlineSnapshot( + `"Type foo: Uusing modelVersions requires to specify switchToModelVersionAt"` + ); + }); + + it('throws if the version number is invalid', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '3.1.0', + modelVersions: { + '1.1': someModelVersion, + }, + }); + + expect(() => validate({ type, kibanaVersion: '3.2.3' })).toThrowErrorMatchingInlineSnapshot( + `"Model version must be an integer"` + ); + }); + + it('throws when starting with a version higher than 1', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '3.1.0', + modelVersions: { + '2': someModelVersion, + }, + }); + + expect(() => validate({ type, kibanaVersion: '3.2.3' })).toThrowErrorMatchingInlineSnapshot( + `"Type foo: model versioning must start with version 1"` + ); + }); + + it('throws when there is a gap in versions', () => { + const type = createType({ + name: 'foo', + switchToModelVersionAt: '3.1.0', + modelVersions: { + '1': someModelVersion, + '3': someModelVersion, + '6': someModelVersion, + }, + }); + + expect(() => validate({ type, kibanaVersion: '3.2.3' })).toThrowErrorMatchingInlineSnapshot( + `"Type foo: gaps between model versions aren't allowed (missing versions: 2,4,5)"` + ); + }); + }); + + describe('convertToMultiNamespaceTypeVersion', () => { + it(`validates convertToMultiNamespaceTypeVersion can only be used with namespaceType 'multiple' or 'multiple-isolated'`, () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: 'bar', + }); + expect(() => validate({ type })).toThrow( + `Invalid convertToMultiNamespaceTypeVersion for type foo. Expected namespaceType to be 'multiple' or 'multiple-isolated', but got 'single'.` + ); + }); + + it(`validates convertToMultiNamespaceTypeVersion must be a semver`, () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: 'bar', + namespaceType: 'multiple', + }); + expect(() => validate({ type })).toThrow( + `Invalid convertToMultiNamespaceTypeVersion for type foo. Expected value to be a semver, but got 'bar'.` + ); + }); + + it('validates convertToMultiNamespaceTypeVersion matches the convertVersion, if specified', () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: '3.2.4', + namespaceType: 'multiple', + }); + expect(() => + validate({ type, convertVersion: '3.2.3', kibanaVersion: '3.2.3' }) + ).toThrowError( + `Invalid convertToMultiNamespaceTypeVersion for type foo. Value '3.2.4' cannot be any other than '3.2.3'.` + ); + }); + + it('validates convertToMultiNamespaceTypeVersion is not greater than the current Kibana version', () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: '3.2.4', + namespaceType: 'multiple', + }); + expect(() => + validateTypeMigrations({ type, kibanaVersion: defaultKibanaVersion }) + ).toThrowError( + `Invalid convertToMultiNamespaceTypeVersion for type foo. Value '3.2.4' cannot be greater than the current Kibana version '3.2.3'.` + ); + }); + + it('validates convertToMultiNamespaceTypeVersion is not used on a patch version', () => { + const type = createType({ + name: 'foo', + convertToMultiNamespaceTypeVersion: '3.1.1', + namespaceType: 'multiple', + }); + expect(() => + validateTypeMigrations({ type, kibanaVersion: defaultKibanaVersion }) + ).toThrowError( + `Invalid convertToMultiNamespaceTypeVersion for type foo. Value '3.1.1' cannot be used on a patch version (must be like 'x.y.0').` + ); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts index 463ee85ac1808..2dc93f0e54353 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/validate_migrations.ts @@ -8,114 +8,180 @@ import Semver from 'semver'; import type { SavedObjectsNamespaceType } from '@kbn/core-saved-objects-common'; -import type { - SavedObjectMigrationMap, - ISavedObjectTypeRegistry, -} from '@kbn/core-saved-objects-server'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { assertValidModelVersion } from '@kbn/core-saved-objects-base-server-internal'; /** - * Basic validation that the migration definition matches our expectations. We can't - * rely on TypeScript here, as the caller may be JavaScript / ClojureScript / any compile-to-js - * language. So, this is just to provide a little developer-friendly error messaging. Joi was - * giving weird errors, so we're just doing manual validation. + * Validates the consistency of the given type for use with the document migrator. */ -export function validateMigrationDefinition( - registry: ISavedObjectTypeRegistry, - kibanaVersion: string, - convertVersion?: string -) { - function assertObjectOrFunction(entity: any, prefix: string) { - if (!entity || (typeof entity !== 'function' && typeof entity !== 'object')) { - throw new Error(`${prefix} Got! ${typeof entity}, ${JSON.stringify(entity)}.`); - } - } - - function assertValidConvertToMultiNamespaceType( - namespaceType: SavedObjectsNamespaceType, - convertToMultiNamespaceTypeVersion: string, - type: string - ) { - if (namespaceType !== 'multiple' && namespaceType !== 'multiple-isolated') { - throw new Error( - `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Expected namespaceType to be 'multiple' or 'multiple-isolated', but got '${namespaceType}'.` - ); - } else if (!Semver.valid(convertToMultiNamespaceTypeVersion)) { - throw new Error( - `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Expected value to be a semver, but got '${convertToMultiNamespaceTypeVersion}'.` - ); - } else if (convertVersion && Semver.neq(convertToMultiNamespaceTypeVersion, convertVersion)) { +export function validateTypeMigrations({ + type, + kibanaVersion, + convertVersion, +}: { + type: SavedObjectsType; + kibanaVersion: string; + convertVersion?: string; +}) { + if (type.switchToModelVersionAt) { + const switchToModelVersionAt = Semver.parse(type.switchToModelVersionAt); + if (!switchToModelVersionAt) { throw new Error( - `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Value '${convertToMultiNamespaceTypeVersion}' cannot be any other than '${convertVersion}'.` - ); - } else if (Semver.gt(convertToMultiNamespaceTypeVersion, kibanaVersion)) { - throw new Error( - `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Value '${convertToMultiNamespaceTypeVersion}' cannot be greater than the current Kibana version '${kibanaVersion}'.` - ); - } else if (Semver.patch(convertToMultiNamespaceTypeVersion)) { - throw new Error( - `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Value '${convertToMultiNamespaceTypeVersion}' cannot be used on a patch version (must be like 'x.y.0').` + `Type ${type.name}: invalid version specified for switchToModelVersionAt: ${type.switchToModelVersionAt}` ); } + if (switchToModelVersionAt.patch !== 0) { + throw new Error(`Type ${type.name}: can't use a patch version for switchToModelVersionAt`); + } } - registry.getAllTypes().forEach((type) => { - const { name, migrations, convertToMultiNamespaceTypeVersion, namespaceType } = type; - if (migrations) { - assertObjectOrFunction( - type.migrations, - `Migration for type ${name} should be an object or a function returning an object like { '2.0.0': (doc) => doc }.` - ); - } - if (convertToMultiNamespaceTypeVersion) { - // CHECKPOINT 1 - assertValidConvertToMultiNamespaceType( - namespaceType, - convertToMultiNamespaceTypeVersion, - name - ); - } - }); -} + if (type.migrations) { + assertObjectOrFunction( + type.migrations, + `Migration for type ${type.name} should be an object or a function returning an object like { '2.0.0': (doc) => doc }.` + ); -export function validateMigrationsMapObject( - name: string, - kibanaVersion: string, - migrationsMap?: SavedObjectMigrationMap -) { - function assertObject(obj: any, prefix: string) { - if (!obj || typeof obj !== 'object') { - throw new Error(`${prefix} Got ${obj}.`); - } + const migrationMap = + typeof type.migrations === 'function' ? type.migrations() : type.migrations ?? {}; + + assertObject( + migrationMap, + `Migrations map for type ${type.name} should be an object like { '2.0.0': (doc) => doc }.` + ); + + Object.entries(migrationMap).forEach(([version, fn]) => { + assertValidSemver(kibanaVersion, version, type.name); + assertValidTransform(fn, version, type.name); + if (type.switchToModelVersionAt && Semver.gte(version, type.switchToModelVersionAt)) { + throw new Error( + `Migration for type ${type.name} for version ${version} registered after switchToModelVersionAt (${type.switchToModelVersionAt})` + ); + } + }); } - function assertValidSemver(version: string, type: string) { - if (!Semver.valid(version)) { + if (type.modelVersions) { + const modelVersionMap = + typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {}; + + if (Object.keys(modelVersionMap).length > 0 && !type.switchToModelVersionAt) { throw new Error( - `Invalid migration for type ${type}. Expected all properties to be semvers, but got ${version}.` + `Type ${type.name}: Uusing modelVersions requires to specify switchToModelVersionAt` ); } - if (Semver.gt(version, kibanaVersion)) { + + Object.entries(modelVersionMap).forEach(([version, definition]) => { + assertValidModelVersion(version); + }); + + const { min: minVersion, max: maxVersion } = Object.keys(modelVersionMap).reduce( + (minMax, rawVersion) => { + const version = Number.parseInt(rawVersion, 10); + minMax.min = Math.min(minMax.min, version); + minMax.max = Math.max(minMax.max, version); + return minMax; + }, + { min: Infinity, max: -Infinity } + ); + + if (minVersion > 1) { + throw new Error(`Type ${type.name}: model versioning must start with version 1`); + } + const missingVersions = getMissingVersions( + minVersion, + maxVersion, + Object.keys(modelVersionMap).map((v) => Number.parseInt(v, 10)) + ); + if (missingVersions.length) { throw new Error( - `Invalid migration for type ${type}. Property '${version}' cannot be greater than the current Kibana version '${kibanaVersion}'.` + `Type ${ + type.name + }: gaps between model versions aren't allowed (missing versions: ${missingVersions.join( + ',' + )})` ); } } - function assertValidTransform(fn: any, version: string, type: string) { - if (typeof fn !== 'function') { - throw new Error(`Invalid migration ${type}.${version}: expected a function, but got ${fn}.`); - } + if (type.convertToMultiNamespaceTypeVersion) { + assertValidConvertToMultiNamespaceType( + kibanaVersion, + convertVersion, + type.namespaceType, + type.convertToMultiNamespaceTypeVersion, + type.name + ); } +} - if (migrationsMap) { - assertObject( - migrationsMap, - `Migrations map for type ${name} should be an object like { '2.0.0': (doc) => doc }.` +const assertObjectOrFunction = (entity: any, prefix: string) => { + if (!entity || (typeof entity !== 'function' && typeof entity !== 'object')) { + throw new Error(`${prefix} Got! ${typeof entity}, ${JSON.stringify(entity)}.`); + } +}; + +const assertObject = (obj: any, prefix: string) => { + if (!obj || typeof obj !== 'object') { + throw new Error(`${prefix} Got ${obj}.`); + } +}; + +const assertValidSemver = (kibanaVersion: string, version: string, type: string) => { + if (!Semver.valid(version)) { + throw new Error( + `Invalid migration for type ${type}. Expected all properties to be semvers, but got ${version}.` + ); + } + if (Semver.gt(version, kibanaVersion)) { + throw new Error( + `Invalid migration for type ${type}. Property '${version}' cannot be greater than the current Kibana version '${kibanaVersion}'.` ); + } +}; - Object.entries(migrationsMap).forEach(([version, fn]) => { - assertValidSemver(version, name); - assertValidTransform(fn, version, name); - }); +const assertValidConvertToMultiNamespaceType = ( + kibanaVersion: string, + convertVersion: string | undefined, + namespaceType: SavedObjectsNamespaceType, + convertToMultiNamespaceTypeVersion: string, + type: string +) => { + if (namespaceType !== 'multiple' && namespaceType !== 'multiple-isolated') { + throw new Error( + `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Expected namespaceType to be 'multiple' or 'multiple-isolated', but got '${namespaceType}'.` + ); + } else if (!Semver.valid(convertToMultiNamespaceTypeVersion)) { + throw new Error( + `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Expected value to be a semver, but got '${convertToMultiNamespaceTypeVersion}'.` + ); + } else if (convertVersion && Semver.neq(convertToMultiNamespaceTypeVersion, convertVersion)) { + throw new Error( + `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Value '${convertToMultiNamespaceTypeVersion}' cannot be any other than '${convertVersion}'.` + ); + } else if (Semver.gt(convertToMultiNamespaceTypeVersion, kibanaVersion)) { + throw new Error( + `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Value '${convertToMultiNamespaceTypeVersion}' cannot be greater than the current Kibana version '${kibanaVersion}'.` + ); + } else if (Semver.patch(convertToMultiNamespaceTypeVersion)) { + throw new Error( + `Invalid convertToMultiNamespaceTypeVersion for type ${type}. Value '${convertToMultiNamespaceTypeVersion}' cannot be used on a patch version (must be like 'x.y.0').` + ); } -} +}; + +const assertValidTransform = (fn: any, version: string, type: string) => { + if (typeof fn !== 'function') { + throw new Error(`Invalid migration ${type}.${version}: expected a function, but got ${fn}.`); + } +}; + +const getMissingVersions = (from: number, to: number, versions: number[]): number[] => { + const versionSet = new Set(versions); + const missing: number[] = []; + for (let i = from; i <= to; i++) { + if (!versionSet.has(i)) { + missing.push(i); + } + } + return missing; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json index 3612fae05aba1..d1751da1050f5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json @@ -29,6 +29,7 @@ "@kbn/core-elasticsearch-server-mocks", "@kbn/doc-links", "@kbn/safer-lodash-set", + "@kbn/logging-mocks", ], "exclude": [ "target/**/*", diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts index 5af4866ddd576..5d24b74d3ab89 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/transformations.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import type { SavedObjectSanitizedDoc } from '../serialization'; +import type { SavedObjectUnsanitizedDoc } from '../serialization'; import type { SavedObjectsMigrationLogger } from '../migration'; /** @@ -14,7 +14,7 @@ import type { SavedObjectsMigrationLogger } from '../migration'; * * @public */ -export type SavedObjectModelTransformationDoc = SavedObjectSanitizedDoc; +export type SavedObjectModelTransformationDoc = SavedObjectUnsanitizedDoc; /** * Context passed down to {@link SavedObjectModelTransformationFn | transformation functions}.