Adapt the ZDT migration algorithm to support v2 migrations (#155981)

## Summary

Adapt the `zdt` migration algorithm to run the v2 migrations in addition
to the model version transformations.

The intent is to be able to use the current migration system in a
zero-downtime upgrade friendly-ish way

### Technicals  

#148656 was a precondition, as the
zdt algo requires that mapping changes are compatible (given we keep the
same index).

Technically, it means we're now storing the `virtualVersion` instead of
the `modelVersion` in the index's meta (`mappingVersions` and
`docVersions`), to allow keeping track of mixed stack and model versions
per type.

Note that switching to model versioning is still a one way,
non-revertible action. Once using model versioning (by specifying
`switchToModelVersionAt` on your type), there is no going back.

packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts

@@ -49,13 +49,17 @@ export {
   modelVersionToVirtualVersion,
   getModelVersionMapForTypes,
   getLatestModelVersion,
+  getCurrentVirtualVersion,
+  getLatestMigrationVersion,
+  getVirtualVersionMap,
   type ModelVersionMap,
-  compareModelVersions,
+  type VirtualVersionMap,
+  compareVirtualVersions,
   type CompareModelVersionMapParams,
   type CompareModelVersionStatus,
   type CompareModelVersionDetails,
   type CompareModelVersionResult,
-  getModelVersionsFromMappings,
-  getModelVersionsFromMappingMeta,
+  getVirtualVersionsFromMappings,
+  getVirtualVersionsFromMappingMeta,
   getModelVersionDelta,
 } from './src/model_version';

packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts

@@ -9,6 +9,8 @@
 export { getTypes, getProperty, getRootProperties, getRootPropertiesObjects } from './lib';
 export type {
   SavedObjectsTypeMappingDefinitions,
+  V2AlgoIndexMappingMeta,
+  ZdtAlgoIndexMappingMeta,
   IndexMappingMeta,
   IndexMapping,
   IndexTypesMap,

packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts

@@ -59,7 +59,7 @@ export interface IndexMapping {
 export type IndexTypesMap = Record<string, string[]>;
 
 /** @internal */
-export interface IndexMappingMeta {
+export interface V2AlgoIndexMappingMeta {
   /**
    * A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa')
    * with each key being a root-level mapping property, and each value being
@@ -74,18 +74,22 @@ export interface IndexMappingMeta {
    * @remark: Only defined for indices using the v2 migration algorithm.
    */
   indexTypesMap?: IndexTypesMap;
+}
+
+/** @internal */
+export interface ZdtAlgoIndexMappingMeta {
   /**
-   * The current model versions of the mapping of the index.
+   * The current virtual version of the mapping of the index.
    *
    * @remark: Only defined for indices using the zdt migration algorithm.
    */
-  mappingVersions?: { [k: string]: number };
+  mappingVersions?: { [k: string]: string };
   /**
-   * The current model versions of the documents of the index.
+   * The current virtual versions of the documents of the index.
    *
    * @remark: Only defined for indices using the zdt migration algorithm.
    */
-  docVersions?: { [k: string]: number };
+  docVersions?: { [k: string]: string };
   /**
    * Info about the current state of the migration.
    * Should only be present if a migration is in progress or was interrupted.
@@ -95,6 +99,9 @@ export interface IndexMappingMeta {
   migrationState?: IndexMappingMigrationStateMeta;
 }
 
+/** @internal */
+export type IndexMappingMeta = V2AlgoIndexMappingMeta & ZdtAlgoIndexMappingMeta;
+
 /** @internal */
 export interface IndexMappingMigrationStateMeta {
   /**

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.test.ts

@@ -11,6 +11,7 @@ import {
   virtualVersionToModelVersion,
   modelVersionToVirtualVersion,
   assertValidModelVersion,
+  assertValidVirtualVersion,
 } from './conversion';
 
 describe('isVirtualModelVersion', () => {
@@ -103,3 +104,19 @@ describe('assertValidModelVersion', () => {
     expect(assertValidModelVersion('3')).toEqual(3);
   });
 });
+
+describe('assertValidVirtualVersion', () => {
+  it('throws if the provided value is not a valid semver', () => {
+    expect(() => assertValidVirtualVersion('foooo')).toThrowErrorMatchingInlineSnapshot(
+      `"Virtual versions must be valid semver versions"`
+    );
+    expect(() => assertValidVirtualVersion('1.2.3.4.5.6.7')).toThrowErrorMatchingInlineSnapshot(
+      `"Virtual versions must be valid semver versions"`
+    );
+  });
+
+  it('returns the virtual version', () => {
+    expect(assertValidVirtualVersion('7.17.5')).toEqual('7.17.5');
+    expect(assertValidVirtualVersion('10.3.0')).toEqual('10.3.0');
+  });
+});

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/conversion.ts

@@ -89,6 +89,14 @@ export const assertValidModelVersion = (modelVersion: string | number): number =
   return modelVersion;
 };
 
+export const assertValidVirtualVersion = (virtualVersion: string): string => {
+  const semver = Semver.parse(virtualVersion);
+  if (!semver) {
+    throw new Error('Virtual versions must be valid semver versions');
+  }
+  return virtualVersion;
+};
+
 const _isVirtualModelVersion = (semver: Semver.SemVer): boolean => {
   return semver.major === modelVersionVirtualMajor && semver.patch === 0;
 };

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts

@@ -12,12 +12,12 @@ describe('getModelVersionDelta', () => {
   it('generates an upward delta', () => {
     const result = getModelVersionDelta({
       currentVersions: {
-        a: 1,
-        b: 1,
+        a: '10.1.0',
+        b: '10.1.0',
       },
       targetVersions: {
-        a: 2,
-        b: 3,
+        a: '10.2.0',
+        b: '10.3.0',
       },
       deletedTypes: [],
     });
@@ -26,26 +26,26 @@ describe('getModelVersionDelta', () => {
     expect(result.diff).toEqual([
       {
         name: 'a',
-        current: 1,
-        target: 2,
+        current: '10.1.0',
+        target: '10.2.0',
       },
       {
         name: 'b',
-        current: 1,
-        target: 3,
+        current: '10.1.0',
+        target: '10.3.0',
       },
     ]);
   });
 
   it('generates a downward delta', () => {
     const result = getModelVersionDelta({
       currentVersions: {
-        a: 4,
-        b: 2,
+        a: '10.4.0',
+        b: '10.2.0',
       },
       targetVersions: {
-        a: 1,
-        b: 1,
+        a: '10.1.0',
+        b: '7.17.2',
       },
       deletedTypes: [],
     });
@@ -54,26 +54,26 @@ describe('getModelVersionDelta', () => {
     expect(result.diff).toEqual([
       {
         name: 'a',
-        current: 4,
-        target: 1,
+        current: '10.4.0',
+        target: '10.1.0',
       },
       {
         name: 'b',
-        current: 2,
-        target: 1,
+        current: '10.2.0',
+        target: '7.17.2',
       },
     ]);
   });
 
   it('generates a noop delta', () => {
     const result = getModelVersionDelta({
       currentVersions: {
-        a: 4,
-        b: 2,
+        a: '10.4.0',
+        b: '8.9.2',
       },
       targetVersions: {
-        a: 4,
-        b: 2,
+        a: '10.4.0',
+        b: '8.9.2',
       },
       deletedTypes: [],
     });
@@ -85,11 +85,11 @@ describe('getModelVersionDelta', () => {
   it('ignores deleted types', () => {
     const result = getModelVersionDelta({
       currentVersions: {
-        a: 1,
-        b: 3,
+        a: '10.1.0',
+        b: '10.3.0',
       },
       targetVersions: {
-        a: 2,
+        a: '10.2.0',
       },
       deletedTypes: ['b'],
     });
@@ -98,8 +98,8 @@ describe('getModelVersionDelta', () => {
     expect(result.diff).toEqual([
       {
         name: 'a',
-        current: 1,
-        target: 2,
+        current: '10.1.0',
+        target: '10.2.0',
       },
     ]);
   });
@@ -108,12 +108,12 @@ describe('getModelVersionDelta', () => {
     expect(() =>
       getModelVersionDelta({
         currentVersions: {
-          a: 1,
-          b: 2,
+          a: '10.1.0',
+          b: '10.2.0',
         },
         targetVersions: {
-          a: 2,
-          b: 1,
+          a: '10.2.0',
+          b: '10.1.0',
         },
         deletedTypes: [],
       })

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts

@@ -6,12 +6,12 @@
  * Side Public License, v 1.
  */
 
-import type { ModelVersionMap } from './version_map';
-import { compareModelVersions } from './version_compare';
+import type { VirtualVersionMap, VirtualVersion } from './version_map';
+import { compareVirtualVersions } from './version_compare';
 
 interface GetModelVersionDeltaOpts {
-  currentVersions: ModelVersionMap;
-  targetVersions: ModelVersionMap;
+  currentVersions: VirtualVersionMap;
+  targetVersions: VirtualVersionMap;
   deletedTypes: string[];
 }
 
@@ -26,9 +26,9 @@ interface ModelVersionDeltaTypeResult {
   /** the name of the type */
   name: string;
   /** the current version the type is at */
-  current: number;
+  current: VirtualVersion;
   /** the target version the type should go to */
-  target: number;
+  target: VirtualVersion;
 }
 
 /**
@@ -41,7 +41,7 @@ export const getModelVersionDelta = ({
   targetVersions,
   deletedTypes,
 }: GetModelVersionDeltaOpts): ModelVersionDeltaResult => {
-  const compared = compareModelVersions({
+  const compared = compareVirtualVersions({
     indexVersions: currentVersions,
     appVersions: targetVersions,
     deletedTypes,
@@ -78,8 +78,8 @@ const getTypeDelta = ({
   targetVersions,
 }: {
   type: string;
-  currentVersions: ModelVersionMap;
-  targetVersions: ModelVersionMap;
+  currentVersions: VirtualVersionMap;
+  targetVersions: VirtualVersionMap;
 }): ModelVersionDeltaTypeResult => {
   const currentVersion = currentVersions[type];
   const targetVersion = targetVersions[type];

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts

@@ -16,17 +16,21 @@ export {
 export {
   getModelVersionMapForTypes,
   getLatestModelVersion,
+  getCurrentVirtualVersion,
+  getVirtualVersionMap,
+  getLatestMigrationVersion,
   type ModelVersionMap,
+  type VirtualVersionMap,
 } from './version_map';
 export {
-  compareModelVersions,
+  compareVirtualVersions,
   type CompareModelVersionMapParams,
   type CompareModelVersionStatus,
   type CompareModelVersionDetails,
   type CompareModelVersionResult,
 } from './version_compare';
 export {
-  getModelVersionsFromMappings,
-  getModelVersionsFromMappingMeta,
+  getVirtualVersionsFromMappings,
+  getVirtualVersionsFromMappingMeta,
 } from './model_version_from_mappings';
 export { getModelVersionDelta } from './get_version_delta';