Implement first stages of the ZDT migration algorithm (elastic#152219)

## Summary

Part of elastic#150309

This PR implements the first stage (mapping check / update) of the ZDT
algorithm, following the schema from the design document:

<img width="1114" alt="Screenshot 2023-02-28 at 09 23 07"
src="https://user-images.githubusercontent.com/1532934/221795647-4e3d8ad0-18a1-4e2a-8c0d-dd70e66a3c25.png">

Which translates to this:

<img width="700" alt="Screenshot 2023-03-01 at 14 30 50"
src="https://user-images.githubusercontent.com/1532934/222153028-8e2cc6e8-4da2-4ca6-b299-61db6fbb624e.png">

packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts

@@ -45,4 +45,15 @@ export {
   isVirtualModelVersion,
   virtualVersionToModelVersion,
   modelVersionToVirtualVersion,
+  getModelVersionMapForTypes,
+  getLatestModelVersion,
+  type ModelVersionMap,
+  compareModelVersions,
+  type CompareModelVersionMapParams,
+  type CompareModelVersionStatus,
+  type CompareModelVersionDetails,
+  type CompareModelVersionResult,
+  getModelVersionsFromMappings,
+  getModelVersionsFromMappingMeta,
+  getModelVersionDelta,
 } from './src/model_version';

packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts

@@ -57,8 +57,24 @@ export interface IndexMapping {
 
 /** @internal */
 export interface IndexMappingMeta {
-  // A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa')
-  // with each key being a root-level mapping property, and each value being
-  // the md5 hash of that mapping's value when the index was created.
+  /**
+   * A dictionary of key -> md5 hash (e.g. 'dashboard': '24234qdfa3aefa3wa')
+   * with each key being a root-level mapping property, and each value being
+   * the md5 hash of that mapping's value when the index was created.
+   *
+   * @remark: Only defined for indices using the v2 migration algorithm.
+   */
   migrationMappingPropertyHashes?: { [k: string]: string };
+  /**
+   * The current model versions of the mapping of the index.
+   *
+   * @remark: Only defined for indices using the zdt migration algorithm.
+   */
+  mappingVersions?: { [k: string]: number };
+  /**
+   * The current model versions of the documents of the index.
+   *
+   * @remark: Only defined for indices using the zdt migration algorithm.
+   */
+  docVersions?: { [k: string]: number };
 }

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.test.ts

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getModelVersionDelta } from './get_version_delta';
+
+describe('getModelVersionDelta', () => {
+  it('generates an upward delta', () => {
+    const result = getModelVersionDelta({
+      currentVersions: {
+        a: 1,
+        b: 1,
+      },
+      targetVersions: {
+        a: 2,
+        b: 3,
+      },
+      deletedTypes: [],
+    });
+
+    expect(result.status).toEqual('upward');
+    expect(result.diff).toEqual([
+      {
+        name: 'a',
+        current: 1,
+        target: 2,
+      },
+      {
+        name: 'b',
+        current: 1,
+        target: 3,
+      },
+    ]);
+  });
+
+  it('generates a downward delta', () => {
+    const result = getModelVersionDelta({
+      currentVersions: {
+        a: 4,
+        b: 2,
+      },
+      targetVersions: {
+        a: 1,
+        b: 1,
+      },
+      deletedTypes: [],
+    });
+
+    expect(result.status).toEqual('downward');
+    expect(result.diff).toEqual([
+      {
+        name: 'a',
+        current: 4,
+        target: 1,
+      },
+      {
+        name: 'b',
+        current: 2,
+        target: 1,
+      },
+    ]);
+  });
+
+  it('generates a noop delta', () => {
+    const result = getModelVersionDelta({
+      currentVersions: {
+        a: 4,
+        b: 2,
+      },
+      targetVersions: {
+        a: 4,
+        b: 2,
+      },
+      deletedTypes: [],
+    });
+
+    expect(result.status).toEqual('noop');
+    expect(result.diff).toEqual([]);
+  });
+
+  it('ignores deleted types', () => {
+    const result = getModelVersionDelta({
+      currentVersions: {
+        a: 1,
+        b: 3,
+      },
+      targetVersions: {
+        a: 2,
+      },
+      deletedTypes: ['b'],
+    });
+
+    expect(result.status).toEqual('upward');
+    expect(result.diff).toEqual([
+      {
+        name: 'a',
+        current: 1,
+        target: 2,
+      },
+    ]);
+  });
+
+  it('throws if the provided version maps are in conflict', () => {
+    expect(() =>
+      getModelVersionDelta({
+        currentVersions: {
+          a: 1,
+          b: 2,
+        },
+        targetVersions: {
+          a: 2,
+          b: 1,
+        },
+        deletedTypes: [],
+      })
+    ).toThrow();
+  });
+});

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/get_version_delta.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ModelVersionMap } from './version_map';
+import { compareModelVersions } from './version_compare';
+
+interface GetModelVersionDeltaOpts {
+  currentVersions: ModelVersionMap;
+  targetVersions: ModelVersionMap;
+  deletedTypes: string[];
+}
+
+type ModelVersionDeltaResultStatus = 'upward' | 'downward' | 'noop';
+
+interface ModelVersionDeltaResult {
+  status: ModelVersionDeltaResultStatus;
+  diff: ModelVersionDeltaTypeResult[];
+}
+
+interface ModelVersionDeltaTypeResult {
+  /** the name of the type */
+  name: string;
+  /** the current version the type is at */
+  current: number;
+  /** the target version the type should go to */
+  target: number;
+}
+
+/**
+ * Will generate the difference to go from `currentVersions` to `targetVersions`.
+ *
+ * @remarks: will throw if the version maps are in conflict
+ */
+export const getModelVersionDelta = ({
+  currentVersions,
+  targetVersions,
+  deletedTypes,
+}: GetModelVersionDeltaOpts): ModelVersionDeltaResult => {
+  const compared = compareModelVersions({
+    indexVersions: currentVersions,
+    appVersions: targetVersions,
+    deletedTypes,
+  });
+
+  if (compared.status === 'conflict') {
+    throw new Error('Cannot generate model version difference: conflict between versions');
+  }
+
+  const status: ModelVersionDeltaResultStatus =
+    compared.status === 'lesser' ? 'downward' : compared.status === 'greater' ? 'upward' : 'noop';
+
+  const result: ModelVersionDeltaResult = {
+    status,
+    diff: [],
+  };
+
+  if (compared.status === 'greater') {
+    compared.details.greater.forEach((type) => {
+      result.diff.push(getTypeDelta({ type, currentVersions, targetVersions }));
+    });
+  } else if (compared.status === 'lesser') {
+    compared.details.lesser.forEach((type) => {
+      result.diff.push(getTypeDelta({ type, currentVersions, targetVersions }));
+    });
+  }
+
+  return result;
+};
+
+const getTypeDelta = ({
+  type,
+  currentVersions,
+  targetVersions,
+}: {
+  type: string;
+  currentVersions: ModelVersionMap;
+  targetVersions: ModelVersionMap;
+}): ModelVersionDeltaTypeResult => {
+  const currentVersion = currentVersions[type];
+  const targetVersion = targetVersions[type];
+  if (currentVersion === undefined || targetVersion === undefined) {
+    // should never occur given we've been checking consistency numerous times before getting there
+    // but better safe than sorry.
+    throw new Error(
+      `Consistency error: trying to generate delta with missing entry for type ${type}`
+    );
+  }
+  return {
+    name: type,
+    current: currentVersion,
+    target: targetVersion,
+  };
+};

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts

@@ -13,3 +13,20 @@ export {
   modelVersionToVirtualVersion,
   virtualVersionToModelVersion,
 } from './conversion';
+export {
+  getModelVersionMapForTypes,
+  getLatestModelVersion,
+  type ModelVersionMap,
+} from './version_map';
+export {
+  compareModelVersions,
+  type CompareModelVersionMapParams,
+  type CompareModelVersionStatus,
+  type CompareModelVersionDetails,
+  type CompareModelVersionResult,
+} from './version_compare';
+export {
+  getModelVersionsFromMappings,
+  getModelVersionsFromMappingMeta,
+} from './model_version_from_mappings';
+export { getModelVersionDelta } from './get_version_delta';

packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.test.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { IndexMapping, IndexMappingMeta } from '../mappings';
+import { getModelVersionsFromMappings } from './model_version_from_mappings';
+
+describe('getModelVersionsFromMappings', () => {
+  const createIndexMapping = (parts: Partial<IndexMappingMeta> = {}): IndexMapping => ({
+    properties: {},
+    _meta: {
+      ...parts,
+    },
+  });
+
+  it('retrieves the version map from docVersions', () => {
+    const mappings = createIndexMapping({
+      docVersions: {
+        foo: 3,
+        bar: 5,
+      },
+    });
+    const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' });
+
+    expect(versionMap).toEqual({
+      foo: 3,
+      bar: 5,
+    });
+  });
+
+  it('retrieves the version map from mappingVersions', () => {
+    const mappings = createIndexMapping({
+      mappingVersions: {
+        foo: 2,
+        bar: 7,
+      },
+    });
+    const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' });
+
+    expect(versionMap).toEqual({
+      foo: 2,
+      bar: 7,
+    });
+  });
+
+  it('returns undefined for docVersions if meta field is not present', () => {
+    const mappings = createIndexMapping({
+      mappingVersions: {
+        foo: 3,
+        bar: 5,
+      },
+    });
+    const versionMap = getModelVersionsFromMappings({ mappings, source: 'docVersions' });
+
+    expect(versionMap).toBeUndefined();
+  });
+
+  it('returns undefined for mappingVersions if meta field is not present', () => {
+    const mappings = createIndexMapping({
+      docVersions: {
+        foo: 3,
+        bar: 5,
+      },
+    });
+    const versionMap = getModelVersionsFromMappings({ mappings, source: 'mappingVersions' });
+
+    expect(versionMap).toBeUndefined();
+  });
+});