diff --git a/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts b/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts index ce225b86e46d8..c269235019842 100644 --- a/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts +++ b/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts @@ -181,7 +181,7 @@ export interface SavedObjectsClientContract { * @param {object} attributes - the attributes to update * @param {object} options {@link SavedObjectsUpdateOptions} * @prop {integer} options.version - ensures version matches that of persisted object - * @returns the udpated simple saved object + * @returns the updated simple saved object * @deprecated See https://github.com/elastic/kibana/issues/149098 */ update( diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index 6eef9105198bf..7b4ab75c9efca 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export { DEFAULT_INDEX_TYPES_MAP } from './src/constants'; +export { DEFAULT_INDEX_TYPES_MAP, HASH_TO_VERSION_MAP } from './src/constants'; export { LEGACY_URL_ALIAS_TYPE, type LegacyUrlAlias } from './src/legacy_alias'; export { getProperty, @@ -65,6 +65,7 @@ export { getCurrentVirtualVersion, getLatestMigrationVersion, getVirtualVersionMap, + getLatestMappingsVirtualVersionMap, type ModelVersionMap, type VirtualVersionMap, compareVirtualVersions, diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts index c4a3018fdfb37..b001229dd232d 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts @@ -114,3 +114,126 @@ export const DEFAULT_INDEX_TYPES_MAP: IndexTypesMap = { 'workplace_search_telemetry', ], }; + +/** + * In order to be FIPS compliant, the migration logic has switched + * from using hashes (stored in _meta.migrationMappingPropertyHashes) + * to using model versions (stored in _meta.mappingVersions). + * + * This map holds a breakdown of md5 hashes to model versions. + * This allows keeping track of changes in mappings for the different SO types: + * When upgrading from a Kibana version prior to the introduction of model versions for V2, + * the V2 logic will map stored hashes to their corresponding model versions. + * These model versions will then be compared against the ones defined in the typeRegistry, + * in order to determine which types' mappings have changed. + */ +export const HASH_TO_VERSION_MAP = { + 'action_task_params|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'action|0be88ebcc8560a075b6898236a202eb1': '10.0.0', + 'alert|96a5a144778243a9f4fece0e71c2197f': '10.0.0', + 'api_key_pending_invalidation|16f515278a295f6245149ad7c5ddedb7': '10.0.0', + 'apm-custom-dashboards|561810b957ac3c09fcfc08f32f168e97': '10.0.0', + 'apm-indices|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'apm-server-schema|b1d71908f324c17bf744ac72af5038fb': '10.0.0', + 'apm-service-group|2af509c6506f29a858e5a0950577d9fa': '10.0.0', + 'apm-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'app_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'application_usage_daily|43b8830d5d0df85a6823d290885fc9fd': '10.0.0', + 'application_usage_totals|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'canvas-element|7390014e1091044523666d97247392fc': '10.0.0', + 'canvas-workpad-template|ae2673f678281e2c055d764b153e9715': '10.0.0', + 'canvas-workpad|b0a1706d356228dbdcb4a17e6b9eb231': '10.0.0', + 'cases-comments|93535d41ca0279a4a2e5d08acd3f28e3': '10.0.0', + 'cases-configure|c124bd0be4c139d0f0f91fb9eeca8e37': '10.0.0', + 'cases-connector-mappings|a98c33813f364f0b068e8c592ac6ef6d': '10.0.0', + 'cases-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'cases-user-actions|07a6651cf37853dd5d64bfb2c796e102': '10.0.0', + 'cases|8f7dc53b17c272ea19f831537daa082d': '10.1.0', + 'cloud-security-posture-settings|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'config-global|c63748b75f39d0c54de12d12c1ccbc20': '10.0.0', + 'config|c63748b75f39d0c54de12d12c1ccbc20': '10.0.0', + 'connector_token|740b3fd18387d4097dca8d177e6a35c6': '10.0.0', + 'core-usage-stats|3d1b76c39bfb2cc8296b024d73854724': '7.14.1', + 'csp-rule-template|6ee70dc06c0ca3ddffc18222f202ab25': '10.0.0', + 'dashboard|b8aa800aa5e0d975c5e8dc57f03d41f8': '10.2.0', + 'endpoint:user-artifact-manifest|7502b5c5bc923abe8aa5ccfd636e8c3d': '10.0.0', + 'enterprise_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'epm-packages-assets|44621b2f6052ef966da47b7c3a00f33b': '10.0.0', + 'epm-packages|c1e2020399dbebba2448096ca007c668': '10.1.0', + 'event_loop_delays_daily|5df7e292ddd5028e07c1482e130e6654': '10.0.0', + 'event-annotation-group|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0', + 'exception-list-agnostic|8a1defe5981db16792cb9a772e84bb9a': '10.0.0', + 'exception-list|8a1defe5981db16792cb9a772e84bb9a': '10.0.0', + 'file-upload-usage-collection-telemetry|a34fbb8e3263d105044869264860c697': '10.0.0', + 'file|8e9dd7f8a22efdb8fb1c15ed38fde9f6': '10.0.0', + 'fileShare|aa8f7ac2ddf8ab1a91bd34e347046caa': '10.0.0', + 'fleet-fleet-server-host|c28ce72481d1696a9aac8b2cdebcecfa': '10.1.0', + 'fleet-message-signing-keys|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'fleet-preconfiguration-deletion-record|4c36f199189a367e43541f236141204c': '10.0.0', + 'fleet-proxy|05b7a22977de25ce67a77e44dd8e6c33': '10.0.0', + 'fleet-uninstall-tokens|cdb2b655f6b468ecb57d132972425f2e': '10.0.0', + 'graph-workspace|27a94b2edcb0610c6aea54a7c56d7752': '10.0.0', + 'guided-onboarding-guide-state|a3db59c45a3fd2730816d4f53c35c7d9': '10.0.0', + 'guided-onboarding-plugin-state|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'index-pattern|83c02d842fe2a94d14dfa13f7dcd6e87': '10.0.0', + 'infra-custom-dashboards|6eed22cbe14594bad8c076fa864930de': '10.0.0', + 'infrastructure-monitoring-log-view|c50526fc6040c5355ed027d34d05b35c': '10.0.0', + 'infrastructure-ui-source|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'ingest_manager_settings|b91ffb075799c78ffd7dbd51a279c8c9': '10.1.0', + 'ingest-agent-policies|20768dc7ce5eced3eb309e50d8a6cf76': '10.0.0', + 'ingest-download-sources|0b0f6828e59805bd07a650d80817c342': '10.0.0', + 'ingest-outputs|b1237f7fdc0967709e75d65d208ace05': '10.6.0', + 'ingest-package-policies|a1a074bad36e68d54f98d2158d60f879': '10.0.0', + 'inventory-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'kql-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'legacy-url-alias|0750774cf16475f88f2361e99cc5c8f0': '8.2.0', + 'lens-ui-telemetry|509bfa5978586998e05f9e303c07a327': '10.0.0', + 'lens|b0da10d5ab9ebd81d61700737ddc76c9': '10.0.0', + 'links|3378bb9b651572865d9f61f5b448e415': '10.0.0', + 'maintenance-window|a58ac2ef53ff5103710093e669dcc1d8': '10.0.0', + 'map|9134b47593116d7953f6adba096fc463': '10.0.0', + 'metrics-data-source|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'metrics-explorer-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'ml-job|3bb64c31915acf93fc724af137a0891b': '10.0.0', + 'ml-module|f6c6b7b7ebdca4154246923f24d6340d': '10.0.0', + 'ml-trained-model|d2f03c1a5dd038fa58af14a56944312b': '10.0.0', + 'monitoring-telemetry|2669d5ec15e82391cf58df4294ee9c68': '10.0.0', + 'observability-onboarding-state|a4e5c9d018037114140bdb1647c2d568': '10.0.0', + 'osquery-manager-usage-metric|4dc4f647d27247c002f56f22742175fe': '10.0.0', + 'osquery-pack-asset|fe0dfa13c4c24ac37ce1aec04c560a81': '10.1.0', + 'osquery-pack|6bc20973adab06f00156cbc4578a19ac': '10.1.0', + 'osquery-saved-query|a05ec7031231a4b71bfb4493a07b2dc5': '10.1.0', + 'policy-settings-protection-updates-note|37d4035a1dc3c5e58f1b519f99093f21': '10.0.0', + 'query|aa811b49f48906074f59110bfa83984c': '10.2.0', + 'risk-engine-configuration|431232781a82926aad5b1fd849715c0f': '10.1.0', + 'rules-settings|001f60645e96c71520214b57f3ea7590': '10.0.0', + 'sample-data-telemetry|7d3cfeb915303c9641c59681967ffeb4': '10.0.0', + 'search-session|fea3612a90b81672991617646f229a61': '10.0.0', + 'search-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'search|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0', + 'security-rule|9d9d11b97e3aaa87fbaefbace2b5c25f': '10.0.0', + 'security-solution-signals-migration|4060b5a63dddfd54d2cd56450882cc0e': '10.0.0', + 'siem-detection-engine-rule-actions|f5c218f837bac10ab2c3980555176cf9': '10.0.0', + 'siem-ui-timeline-note|28393dfdeb4e4413393eb5f7ec8c5436': '10.0.0', + 'siem-ui-timeline-pinned-event|293fce142548281599060e07ad2c9ddb': '10.0.0', + 'siem-ui-timeline|f6739fd4b17646a6c86321a746c247ef': '10.1.0', + 'slo|dc7f35c0cf07d71bb36f154996fe10c6': '10.1.0', + 'space|c3aec2a5d4afcb75554fed96411170e1': '10.0.0', + 'spaces-usage-stats|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'synthetics-monitor|50b48ccda9f2f7d73d31fd50c41bf305': '10.0.0', + 'synthetics-param|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'synthetics-privates-locations|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'tag|83d55da58f6530f7055415717ec06474': '10.0.0', + 'task|b4a368fd68cd32ef6990877634639db6': '10.0.0', + 'telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'threshold-explorer-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'ui-metric|0d409297dc5ebe1e3a1da691c6ee32e3': '10.0.0', + 'upgrade-assistant-ml-upgrade-operation|3caf305ad2da94d80d49453b0970156d': '10.0.0', + 'upgrade-assistant-reindex-operation|6d1e2aca91767634e1829c30f20f6b16': '10.0.0', + 'uptime-dynamic-settings|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', + 'uptime-synthetics-api-key|c3178f0fde61e18d3530ba9a70bc278a': '10.0.0', + 'url|a37dbae7645ad5811045f4dd3dc1c0a8': '10.0.0', + 'usage-counters|8cc260bdceffec4ffc3ad165c97dc1b4': '10.0.0', + 'visualization|4891c012863513388881fc109fec4809': '10.0.0', + 'workplace_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index a8c74646de1b1..37ec0ac998db1 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -66,6 +66,7 @@ export interface V2AlgoIndexMappingMeta { * the md5 hash of that mapping's value when the index was created. * * @remark: Only defined for indices using the v2 migration algorithm. + * @deprecated Replaced by mappingVersions (FIPS-compliant initiative) */ migrationMappingPropertyHashes?: { [k: string]: string }; /** @@ -74,20 +75,20 @@ export interface V2AlgoIndexMappingMeta { * @remark: Only defined for indices using the v2 migration algorithm. */ indexTypesMap?: IndexTypesMap; + /** + * The current virtual version of the mapping of the index. + */ + mappingVersions?: { [k: string]: string }; } /** @internal */ export interface ZdtAlgoIndexMappingMeta { /** * The current virtual version of the mapping of the index. - * - * @remark: Only defined for indices using the zdt migration algorithm. */ mappingVersions?: { [k: string]: string }; /** * The current virtual versions of the documents of the index. - * - * @remark: Only defined for indices using the zdt migration algorithm. */ docVersions?: { [k: string]: string }; /** diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts index 3c189977731e3..8becd4d277899 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/index.ts @@ -19,6 +19,7 @@ export { getCurrentVirtualVersion, getVirtualVersionMap, getLatestMigrationVersion, + getLatestMappingsVirtualVersionMap, type ModelVersionMap, type VirtualVersionMap, } from './version_map'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts index 16a5eb1d6f357..2e6227eb5272d 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.test.ts @@ -13,6 +13,9 @@ import { getLatestMigrationVersion, getCurrentVirtualVersion, getVirtualVersionMap, + getLatestMappingsVersionNumber, + getLatestMappingsModelVersion, + getLatestMappingsVirtualVersionMap, } from './version_map'; describe('ModelVersion map utilities', () => { @@ -28,6 +31,24 @@ describe('ModelVersion map utilities', () => { changes: [], }); + const dummyModelVersionWithMappingsChanges = (): SavedObjectsModelVersion => ({ + changes: [ + { + type: 'mappings_addition', + addedMappings: {}, + }, + ], + }); + + const dummyModelVersionWithDataRemoval = (): SavedObjectsModelVersion => ({ + changes: [ + { + type: 'data_removal', + removedAttributePaths: ['some.attribute'], + }, + ], + }); + const dummyMigration = jest.fn(); describe('getLatestModelVersion', () => { @@ -259,4 +280,146 @@ describe('ModelVersion map utilities', () => { }); }); }); + + describe('getLatestMappingsVersionNumber', () => { + it('returns 0 when no model versions are registered', () => { + expect(getLatestMappingsVersionNumber(buildType({ modelVersions: {} }))).toEqual(0); + expect(getLatestMappingsVersionNumber(buildType({ modelVersions: undefined }))).toEqual(0); + }); + + it('throws if an invalid version is provided', () => { + expect(() => + getLatestMappingsVersionNumber( + buildType({ + modelVersions: { + foo: dummyModelVersionWithMappingsChanges(), + }, + }) + ) + ).toThrow(); + }); + + it('returns the latest version that brings mappings changes', () => { + expect( + getLatestMappingsVersionNumber( + buildType({ + modelVersions: { + '1': dummyModelVersion(), + '2': dummyModelVersionWithMappingsChanges(), + '3': dummyModelVersionWithDataRemoval(), + }, + }) + ) + ).toEqual(2); + }); + + it('accepts provider functions', () => { + expect( + getLatestMappingsVersionNumber( + buildType({ + modelVersions: () => ({ + '1': dummyModelVersion(), + '2': dummyModelVersionWithMappingsChanges(), + '3': dummyModelVersionWithDataRemoval(), + }), + }) + ) + ).toEqual(2); + }); + + it('supports unordered maps', () => { + expect( + getLatestMappingsVersionNumber( + buildType({ + modelVersions: { + '3': dummyModelVersionWithDataRemoval(), + '1': dummyModelVersion(), + '2': dummyModelVersionWithMappingsChanges(), + }, + }) + ) + ).toEqual(2); + }); + }); + + describe('getLatestMappingsModelVersion', () => { + it('returns the latest registered migration if switchToModelVersionAt is unset', () => { + expect( + getLatestMappingsModelVersion( + buildType({ + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersionWithMappingsChanges(), + 2: dummyModelVersion(), + }, + }) + ) + ).toEqual('8.6.0'); + }); + + it('returns the virtual version of the latest model version if switchToModelVersionAt is set', () => { + expect( + getLatestMappingsModelVersion( + buildType({ + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersionWithMappingsChanges(), + 2: dummyModelVersion(), + }, + }) + ) + ).toEqual('10.1.0'); + }); + }); + + describe('getLatestMappingsVirtualVersionMap', () => { + it('returns the virtual version for each of the provided types', () => { + expect( + getLatestMappingsVirtualVersionMap([ + buildType({ + name: 'foo', + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersionWithMappingsChanges(), + 2: dummyModelVersion(), + }, + }), + buildType({ + name: 'bar', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + modelVersions: { + 1: dummyModelVersionWithMappingsChanges(), + 2: dummyModelVersion(), + }, + }), + buildType({ + name: 'dolly', + switchToModelVersionAt: '8.7.0', + migrations: { + '7.17.2': dummyMigration, + '8.6.0': dummyMigration, + }, + }), + ]) + ).toEqual({ + foo: '10.1.0', + bar: '8.6.0', + dolly: '10.0.0', + }); + }); + }); }); diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts index aafac41df794a..46a415fdc7870 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/version_map.ts @@ -86,3 +86,47 @@ export const getVirtualVersionMap = (types: SavedObjectsType[]): VirtualVersionM return versionMap; }, {}); }; + +/** + * Returns the latest version number that includes changes in the mappings, for the given type. + * If none of the versions are updating the mappings, it will return 0 + */ +export const getLatestMappingsVersionNumber = (type: SavedObjectsType): number => { + const versionMap = + typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {}; + return Object.entries(versionMap) + .filter(([version, info]) => + info.changes?.some((change) => change.type === 'mappings_addition') + ) + .reduce((memo, [current]) => { + return Math.max(memo, assertValidModelVersion(current)); + }, 0); +}; + +/** + * Returns the latest model version that includes changes in the mappings, for the given type. + * It will either be a model version if the type + * already switched to using them (switchToModelVersionAt is set), + * or the latest migration version for the type otherwise. + */ +export const getLatestMappingsModelVersion = (type: SavedObjectsType): string => { + if (type.switchToModelVersionAt) { + const modelVersion = getLatestMappingsVersionNumber(type); + return modelVersionToVirtualVersion(modelVersion); + } else { + return getLatestMigrationVersion(type); + } +}; + +/** + * Returns a map of virtual model version for the given types. + * See {@link getLatestMappingsModelVersion} + */ +export const getLatestMappingsVirtualVersionMap = ( + types: SavedObjectsType[] +): VirtualVersionMap => { + return types.reduce((versionMap, type) => { + versionMap[type.name] = getLatestMappingsModelVersion(type); + return versionMap; + }, {}); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/kibana_migrator.test.ts.snap b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/kibana_migrator.test.ts.snap index b035c64617ca2..420a051eb47d9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/kibana_migrator.test.ts.snap +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/kibana_migrator.test.ts.snap @@ -2,22 +2,6 @@ exports[`KibanaMigrator getActiveMappings returns full index mappings w/ core properties 1`] = ` Object { - "_meta": Object { - "migrationMappingPropertyHashes": Object { - "amap": "510f1f0adb69830cf8a1c5ce2923ed82", - "bmap": "510f1f0adb69830cf8a1c5ce2923ed82", - "coreMigrationVersion": "2f4316de49999235636386fe51dc06c1", - "created_at": "00da57df13e94e9d98437d13ace4bfe0", - "managed": "88cf246b441a6362458cb6a56ca3f7d7", - "namespace": "2f4316de49999235636386fe51dc06c1", - "namespaces": "2f4316de49999235636386fe51dc06c1", - "originId": "2f4316de49999235636386fe51dc06c1", - "references": "7997cf5a56cc02bdc9c93361bde732b0", - "type": "2f4316de49999235636386fe51dc06c1", - "typeMigrationVersion": "539e3ecebb3abc1133618094cc3b7ae7", - "updated_at": "00da57df13e94e9d98437d13ace4bfe0", - }, - }, "dynamic": "strict", "properties": Object { "amap": Object { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap index f709e0137730b..9f59f9d5ea6d9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap @@ -141,7 +141,20 @@ Object { ], }, }, + "hashToVersionMap": Object { + "task|someHash": "10.1.0", + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + "typeD|someHash": "10.1.0", + "typeE|someHash": "10.1.0", + }, "indexPrefix": ".my-so-index", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -158,6 +171,7 @@ Object { }, "kibanaVersion": "7.11.0", "knownTypes": Array [], + "latestMappingsVersions": Object {}, "legacyIndex": ".my-so-index", "logs": Array [ Object { @@ -188,22 +202,6 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { - "_meta": Object { - "indexTypesMap": Object { - ".kibana": Array [ - "typeA", - "typeB", - "typeC", - ], - ".kibana_cases": Array [ - "typeD", - "typeE", - ], - ".kibana_task_manager": Array [ - "task", - ], - }, - }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -365,7 +363,20 @@ Object { ], }, }, + "hashToVersionMap": Object { + "task|someHash": "10.1.0", + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + "typeD|someHash": "10.1.0", + "typeE|someHash": "10.1.0", + }, "indexPrefix": ".my-so-index", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -382,6 +393,7 @@ Object { }, "kibanaVersion": "7.11.0", "knownTypes": Array [], + "latestMappingsVersions": Object {}, "legacyIndex": ".my-so-index", "logs": Array [ Object { @@ -416,22 +428,6 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { - "_meta": Object { - "indexTypesMap": Object { - ".kibana": Array [ - "typeA", - "typeB", - "typeC", - ], - ".kibana_cases": Array [ - "typeD", - "typeE", - ], - ".kibana_task_manager": Array [ - "task", - ], - }, - }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -593,7 +589,20 @@ Object { ], }, }, + "hashToVersionMap": Object { + "task|someHash": "10.1.0", + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + "typeD|someHash": "10.1.0", + "typeE|someHash": "10.1.0", + }, "indexPrefix": ".my-so-index", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -610,6 +619,7 @@ Object { }, "kibanaVersion": "7.11.0", "knownTypes": Array [], + "latestMappingsVersions": Object {}, "legacyIndex": ".my-so-index", "logs": Array [ Object { @@ -648,22 +658,6 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { - "_meta": Object { - "indexTypesMap": Object { - ".kibana": Array [ - "typeA", - "typeB", - "typeC", - ], - ".kibana_cases": Array [ - "typeD", - "typeE", - ], - ".kibana_task_manager": Array [ - "task", - ], - }, - }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -825,7 +819,20 @@ Object { ], }, }, + "hashToVersionMap": Object { + "task|someHash": "10.1.0", + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + "typeD|someHash": "10.1.0", + "typeE|someHash": "10.1.0", + }, "indexPrefix": ".my-so-index", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -842,6 +849,7 @@ Object { }, "kibanaVersion": "7.11.0", "knownTypes": Array [], + "latestMappingsVersions": Object {}, "legacyIndex": ".my-so-index", "logs": Array [ Object { @@ -884,22 +892,6 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { - "_meta": Object { - "indexTypesMap": Object { - ".kibana": Array [ - "typeA", - "typeB", - "typeC", - ], - ".kibana_cases": Array [ - "typeD", - "typeE", - ], - ".kibana_task_manager": Array [ - "task", - ], - }, - }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -1093,7 +1085,20 @@ Object { ], }, }, + "hashToVersionMap": Object { + "task|someHash": "10.1.0", + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + "typeD|someHash": "10.1.0", + "typeE|someHash": "10.1.0", + }, "indexPrefix": ".my-so-index", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -1110,6 +1115,7 @@ Object { }, "kibanaVersion": "7.11.0", "knownTypes": Array [], + "latestMappingsVersions": Object {}, "legacyIndex": ".my-so-index", "logs": Array [ Object { @@ -1145,22 +1151,6 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { - "_meta": Object { - "indexTypesMap": Object { - ".kibana": Array [ - "typeA", - "typeB", - "typeC", - ], - ".kibana_cases": Array [ - "typeD", - "typeE", - ], - ".kibana_task_manager": Array [ - "task", - ], - }, - }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -1328,7 +1318,20 @@ Object { ], }, }, + "hashToVersionMap": Object { + "task|someHash": "10.1.0", + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + "typeD|someHash": "10.1.0", + "typeE|someHash": "10.1.0", + }, "indexPrefix": ".my-so-index", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -1345,6 +1348,7 @@ Object { }, "kibanaVersion": "7.11.0", "knownTypes": Array [], + "latestMappingsVersions": Object {}, "legacyIndex": ".my-so-index", "logs": Array [ Object { @@ -1384,22 +1388,6 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { - "_meta": Object { - "indexTypesMap": Object { - ".kibana": Array [ - "typeA", - "typeB", - "typeC", - ], - ".kibana_cases": Array [ - "typeD", - "typeE", - ], - ".kibana_task_manager": Array [ - "task", - ], - }, - }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts index 4e04cc2a70539..b53ace7e6843a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.test.ts @@ -9,28 +9,23 @@ import * as Either from 'fp-ts/lib/Either'; import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; -import { - checkTargetMappings, - type ComparedMappingsChanged, - type ComparedMappingsMatch, -} from './check_target_mappings'; -import { getUpdatedHashes } from '../core/build_active_mappings'; +import { checkTargetMappings } from './check_target_mappings'; +import { getBaseMappings } from '../core'; -jest.mock('../core/build_active_mappings'); - -const getUpdatedHashesMock = getUpdatedHashes as jest.MockedFn; +const indexTypes = ['type1', 'type2', 'type3']; const properties: SavedObjectsMappingProperties = { + ...getBaseMappings().properties, type1: { type: 'long' }, type2: { type: 'long' }, }; const migrationMappingPropertyHashes = { - type1: 'type1Hash', - type2: 'type2Hash', + type1: 'someHash', + type2: 'anotherHash', }; -const expectedMappings: IndexMapping = { +const legacyMappings: IndexMapping = { properties, dynamic: 'strict', _meta: { @@ -38,96 +33,231 @@ const expectedMappings: IndexMapping = { }, }; +const outdatedModelVersions = { + type1: '10.1.0', + type2: '10.2.0', + type3: '10.4.0', +}; + +const modelVersions = { + type1: '10.1.0', + type2: '10.3.0', + type3: '10.5.0', +}; + +const latestMappingsVersions = { + type1: '10.1.0', + type2: '10.2.0', // type is on '10.3.0' but its mappings were last updated on 10.2.0 + type3: '10.5.0', +}; + +const appMappings: IndexMapping = { + properties, + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes, // deprecated, but preserved to facilitate rollback + mappingVersions: modelVersions, + }, +}; + describe('checkTargetMappings', () => { beforeEach(() => { jest.clearAllMocks(); }); - describe('when actual mappings are incomplete', () => { - it("returns 'actual_mappings_incomplete' if actual mappings are not defined", async () => { + describe('when index mappings are missing required properties', () => { + it("returns 'index_mappings_incomplete' if index mappings are not defined", async () => { const task = checkTargetMappings({ - expectedMappings, + indexTypes, + appMappings, + latestMappingsVersions, }); const result = await task(); - expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const })); + expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const })); }); - it("returns 'actual_mappings_incomplete' if actual mappings do not define _meta", async () => { + it("returns 'index_mappings_incomplete' if index mappings do not define _meta", async () => { const task = checkTargetMappings({ - expectedMappings, - actualMappings: { + indexTypes, + appMappings, + indexMappings: { properties, dynamic: 'strict', }, + latestMappingsVersions, }); const result = await task(); - expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const })); + expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const })); }); - it("returns 'actual_mappings_incomplete' if actual mappings do not define migrationMappingPropertyHashes", async () => { + it("returns 'index_mappings_incomplete' if index mappings do not define migrationMappingPropertyHashes nor mappingVersions", async () => { const task = checkTargetMappings({ - expectedMappings, - actualMappings: { + indexTypes, + appMappings, + indexMappings: { properties, dynamic: 'strict', _meta: {}, }, + latestMappingsVersions, }); const result = await task(); - expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const })); + expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const })); }); - it("returns 'actual_mappings_incomplete' if actual mappings define a different value for 'dynamic' property", async () => { + it("returns 'index_mappings_incomplete' if index mappings define a different value for 'dynamic' property", async () => { const task = checkTargetMappings({ - expectedMappings, - actualMappings: { + indexTypes, + appMappings, + indexMappings: { properties, dynamic: false, - _meta: { migrationMappingPropertyHashes }, + _meta: appMappings._meta, }, + latestMappingsVersions, }); const result = await task(); - expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const })); + expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const })); }); }); - describe('when actual mappings are complete', () => { - describe('and mappings do not match', () => { - it('returns the lists of changed root fields and types', async () => { + describe('when index mappings have all required properties', () => { + describe('when some core properties (aka root fields) have changed', () => { + it('returns the list of fields that have changed', async () => { const task = checkTargetMappings({ - expectedMappings, - actualMappings: expectedMappings, + indexTypes, + appMappings, + indexMappings: { + ...legacyMappings, + properties: { + ...legacyMappings.properties, + references: { + properties: { + ...legacyMappings.properties.references.properties, + description: { type: 'text' }, + }, + }, + }, + }, + latestMappingsVersions, }); - getUpdatedHashesMock.mockReturnValueOnce(['type1', 'type2', 'someRootField']); - const result = await task(); - const expected: ComparedMappingsChanged = { - type: 'compared_mappings_changed' as const, - updatedHashes: ['type1', 'type2', 'someRootField'], - }; - expect(result).toEqual(Either.left(expected)); + expect(result).toEqual( + Either.left({ + type: 'root_fields_changed' as const, + updatedFields: ['references'], + }) + ); }); }); - describe('and mappings match', () => { - it('returns a compared_mappings_match response', async () => { - const task = checkTargetMappings({ - expectedMappings, - actualMappings: expectedMappings, + describe('when core properties have NOT changed', () => { + describe('when index mappings ONLY contain the legacy hashes', () => { + describe('and legacy hashes match the current model versions', () => { + it('returns a compared_mappings_match response', async () => { + const task = checkTargetMappings({ + indexTypes, + appMappings, + indexMappings: legacyMappings, + hashToVersionMap: { + 'type1|someHash': '10.1.0', + 'type2|anotherHash': '10.2.0', + // type 3 is a new type + }, + latestMappingsVersions, + }); + + const result = await task(); + expect(result).toEqual( + Either.right({ + type: 'compared_mappings_match' as const, + }) + ); + }); }); - getUpdatedHashesMock.mockReturnValueOnce([]); + describe('and legacy hashes do NOT match the current model versions', () => { + it('returns the list of updated SO types', async () => { + const task = checkTargetMappings({ + indexTypes, + appMappings, + indexMappings: legacyMappings, + hashToVersionMap: { + 'type1|someHash': '10.1.0', + 'type2|anotherHash': '10.1.0', // type2's mappings were updated on 10.2.0 + }, + latestMappingsVersions, + }); - const result = await task(); - const expected: ComparedMappingsMatch = { - type: 'compared_mappings_match' as const, - }; - expect(result).toEqual(Either.right(expected)); + const result = await task(); + expect(result).toEqual( + Either.left({ + type: 'types_changed' as const, + updatedTypes: ['type2'], + }) + ); + }); + }); + }); + + describe('when index mappings contain the mappingVersions', () => { + describe('and mappingVersions match', () => { + it('returns a compared_mappings_match response', async () => { + const task = checkTargetMappings({ + indexTypes, + appMappings, + indexMappings: { + ...appMappings, + _meta: { + ...appMappings._meta, + mappingVersions: { + type1: '10.1.0', + type2: '10.2.0', // type 2 was still on 10.2.0, but 10.3.0 does not bring mappings changes + type3: '10.5.0', + }, + }, + }, + latestMappingsVersions, + }); + + const result = await task(); + expect(result).toEqual( + Either.right({ + type: 'compared_mappings_match' as const, + }) + ); + }); + }); + + describe('and mappingVersions do NOT match', () => { + it('returns the list of updated SO types', async () => { + const task = checkTargetMappings({ + indexTypes, + appMappings, + indexMappings: { + properties, + dynamic: 'strict', + _meta: { + mappingVersions: outdatedModelVersions, + }, + }, + latestMappingsVersions, + }); + + const result = await task(); + expect(result).toEqual( + Either.left({ + type: 'types_changed' as const, + updatedTypes: ['type3'], + }) + ); + }); + }); }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts index 576459beadd74..ac9f047a081e5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/check_target_mappings.ts @@ -8,13 +8,16 @@ import * as Either from 'fp-ts/lib/Either'; import * as TaskEither from 'fp-ts/lib/TaskEither'; -import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; -import { getUpdatedHashes } from '../core/build_active_mappings'; +import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import { getUpdatedRootFields, getUpdatedTypes } from '../core/compare_mappings'; /** @internal */ export interface CheckTargetMappingsParams { - actualMappings?: IndexMapping; - expectedMappings: IndexMapping; + indexTypes: string[]; + indexMappings?: IndexMapping; + appMappings: IndexMapping; + latestMappingsVersions: VirtualVersionMap; + hashToVersionMap?: Record; } /** @internal */ @@ -22,40 +25,60 @@ export interface ComparedMappingsMatch { type: 'compared_mappings_match'; } -export interface ActualMappingsIncomplete { - type: 'actual_mappings_incomplete'; +export interface IndexMappingsIncomplete { + type: 'index_mappings_incomplete'; } -export interface ComparedMappingsChanged { - type: 'compared_mappings_changed'; - updatedHashes: string[]; +export interface RootFieldsChanged { + type: 'root_fields_changed'; + updatedFields: string[]; +} + +export interface TypesChanged { + type: 'types_changed'; + updatedTypes: string[]; } export const checkTargetMappings = ({ - actualMappings, - expectedMappings, + indexTypes, + indexMappings, + appMappings, + latestMappingsVersions, + hashToVersionMap = {}, }: CheckTargetMappingsParams): TaskEither.TaskEither< - ActualMappingsIncomplete | ComparedMappingsChanged, + IndexMappingsIncomplete | RootFieldsChanged | TypesChanged, ComparedMappingsMatch > => async () => { if ( - !actualMappings?._meta?.migrationMappingPropertyHashes || - actualMappings.dynamic !== expectedMappings.dynamic + (!indexMappings?._meta?.migrationMappingPropertyHashes && + !indexMappings?._meta?.mappingVersions) || + indexMappings.dynamic !== appMappings.dynamic ) { - return Either.left({ type: 'actual_mappings_incomplete' as const }); + return Either.left({ type: 'index_mappings_incomplete' as const }); + } + + const updatedFields = getUpdatedRootFields(indexMappings); + + if (updatedFields.length) { + return Either.left({ + type: 'root_fields_changed', + updatedFields, + }); } - const updatedHashes = getUpdatedHashes({ - actual: actualMappings, - expected: expectedMappings, + const updatedTypes = getUpdatedTypes({ + indexTypes, + indexMeta: indexMappings?._meta, + latestMappingsVersions, + hashToVersionMap, }); - if (updatedHashes.length) { + if (updatedTypes.length) { return Either.left({ - type: 'compared_mappings_changed' as const, - updatedHashes, + type: 'types_changed' as const, + updatedTypes, }); } else { return Either.right({ type: 'compared_mappings_match' as const }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index c06cd5f05c135..afdeb6628a1ba 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -108,7 +108,11 @@ import type { UnknownDocsFound } from './check_for_unknown_docs'; import type { IncompatibleClusterRoutingAllocation } from './initialize_action'; import type { ClusterShardLimitExceeded } from './create_index'; import type { SynchronizationFailed } from './synchronize_migrators'; -import type { ActualMappingsIncomplete, ComparedMappingsChanged } from './check_target_mappings'; +import type { + IndexMappingsIncomplete, + RootFieldsChanged, + TypesChanged, +} from './check_target_mappings'; export type { CheckForUnknownDocsParams, @@ -182,8 +186,9 @@ export interface ActionErrorTypeMap { cluster_shard_limit_exceeded: ClusterShardLimitExceeded; es_response_too_large: EsResponseTooLargeError; synchronization_failed: SynchronizationFailed; - actual_mappings_incomplete: ActualMappingsIncomplete; - compared_mappings_changed: ComparedMappingsChanged; + index_mappings_incomplete: IndexMappingsIncomplete; + root_fields_changed: RootFieldsChanged; + types_changed: TypesChanged; operation_not_supported: OperationNotSupported; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_mappings.ts index 159cd04c7501b..32774208c7142 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_mappings.ts @@ -27,8 +27,11 @@ export interface IncompatibleMappingException { } /** - * Updates an index's mappings and runs an pickupUpdatedMappings task so that the mapping - * changes are "picked up". Returns a taskId to track progress. + * Attempts to update the SO index mappings. + * Includes an automatic retry mechanism for retriable errors. + * Returns an 'update_mappings_succeeded' upon success. + * If changes in the mappings are NOT compatible and the update fails on ES side, + * this method will return an 'incompatible_mapping_exception'. */ export const updateMappings = ({ client, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts index 54814693d599f..43fbd60a192a5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.test.ts @@ -13,6 +13,7 @@ import { updateSourceMappingsProperties, type UpdateSourceMappingsPropertiesParams, } from './update_source_mappings_properties'; +import { getBaseMappings } from '../core'; describe('updateSourceMappingsProperties', () => { let client: ReturnType; @@ -22,11 +23,18 @@ describe('updateSourceMappingsProperties', () => { client = elasticsearchClientMock.createInternalClient(); params = { client, + indexTypes: ['a', 'b', 'c'], + latestMappingsVersions: { + a: '10.1.0', + b: '10.2.0', + c: '10.5.0', + }, sourceIndex: '.kibana_8.7.0_001', - sourceMappings: { + indexMappings: { properties: { a: { type: 'keyword' }, b: { type: 'long' }, + ...getBaseMappings().properties, }, _meta: { migrationMappingPropertyHashes: { @@ -35,23 +43,33 @@ describe('updateSourceMappingsProperties', () => { }, }, }, - targetMappings: { + appMappings: { properties: { a: { type: 'keyword' }, c: { type: 'long' }, + ...getBaseMappings().properties, }, _meta: { - migrationMappingPropertyHashes: { - a: '000', - c: '222', + mappingVersions: { + a: '10.1.0', + b: '10.3.0', + c: '10.5.0', }, }, }, + hashToVersionMap: { + 'a|000': '10.1.0', + 'b|111': '10.1.0', + }, }; }); it('should not update mappings when there are no changes', async () => { - const sameMappingsParams = chain(params).set('targetMappings', params.sourceMappings).value(); + // we overwrite the app mappings to have the "unchanged" values with respect to the index mappings + const sameMappingsParams = chain(params) + // even if the app versions are more recent, we emulate a scenario where mappings haven NOT changed + .set('latestMappingsVersions', { a: '10.1.0', b: '10.1.0', c: '10.1.0' }) + .value(); const result = await updateSourceMappingsProperties(sameMappingsParams)(); expect(client.indices.putMapping).not.toHaveBeenCalled(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.ts index 51113f2892095..3a73f948efa20 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/update_source_mappings_properties.ts @@ -10,8 +10,8 @@ import { omit } from 'lodash'; import * as TaskEither from 'fp-ts/lib/TaskEither'; import { pipe } from 'fp-ts/lib/pipeable'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; -import { diffMappings } from '../core/build_active_mappings'; +import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import { diffMappings } from '../core/diff_mappings'; import type { RetryableEsClientError } from './catch_retryable_es_client_errors'; import { updateMappings } from './update_mappings'; import type { IncompatibleMappingException } from './update_mappings'; @@ -20,8 +20,11 @@ import type { IncompatibleMappingException } from './update_mappings'; export interface UpdateSourceMappingsPropertiesParams { client: ElasticsearchClient; sourceIndex: string; - sourceMappings: IndexMapping; - targetMappings: IndexMapping; + indexMappings: IndexMapping; + appMappings: IndexMapping; + indexTypes: string[]; + latestMappingsVersions: VirtualVersionMap; + hashToVersionMap: Record; } /** @@ -31,14 +34,23 @@ export interface UpdateSourceMappingsPropertiesParams { export const updateSourceMappingsProperties = ({ client, sourceIndex, - sourceMappings, - targetMappings, + indexMappings, + appMappings, + indexTypes, + latestMappingsVersions, + hashToVersionMap, }: UpdateSourceMappingsPropertiesParams): TaskEither.TaskEither< RetryableEsClientError | IncompatibleMappingException, 'update_mappings_succeeded' > => { return pipe( - diffMappings(sourceMappings, targetMappings), + diffMappings({ + indexMappings, + appMappings, + indexTypes, + latestMappingsVersions, + hashToVersionMap, + }), TaskEither.fromPredicate( (changes) => !!changes, () => 'update_mappings_succeeded' as const @@ -48,7 +60,7 @@ export const updateSourceMappingsProperties = ({ updateMappings({ client, index: sourceIndex, - mappings: omit(targetMappings, ['_meta']), // ._meta property will be updated on a later step + mappings: omit(appMappings, ['_meta']), // ._meta property will be updated on a later step }) ) ); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/__snapshots__/build_active_mappings.test.ts.snap b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/__snapshots__/build_active_mappings.test.ts.snap index 9941f2a70e696..2ae6c3e5685ff 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/__snapshots__/build_active_mappings.test.ts.snap +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/__snapshots__/build_active_mappings.test.ts.snap @@ -2,22 +2,6 @@ exports[`buildActiveMappings combines all mappings and includes core mappings 1`] = ` Object { - "_meta": Object { - "migrationMappingPropertyHashes": Object { - "aaa": "625b32086eb1d1203564cf85062dd22e", - "bbb": "18c78c995965207ed3f6e7fc5c6e55fe", - "coreMigrationVersion": "2f4316de49999235636386fe51dc06c1", - "created_at": "00da57df13e94e9d98437d13ace4bfe0", - "managed": "88cf246b441a6362458cb6a56ca3f7d7", - "namespace": "2f4316de49999235636386fe51dc06c1", - "namespaces": "2f4316de49999235636386fe51dc06c1", - "originId": "2f4316de49999235636386fe51dc06c1", - "references": "7997cf5a56cc02bdc9c93361bde732b0", - "type": "2f4316de49999235636386fe51dc06c1", - "typeMigrationVersion": "539e3ecebb3abc1133618094cc3b7ae7", - "updated_at": "00da57df13e94e9d98437d13ace4bfe0", - }, - }, "dynamic": "strict", "properties": Object { "aaa": Object { @@ -73,23 +57,6 @@ Object { exports[`buildActiveMappings handles the \`dynamic\` property of types 1`] = ` Object { - "_meta": Object { - "migrationMappingPropertyHashes": Object { - "coreMigrationVersion": "2f4316de49999235636386fe51dc06c1", - "created_at": "00da57df13e94e9d98437d13ace4bfe0", - "firstType": "635418ab953d81d93f1190b70a8d3f57", - "managed": "88cf246b441a6362458cb6a56ca3f7d7", - "namespace": "2f4316de49999235636386fe51dc06c1", - "namespaces": "2f4316de49999235636386fe51dc06c1", - "originId": "2f4316de49999235636386fe51dc06c1", - "references": "7997cf5a56cc02bdc9c93361bde732b0", - "secondType": "72d57924f415fbadb3ee293b67d233ab", - "thirdType": "510f1f0adb69830cf8a1c5ce2923ed82", - "type": "2f4316de49999235636386fe51dc06c1", - "typeMigrationVersion": "539e3ecebb3abc1133618094cc3b7ae7", - "updated_at": "00da57df13e94e9d98437d13ace4bfe0", - }, - }, "dynamic": "strict", "properties": Object { "coreMigrationVersion": Object { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.test.ts index 7139e1e60b584..f2758845a4c3f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.test.ts @@ -7,10 +7,10 @@ */ import type { - IndexMapping, + IndexMappingMeta, SavedObjectsTypeMappingDefinitions, } from '@kbn/core-saved-objects-base-server-internal'; -import { buildActiveMappings, diffMappings, getUpdatedHashes } from './build_active_mappings'; +import { buildActiveMappings, getBaseMappings } from './build_active_mappings'; describe('buildActiveMappings', () => { test('creates a strict mapping', () => { @@ -58,215 +58,74 @@ describe('buildActiveMappings', () => { expect(buildActiveMappings(typeMappings)).toMatchSnapshot(); }); - test('generated hashes are stable', () => { + test(`includes the provided override properties, except for 'properties'`, () => { const properties = { aaa: { type: 'keyword', fields: { a: { type: 'keyword' }, b: { type: 'text' } } }, bbb: { fields: { b: { type: 'text' }, a: { type: 'keyword' } }, type: 'keyword' }, ccc: { fields: { b: { type: 'text' }, a: { type: 'text' } }, type: 'keyword' }, } as const; - const mappings = buildActiveMappings(properties); - const hashes = mappings._meta!.migrationMappingPropertyHashes!; - - expect(hashes.aaa).toBeDefined(); - expect(hashes.aaa).toEqual(hashes.bbb); - expect(hashes.aaa).not.toEqual(hashes.ccc); - }); -}); - -describe('diffMappings', () => { - test('is different if expected contains extra hashes', () => { - const actual: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar', baz: 'qux' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(diffMappings(actual, expected)!.changedProp).toEqual('properties.baz'); - }); - - test('does nothing if actual contains extra hashes', () => { - const actual: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar', baz: 'qux' }, - }, - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(diffMappings(actual, expected)).toBeUndefined(); - }); - - test('does nothing if actual hashes are identical to expected, but properties differ', () => { - const actual: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: { - foo: { type: 'keyword' }, - }, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: { - foo: { type: 'text' }, - }, - }; - - expect(diffMappings(actual, expected)).toBeUndefined(); - }); - - test('is different if meta hashes change', () => { - const actual: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'baz' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(diffMappings(actual, expected)!.changedProp).toEqual('properties.foo'); - }); - - test('is different if dynamic is different', () => { - const actual: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, + const ourExternallyBuiltMeta: IndexMappingMeta = { + mappingVersions: { + foo: '10.1.0', + bar: '10.2.0', + baz: '10.3.0', }, - // @ts-expect-error - dynamic: 'abcde', - properties: {}, }; - expect(diffMappings(actual, expected)!.changedProp).toEqual('dynamic'); - }); - - test('is different if migrationMappingPropertyHashes is missing from actual', () => { - const actual: IndexMapping = { - _meta: {}, - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(diffMappings(actual, expected)!.changedProp).toEqual('_meta'); - }); - - test('is different if _meta is missing from actual', () => { - const actual: IndexMapping = { - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(diffMappings(actual, expected)!.changedProp).toEqual('_meta'); + const mappings = buildActiveMappings(properties, ourExternallyBuiltMeta); + expect(mappings._meta).toEqual(ourExternallyBuiltMeta); + expect(mappings.properties.ddd).toBeUndefined(); }); }); -describe('getUpdatedHashes', () => { - test('gives all hashes if _meta is missing from actual', () => { - const actual: IndexMapping = { - dynamic: 'strict', - properties: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar', bar: 'baz' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(getUpdatedHashes({ actual, expected })).toEqual(['foo', 'bar']); - }); - - test('gives all hashes if migrationMappingPropertyHashes is missing from actual', () => { - const actual: IndexMapping = { - dynamic: 'strict', - properties: {}, - _meta: {}, - }; - const expected: IndexMapping = { - _meta: { - migrationMappingPropertyHashes: { foo: 'bar', bar: 'baz' }, - }, - dynamic: 'strict', - properties: {}, - }; - - expect(getUpdatedHashes({ actual, expected })).toEqual(['foo', 'bar']); - }); - - test('gives a list of the types with updated hashes', () => { - const actual: IndexMapping = { +describe('getBaseMappings', () => { + test('changes in core fields trigger a pickup of all documents, which can be really costly. Update only if you know what you are doing', () => { + expect(getBaseMappings()).toEqual({ dynamic: 'strict', - properties: {}, - _meta: { - migrationMappingPropertyHashes: { - type1: 'type1hash1', - type2: 'type2hash1', - type3: 'type3hash1', // will be removed + properties: { + type: { + type: 'keyword', }, - }, - }; - const expected: IndexMapping = { - dynamic: 'strict', - properties: {}, - _meta: { - migrationMappingPropertyHashes: { - type1: 'type1hash1', // remains the same - type2: 'type2hash2', // updated - type4: 'type4hash1', // new type + namespace: { + type: 'keyword', + }, + namespaces: { + type: 'keyword', + }, + originId: { + type: 'keyword', + }, + updated_at: { + type: 'date', + }, + created_at: { + type: 'date', + }, + references: { + type: 'nested', + properties: { + name: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + }, + }, + coreMigrationVersion: { + type: 'keyword', + }, + typeMigrationVersion: { + type: 'version', + }, + managed: { + type: 'boolean', }, }, - }; - - expect(getUpdatedHashes({ actual, expected })).toEqual(['type2', 'type4']); + }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts index feec8b5c2aa08..e3de61646d749 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_active_mappings.ts @@ -6,15 +6,11 @@ * Side Public License, v 1. */ -/* - * This file contains logic to build and diff the index mappings for a migration. - */ - -import crypto from 'crypto'; -import { cloneDeep, mapValues } from 'lodash'; +import { cloneDeep } from 'lodash'; import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; import type { IndexMapping, + IndexMappingMeta, SavedObjectsTypeMappingDefinitions, } from '@kbn/core-saved-objects-base-server-internal'; @@ -25,116 +21,21 @@ import type { * @param typeDefinitions - the type definitions to build mapping from. */ export function buildActiveMappings( - typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties + typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties, + _meta?: IndexMappingMeta ): IndexMapping { const mapping = getBaseMappings(); - const mergedProperties = validateAndMerge(mapping.properties, typeDefinitions); - return cloneDeep({ ...mapping, - properties: mergedProperties, - _meta: { - migrationMappingPropertyHashes: md5Values(mergedProperties), - }, + properties: validateAndMerge(mapping.properties, typeDefinitions), + ...(_meta && { _meta }), }); } /** - * Diffs the actual vs expected mappings. The properties are compared using md5 hashes stored in _meta, because - * actual and expected mappings *can* differ, but if the md5 hashes stored in actual._meta.migrationMappingPropertyHashes - * match our expectations, we don't require a migration. This allows ES to tack on additional mappings that Kibana - * doesn't know about or expect, without triggering continual migrations. - */ -export function diffMappings(actual: IndexMapping, expected: IndexMapping) { - if (actual.dynamic !== expected.dynamic) { - return { changedProp: 'dynamic' }; - } - - if (!actual._meta?.migrationMappingPropertyHashes) { - return { changedProp: '_meta' }; - } - - const changedProp = findChangedProp( - actual._meta.migrationMappingPropertyHashes, - expected._meta!.migrationMappingPropertyHashes - ); - - return changedProp ? { changedProp: `properties.${changedProp}` } : undefined; -} - -/** - * Compares the actual vs expected mappings' hashes. - * Returns a list with all the hashes that have been updated. - */ -export const getUpdatedHashes = ({ - actual, - expected, -}: { - actual: IndexMapping; - expected: IndexMapping; -}): string[] => { - if (!actual._meta?.migrationMappingPropertyHashes) { - return Object.keys(expected._meta!.migrationMappingPropertyHashes!); - } - - const updatedHashes = Object.keys(expected._meta!.migrationMappingPropertyHashes!).filter( - (key) => - actual._meta!.migrationMappingPropertyHashes![key] !== - expected._meta!.migrationMappingPropertyHashes![key] - ); - - return updatedHashes; -}; - -// Convert an object to an md5 hash string, using a stable serialization (canonicalStringify) -function md5Object(obj: any) { - return crypto.createHash('md5').update(canonicalStringify(obj)).digest('hex'); -} - -// JSON.stringify is non-canonical, meaning the same object may produce slightly -// different JSON, depending on compiler optimizations (e.g. object keys -// are not guaranteed to be sorted). This function consistently produces the same -// string, if passed an object of the same shape. If the outpuf of this function -// changes from one release to another, migrations will run, so it's important -// that this function remains stable across releases. -function canonicalStringify(obj: any): string { - if (Array.isArray(obj)) { - return `[${obj.map(canonicalStringify)}]`; - } - - if (!obj || typeof obj !== 'object') { - return JSON.stringify(obj); - } - - const keys = Object.keys(obj); - - // This is important for properly handling Date - if (!keys.length) { - return JSON.stringify(obj); - } - - const sortedObj = keys - .sort((a, b) => a.localeCompare(b)) - .map((k) => `${k}: ${canonicalStringify(obj[k])}`); - - return `{${sortedObj}}`; -} - -// Convert an object's values to md5 hash strings -function md5Values(obj: any) { - return mapValues(obj, md5Object); -} - -// If something exists in actual, but is missing in expected, we don't -// care, as it could be a disabled plugin, etc, and keeping stale stuff -// around is better than migrating unecessesarily. -function findChangedProp(actual: any, expected: any) { - return Object.keys(expected).find((k) => actual[k] !== expected[k]); -} - -/** - * These mappings are required for any saved object index. + * Defines the mappings for the root fields, common to all saved objects. + * These are present in all SO indices. * * @returns {IndexMapping} */ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.test.ts index 3895bd9314df3..bb17ab4154529 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.test.ts @@ -9,23 +9,13 @@ import { buildPickupMappingsQuery } from './build_pickup_mappings_query'; describe('buildPickupMappingsQuery', () => { - describe('when no root fields have been updated', () => { - it('builds a boolean query to select the updated types', () => { - const query = buildPickupMappingsQuery(['type1', 'type2']); + it('builds a boolean query to select the updated types', () => { + const query = buildPickupMappingsQuery(['type1', 'type2']); - expect(query).toEqual({ - bool: { - should: [{ term: { type: 'type1' } }, { term: { type: 'type2' } }], - }, - }); - }); - }); - - describe('when some root fields have been updated', () => { - it('returns undefined', () => { - const query = buildPickupMappingsQuery(['type1', 'type2', 'namespaces']); - - expect(query).toBeUndefined(); + expect(query).toEqual({ + bool: { + should: [{ term: { type: 'type1' } }, { term: { type: 'type2' } }], + }, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.ts index ce110f72f66c1..457cfb1ca7926 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/build_pickup_mappings_query.ts @@ -6,22 +6,9 @@ * Side Public License, v 1. */ -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; -import { getBaseMappings } from './build_active_mappings'; - -export const buildPickupMappingsQuery = ( - updatedFields: string[] -): QueryDslQueryContainer | undefined => { - const rootFields = Object.keys(getBaseMappings().properties); - - if (updatedFields.some((field) => rootFields.includes(field))) { - // we are updating some root fields, update ALL documents (no filter query) - return undefined; - } - - // at this point, all updated fields correspond to SO types - const updatedTypes = updatedFields; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +export const buildPickupMappingsQuery = (updatedTypes: string[]): QueryDslQueryContainer => { return { bool: { should: updatedTypes.map((type) => ({ term: { type } })), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts new file mode 100644 index 0000000000000..eec6e52090cae --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { getBaseMappings } from './build_active_mappings'; +import { getUpdatedTypes, getUpdatedRootFields } from './compare_mappings'; + +describe('getUpdatedTypes', () => { + test('returns all types if _meta is missing in indexMappings', () => { + const indexTypes = ['foo', 'bar']; + const latestMappingsVersions = {}; + + expect(getUpdatedTypes({ indexTypes, indexMeta: undefined, latestMappingsVersions })).toEqual([ + 'foo', + 'bar', + ]); + }); + + test('returns all types if migrationMappingPropertyHashes and mappingVersions are missing in indexMappings', () => { + const indexTypes = ['foo', 'bar']; + const indexMeta: IndexMappingMeta = {}; + const latestMappingsVersions = {}; + + expect(getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions })).toEqual([ + 'foo', + 'bar', + ]); + }); + + describe('when ONLY migrationMappingPropertyHashes exists in indexMappings', () => { + test('uses the provided hashToVersionMap to compare changes and return only the types that have changed', async () => { + const indexTypes = ['type1', 'type2', 'type4']; + const indexMeta: IndexMappingMeta = { + migrationMappingPropertyHashes: { + type1: 'someHash', + type2: 'anotherHash', + type3: 'aThirdHash', // will be removed + }, + }; + + const hashToVersionMap = { + 'type1|someHash': '10.1.0', + 'type2|anotherHash': '10.1.0', + 'type3|aThirdHash': '10.1.0', + }; + + const latestMappingsVersions = { + type1: '10.1.0', + type2: '10.2.0', + type4: '10.5.0', // new type, no need to pick it up + }; + + expect( + getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions, hashToVersionMap }) + ).toEqual(['type2']); + }); + }); + + describe('when mappingVersions exist in indexMappings', () => { + test('compares the modelVersions and returns only the types that have changed', async () => { + const indexTypes = ['type1', 'type2', 'type4']; + + const indexMeta: IndexMappingMeta = { + mappingVersions: { + type1: '10.1.0', + type2: '10.1.0', + type3: '10.1.0', // will be removed + }, + // ignored, cause mappingVersions is present + migrationMappingPropertyHashes: { + type1: 'someHash', + type2: 'anotherHash', + type3: 'aThirdHash', + }, + }; + + const latestMappingsVersions = { + type1: '10.1.0', + type2: '10.2.0', + type4: '10.5.0', // new type, no need to pick it up + }; + + const hashToVersionMap = { + // empty on purpose, not used as mappingVersions is present in indexMappings + }; + + expect( + getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions, hashToVersionMap }) + ).toEqual(['type2']); + }); + }); +}); + +describe('getUpdatedRootFields', () => { + it('deep compares provided indexMappings against the current baseMappings()', () => { + const updatedFields = getUpdatedRootFields({ + properties: { + ...getBaseMappings().properties, + namespace: { + type: 'text', + }, + references: { + type: 'nested', + properties: { + ...getBaseMappings().properties.references.properties, + description: { + type: 'text', + }, + }, + }, + }, + }); + + expect(updatedFields).toEqual(['namespace', 'references']); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts new file mode 100644 index 0000000000000..6e2ac42dee2a3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/compare_mappings.ts @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import equals from 'fast-deep-equal'; +import Semver from 'semver'; + +import type { + IndexMappingMeta, + VirtualVersionMap, + IndexMapping, +} from '@kbn/core-saved-objects-base-server-internal'; +import { getBaseMappings } from './build_active_mappings'; + +/** + * Compare the current mappings for root fields Vs those stored in the SO index. + * Relies on getBaseMappings to determine the current mappings. + * @param indexMappings The mappings stored in the SO index + * @returns A list of the root fields whose mappings have changed + */ +export const getUpdatedRootFields = (indexMappings: IndexMapping): string[] => { + const baseMappings = getBaseMappings(); + return Object.entries(baseMappings.properties) + .filter( + ([propertyName, propertyValue]) => + !equals(propertyValue, indexMappings.properties[propertyName]) + ) + .map(([propertyName]) => propertyName); +}; + +/** + * Compares the current vs stored mappings' hashes or modelVersions. + * Returns a list with all the types that have been updated. + * @param indexMeta The meta information stored in the SO index + * @param knownTypes The list of SO types that belong to the index and are enabled + * @param latestMappingsVersions A map holding [type => version] with the latest versions where mappings have changed for each type + * @param hashToVersionMap A map holding information about [md5 => modelVersion] equivalence + * @returns the list of types that have been updated (in terms of their mappings) + */ +export const getUpdatedTypes = ({ + indexMeta, + indexTypes, + latestMappingsVersions, + hashToVersionMap = {}, +}: { + indexMeta?: IndexMappingMeta; + indexTypes: string[]; + latestMappingsVersions: VirtualVersionMap; + hashToVersionMap?: Record; +}): string[] => { + if (!indexMeta || (!indexMeta.mappingVersions && !indexMeta.migrationMappingPropertyHashes)) { + // if we currently do NOT have meta information stored in the index + // we consider that all types have been updated + return indexTypes; + } + + // If something exists in stored, but is missing in current + // we don't care, as it could be a disabled plugin, etc + // and keeping stale stuff around is better than migrating unecessesarily. + return indexTypes.filter((type) => + isTypeUpdated({ + type, + mappingVersion: latestMappingsVersions[type], + indexMeta, + hashToVersionMap, + }) + ); +}; + +/** + * + * @param type The saved object type to check + * @param mappingVersion The most recent model version that includes mappings changes + * @param indexMeta The meta information stored in the SO index + * @param hashToVersionMap A map holding information about [md5 => modelVersion] equivalence + * @returns true if the mappings for the given type have changed since Kibana was last started + */ +function isTypeUpdated({ + type, + mappingVersion, + indexMeta, + hashToVersionMap, +}: { + type: string; + mappingVersion: string; + indexMeta: IndexMappingMeta; + hashToVersionMap: Record; +}): boolean { + const latestMappingsVersion = Semver.parse(mappingVersion); + if (!latestMappingsVersion) { + throw new Error( + `The '${type}' saved object type is not specifying a valid semver: ${mappingVersion}` + ); + } + + if (indexMeta.mappingVersions) { + // the SO index is already using mappingVersions (instead of md5 hashes) + const indexVersion = indexMeta.mappingVersions[type]; + if (!indexVersion) { + // either a new type, and thus there's not need to update + pickup any docs + // or an old re-enabled type, which will be updated on OUTDATED_DOCUMENTS_TRANSFORM + return false; + } + + // if the last version where mappings have changed is more recent than the one stored in the index + // it means that the type has been updated + return latestMappingsVersion.compare(indexVersion) === 1; + } else if (indexMeta.migrationMappingPropertyHashes) { + const latestHash = indexMeta.migrationMappingPropertyHashes?.[type]; + + if (!latestHash) { + // either a new type, and thus there's not need to update + pickup any docs + // or an old re-enabled type, which will be updated on OUTDATED_DOCUMENTS_TRANSFORM + return false; + } + + const indexEquivalentVersion = hashToVersionMap[`${type}|${latestHash}`]; + return !indexEquivalentVersion || latestMappingsVersion.compare(indexEquivalentVersion) === 1; + } + + // at this point, the mappings do not contain any meta informataion + // we consider the type has been updated, out of caution + return true; +} diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts new file mode 100644 index 0000000000000..d7036f1264a99 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.test.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; +import { getBaseMappings } from './build_active_mappings'; +import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings'; +import { diffMappings } from './diff_mappings'; + +jest.mock('./compare_mappings'); +const getUpdatedRootFieldsMock = getUpdatedRootFields as jest.MockedFn; +const getUpdatedTypesMock = getUpdatedTypes as jest.MockedFn; + +const dummyMappings: IndexMapping = { + _meta: { + mappingVersions: { foo: '10.1.0' }, + }, + dynamic: 'strict', + properties: {}, +}; + +const initialMappings: IndexMapping = { + _meta: { + mappingVersions: { foo: '10.1.0' }, + }, + dynamic: 'strict', + properties: { + ...getBaseMappings().properties, + foo: { + type: 'text', + }, + }, +}; +const updatedMappings: IndexMapping = { + _meta: { + mappingVersions: { foo: '10.2.0' }, + }, + dynamic: 'strict', + properties: { + ...getBaseMappings().properties, + foo: { + type: 'keyword', + }, + }, +}; + +const dummyHashToVersionMap = { + 'foo|someHash': '10.1.0', +}; + +describe('diffMappings', () => { + beforeEach(() => { + getUpdatedRootFieldsMock.mockReset(); + getUpdatedTypesMock.mockReset(); + }); + + test('is different if dynamic is different', () => { + const indexMappings = dummyMappings; + const appMappings: IndexMapping = { + ...dummyMappings, + dynamic: false, + }; + + expect( + diffMappings({ indexTypes: ['foo'], appMappings, indexMappings, latestMappingsVersions: {} })! + .changedProp + ).toEqual('dynamic'); + }); + + test('is different if _meta is missing in indexMappings', () => { + const indexMappings: IndexMapping = { + dynamic: 'strict', + properties: {}, + }; + const appMappings: IndexMapping = dummyMappings; + + expect( + diffMappings({ indexTypes: ['foo'], appMappings, indexMappings, latestMappingsVersions: {} })! + .changedProp + ).toEqual('_meta'); + }); + + test('is different if migrationMappingPropertyHashes and mappingVersions are missing in indexMappings', () => { + const indexMappings: IndexMapping = { + _meta: {}, + dynamic: 'strict', + properties: {}, + }; + const appMappings: IndexMapping = dummyMappings; + + expect( + diffMappings({ indexTypes: ['foo'], appMappings, indexMappings, latestMappingsVersions: {} })! + .changedProp + ).toEqual('_meta'); + }); + + describe('if a root field has changed', () => { + test('returns that root field', () => { + getUpdatedRootFieldsMock.mockReturnValueOnce(['references']); + + expect( + diffMappings({ + indexTypes: ['foo'], + appMappings: updatedMappings, + indexMappings: initialMappings, + latestMappingsVersions: {}, + }) + ).toEqual({ changedProp: 'properties.references' }); + + expect(getUpdatedRootFieldsMock).toHaveBeenCalledTimes(1); + expect(getUpdatedRootFieldsMock).toHaveBeenCalledWith(initialMappings); + expect(getUpdatedTypesMock).not.toHaveBeenCalled(); + }); + }); + + describe('if some types have changed', () => { + test('returns a changed type', () => { + getUpdatedRootFieldsMock.mockReturnValueOnce([]); + getUpdatedTypesMock.mockReturnValueOnce(['foo', 'bar']); + + expect( + diffMappings({ + indexTypes: ['foo', 'bar', 'baz'], + appMappings: updatedMappings, + indexMappings: initialMappings, + latestMappingsVersions: { + foo: '10.1.0', + }, + hashToVersionMap: dummyHashToVersionMap, + }) + ).toEqual({ changedProp: 'properties.foo' }); + + expect(getUpdatedRootFieldsMock).toHaveBeenCalledTimes(1); + expect(getUpdatedRootFieldsMock).toHaveBeenCalledWith(initialMappings); + expect(getUpdatedTypesMock).toHaveBeenCalledTimes(1); + expect(getUpdatedTypesMock).toHaveBeenCalledWith({ + indexTypes: ['foo', 'bar', 'baz'], + indexMeta: initialMappings._meta, + latestMappingsVersions: { + foo: '10.1.0', + }, + hashToVersionMap: dummyHashToVersionMap, + }); + }); + }); + + describe('if no root field or types have changed', () => { + test('returns undefined', () => { + getUpdatedRootFieldsMock.mockReturnValueOnce([]); + getUpdatedTypesMock.mockReturnValueOnce([]); + + expect( + diffMappings({ + indexTypes: ['foo', 'bar', 'baz'], + appMappings: updatedMappings, + indexMappings: initialMappings, + latestMappingsVersions: { + foo: '10.1.0', + }, + hashToVersionMap: dummyHashToVersionMap, + }) + ).toBeUndefined(); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts new file mode 100644 index 0000000000000..b7eeaaf40f079 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/diff_mappings.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings'; + +/** + * Diffs the actual vs expected mappings. The properties are compared using md5 hashes stored in _meta, because + * actual and expected mappings *can* differ, but if the md5 hashes stored in actual._meta.migrationMappingPropertyHashes + * match our expectations, we don't require a migration. This allows ES to tack on additional mappings that Kibana + * doesn't know about or expect, without triggering continual migrations. + */ +export function diffMappings({ + indexMappings, + appMappings, + indexTypes, + latestMappingsVersions, + hashToVersionMap = {}, +}: { + indexMappings: IndexMapping; + appMappings: IndexMapping; + indexTypes: string[]; + latestMappingsVersions: VirtualVersionMap; + hashToVersionMap?: Record; +}) { + if (indexMappings.dynamic !== appMappings.dynamic) { + return { changedProp: 'dynamic' }; + } else if ( + !indexMappings._meta?.migrationMappingPropertyHashes && + !indexMappings._meta?.mappingVersions + ) { + return { changedProp: '_meta' }; + } else { + const changedProp = findChangedProp({ + indexMappings, + indexTypes, + latestMappingsVersions, + hashToVersionMap, + }); + return changedProp ? { changedProp: `properties.${changedProp}` } : undefined; + } +} + +/** + * Finds a property that has changed its schema with respect to the mappings stored in the SO index + * It can either be a root field or a SO type + * @returns the name of the property (if any) + */ +function findChangedProp({ + indexMappings, + indexTypes, + hashToVersionMap, + latestMappingsVersions, +}: { + indexMappings: IndexMapping; + indexTypes: string[]; + hashToVersionMap: Record; + latestMappingsVersions: VirtualVersionMap; +}): string | undefined { + const updatedFields = getUpdatedRootFields(indexMappings); + if (updatedFields.length) { + return updatedFields[0]; + } + + const updatedTypes = getUpdatedTypes({ + indexMeta: indexMappings._meta, + indexTypes, + latestMappingsVersions, + hashToVersionMap, + }); + if (updatedTypes.length) { + return updatedTypes[0]; + } + + return undefined; +} diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/index.ts index 97e62e8657d5e..ea696268b92b9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/index.ts @@ -8,5 +8,5 @@ export { KibanaMigrator } from './kibana_migrator'; export type { KibanaMigratorOptions } from './kibana_migrator'; -export { buildActiveMappings, buildTypesMappings } from './core'; +export { buildActiveMappings, buildTypesMappings, getBaseMappings } from './core'; export { DocumentMigrator } from './document_migrator'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts index e173def8da914..aded2cb81f70f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts @@ -31,18 +31,29 @@ const migrationsConfig = { maxReadBatchSizeBytes: ByteSizeValue.parse('500mb'), } as unknown as SavedObjectsMigrationConfigType; +const indexTypesMap = { + '.kibana': ['typeA', 'typeB', 'typeC'], + '.kibana_task_manager': ['task'], + '.kibana_cases': ['typeD', 'typeE'], +}; + const createInitialStateCommonParams = { kibanaVersion: '8.1.0', waitForMigrationCompletion: false, mustRelocateDocuments: true, - indexTypesMap: { - '.kibana': ['typeA', 'typeB', 'typeC'], - '.kibana_task_manager': ['task'], - '.kibana_cases': ['typeD', 'typeE'], + indexTypes: ['typeA', 'typeB', 'typeC'], + indexTypesMap, + hashToVersionMap: { + 'typeA|someHash': '10.1.0', + 'typeB|someHash': '10.1.0', + 'typeC|someHash': '10.1.0', }, - targetMappings: { + targetIndexMappings: { dynamic: 'strict', properties: { my_type: { properties: { title: { type: 'text' } } } }, + _meta: { + indexTypesMap, + }, } as IndexMapping, coreMigrationVersionPerType: {}, migrationVersionPerType: {}, @@ -58,6 +69,37 @@ describe('createInitialState', () => { beforeEach(() => { typeRegistry = new SavedObjectTypeRegistry(); + typeRegistry.registerType({ + name: 'foo', + hidden: false, + mappings: { + properties: {}, + }, + namespaceType: 'single', + modelVersions: { + 1: { + changes: [], + }, + }, + switchToModelVersionAt: '8.10.0', + }); + typeRegistry.registerType({ + name: 'bar', + hidden: false, + mappings: { + properties: {}, + }, + namespaceType: 'single', + modelVersions: { + 1: { + changes: [], + }, + 2: { + changes: [{ type: 'mappings_addition', addedMappings: {} }], + }, + }, + switchToModelVersionAt: '8.10.0', + }); docLinks = docLinksServiceMock.createSetupContract(); logger = mockLogger.get(); createInitialStateParams = { @@ -204,7 +246,17 @@ describe('createInitialState', () => { ], }, }, + "hashToVersionMap": Object { + "typeA|someHash": "10.1.0", + "typeB|someHash": "10.1.0", + "typeC|someHash": "10.1.0", + }, "indexPrefix": ".kibana_task_manager", + "indexTypes": Array [ + "typeA", + "typeB", + "typeC", + ], "indexTypesMap": Object { ".kibana": Array [ "typeA", @@ -220,7 +272,14 @@ describe('createInitialState', () => { ], }, "kibanaVersion": "8.1.0", - "knownTypes": Array [], + "knownTypes": Array [ + "foo", + "bar", + ], + "latestMappingsVersions": Object { + "bar": "10.2.0", + "foo": "10.0.0", + }, "legacyIndex": ".kibana_task_manager", "logs": Array [], "maxBatchSize": 1000, @@ -299,19 +358,6 @@ describe('createInitialState', () => { }); it('returns state with the correct `knownTypes`', () => { - typeRegistry.registerType({ - name: 'foo', - namespaceType: 'single', - hidden: false, - mappings: { properties: {} }, - }); - typeRegistry.registerType({ - name: 'bar', - namespaceType: 'multiple', - hidden: true, - mappings: { properties: {} }, - }); - const initialState = createInitialState({ ...createInitialStateParams, typeRegistry, @@ -325,7 +371,7 @@ describe('createInitialState', () => { it('returns state with the correct `excludeFromUpgradeFilterHooks`', () => { const fooExcludeOnUpgradeHook = jest.fn(); typeRegistry.registerType({ - name: 'foo', + name: 'baz', namespaceType: 'single', hidden: false, mappings: { properties: {} }, @@ -333,7 +379,7 @@ describe('createInitialState', () => { }); const initialState = createInitialState(createInitialStateParams); - expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ foo: fooExcludeOnUpgradeHook }); + expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ baz: fooExcludeOnUpgradeHook }); }); it('returns state with a preMigration script', () => { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts index cf7494e655426..c86ee5d2a6ea0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts @@ -10,7 +10,8 @@ import * as Option from 'fp-ts/Option'; import type { DocLinksServiceStart } from '@kbn/core-doc-links-server'; import type { Logger } from '@kbn/logging'; import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server'; -import type { +import { + getLatestMappingsVirtualVersionMap, IndexMapping, IndexTypesMap, SavedObjectsMigrationConfigType, @@ -28,8 +29,10 @@ export interface CreateInitialStateParams extends OutdatedDocumentsQueryParams { kibanaVersion: string; waitForMigrationCompletion: boolean; mustRelocateDocuments: boolean; + indexTypes: string[]; indexTypesMap: IndexTypesMap; - targetMappings: IndexMapping; + hashToVersionMap: Record; + targetIndexMappings: IndexMapping; preMigrationScript?: string; indexPrefix: string; migrationsConfig: SavedObjectsMigrationConfigType; @@ -39,6 +42,16 @@ export interface CreateInitialStateParams extends OutdatedDocumentsQueryParams { esCapabilities: ElasticsearchCapabilities; } +const TEMP_INDEX_MAPPINGS: IndexMapping = { + dynamic: false, + properties: { + type: { type: 'keyword' }, + typeMigrationVersion: { + type: 'version', + }, + }, +}; + /** * Construct the initial state for the model */ @@ -46,8 +59,10 @@ export const createInitialState = ({ kibanaVersion, waitForMigrationCompletion, mustRelocateDocuments, + indexTypes, indexTypesMap, - targetMappings, + hashToVersionMap, + targetIndexMappings, preMigrationScript, coreMigrationVersionPerType, migrationVersionPerType, @@ -63,16 +78,6 @@ export const createInitialState = ({ migrationVersionPerType, }); - const reindexTargetMappings: IndexMapping = { - dynamic: false, - properties: { - type: { type: 'keyword' }, - typeMigrationVersion: { - type: 'version', - }, - }, - }; - const knownTypes = typeRegistry.getAllTypes().map((type) => type.name); const excludeFilterHooks = Object.fromEntries( typeRegistry @@ -101,19 +106,13 @@ export const createInitialState = ({ ); } - const targetIndexMappings: IndexMapping = { - ...targetMappings, - _meta: { - ...targetMappings._meta, - indexTypesMap, - }, - }; - return { controlState: 'INIT', waitForMigrationCompletion, mustRelocateDocuments, + indexTypes, indexTypesMap, + hashToVersionMap, indexPrefix, legacyIndex: indexPrefix, currentAlias: indexPrefix, @@ -124,7 +123,7 @@ export const createInitialState = ({ kibanaVersion, preMigrationScript: Option.fromNullable(preMigrationScript), targetIndexMappings, - tempIndexMappings: reindexTargetMappings, + tempIndexMappings: TEMP_INDEX_MAPPINGS, outdatedDocumentsQuery, retryCount: 0, retryDelay: 0, @@ -138,6 +137,7 @@ export const createInitialState = ({ logs: [], excludeOnUpgradeQuery: excludeUnusedTypesQuery, knownTypes, + latestMappingsVersions: getLatestMappingsVirtualVersionMap(typeRegistry.getAllTypes()), excludeFromUpgradeFilterHooks: excludeFilterHooks, migrationDocLinks, esCapabilities, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index bb60fd1c60c76..46a7ddf882b8d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -15,7 +15,7 @@ import { type MigrationResult, SavedObjectTypeRegistry, } from '@kbn/core-saved-objects-base-server-internal'; -import { KibanaMigrator } from './kibana_migrator'; +import { KibanaMigrator, type KibanaMigratorOptions } from './kibana_migrator'; import { DocumentMigrator } from './document_migrator'; import { ByteSizeValue } from '@kbn/config-schema'; import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; @@ -235,7 +235,7 @@ describe('KibanaMigrator', () => { }); }); -const mockOptions = (algorithm: 'v2' | 'zdt' = 'v2') => { +const mockOptions = (algorithm: 'v2' | 'zdt' = 'v2'): KibanaMigratorOptions => { const mockedClient = elasticsearchClientMock.createElasticsearchClient(); (mockedClient as any).child = jest.fn().mockImplementation(() => mockedClient); @@ -251,6 +251,7 @@ const mockOptions = (algorithm: 'v2' | 'zdt' = 'v2') => { // are moved over to their new index (.my_index) '.my_complementary_index': ['testtype3'], }, + hashToVersionMap: {}, typeRegistry: createRegistry([ // typeRegistry depicts an updated index map: // .my_index: ['testtype', 'testtype3'], diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index a07d810ee75ab..81c6930952f42 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -19,9 +19,9 @@ import type { ElasticsearchClient, ElasticsearchCapabilities, } from '@kbn/core-elasticsearch-server'; -import { - type SavedObjectUnsanitizedDoc, - type ISavedObjectTypeRegistry, +import type { + SavedObjectUnsanitizedDoc, + ISavedObjectTypeRegistry, } from '@kbn/core-saved-objects-server'; import { SavedObjectsSerializer, @@ -44,6 +44,7 @@ export interface KibanaMigratorOptions { client: ElasticsearchClient; typeRegistry: ISavedObjectTypeRegistry; defaultIndexTypesMap: IndexTypesMap; + hashToVersionMap: Record; soMigrationsConfig: SavedObjectsMigrationConfigType; kibanaIndex: string; kibanaVersion: string; @@ -65,6 +66,7 @@ export class KibanaMigrator implements IKibanaMigrator { private readonly mappingProperties: SavedObjectsTypeMappingDefinitions; private readonly typeRegistry: ISavedObjectTypeRegistry; private readonly defaultIndexTypesMap: IndexTypesMap; + private readonly hashToVersionMap: Record; private readonly serializer: SavedObjectsSerializer; private migrationResult?: Promise; private readonly status$ = new BehaviorSubject({ @@ -87,6 +89,7 @@ export class KibanaMigrator implements IKibanaMigrator { typeRegistry, kibanaIndex, defaultIndexTypesMap, + hashToVersionMap, soMigrationsConfig, kibanaVersion, logger, @@ -100,7 +103,9 @@ export class KibanaMigrator implements IKibanaMigrator { this.soMigrationsConfig = soMigrationsConfig; this.typeRegistry = typeRegistry; this.defaultIndexTypesMap = defaultIndexTypesMap; + this.hashToVersionMap = hashToVersionMap; this.serializer = new SavedObjectsSerializer(this.typeRegistry); + // build mappings.properties for all types, all indices this.mappingProperties = buildTypesMappings(this.typeRegistry.getAllTypes()); this.log = logger; this.kibanaVersion = kibanaVersion; @@ -112,8 +117,8 @@ export class KibanaMigrator implements IKibanaMigrator { }); this.waitForMigrationCompletion = waitForMigrationCompletion; this.nodeRoles = nodeRoles; - // Building the active mappings (and associated md5sums) is an expensive - // operation so we cache the result + // we are no longer adding _meta information to the mappings at this level + // consumers of the exposed mappings are only accessing the 'properties' field this.activeMappings = buildActiveMappings(this.mappingProperties); this.docLinks = docLinks; this.esCapabilities = esCapabilities; @@ -172,6 +177,7 @@ export class KibanaMigrator implements IKibanaMigrator { kibanaIndexPrefix: this.kibanaIndex, typeRegistry: this.typeRegistry, defaultIndexTypesMap: this.defaultIndexTypesMap, + hashToVersionMap: this.hashToVersionMap, logger: this.log, documentMigrator: this.documentMigrator, migrationConfig: this.soMigrationsConfig, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts index c8b630598f2e5..06c4ff9f0c715 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts @@ -39,12 +39,21 @@ describe('migrationsStateActionMachine', () => { kibanaVersion: '7.11.0', waitForMigrationCompletion: false, mustRelocateDocuments: true, + indexTypes: ['typeA', 'typeB', 'typeC'], indexTypesMap: { '.kibana': ['typeA', 'typeB', 'typeC'], '.kibana_task_manager': ['task'], '.kibana_cases': ['typeD', 'typeE'], }, - targetMappings: { properties: {} }, + hashToVersionMap: { + 'typeA|someHash': '10.1.0', + 'typeB|someHash': '10.1.0', + 'typeC|someHash': '10.1.0', + 'task|someHash': '10.1.0', + 'typeD|someHash': '10.1.0', + 'typeE|someHash': '10.1.0', + }, + targetIndexMappings: { properties: {} }, coreMigrationVersionPerType: {}, migrationVersionPerType: {}, indexPrefix: '.my-so-index', diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts index 4588c6639d883..d22b06152b961 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts @@ -44,36 +44,35 @@ export function throwBadResponse(state: { controlState: string }, res: unknown): } /** - * Merge the _meta.migrationMappingPropertyHashes mappings of an index with - * the given target mappings. + * Merge the mappings._meta information of an index with the given target mappings. * * @remarks When another instance already completed a migration, the existing * target index might contain documents and mappings created by a plugin that * is disabled in the current Kibana instance performing this migration. * Mapping updates are commutative (deeply merged) by Elasticsearch, except - * for the `_meta` key. By merging the `_meta.migrationMappingPropertyHashes` - * mappings from the existing target index index into the targetMappings we - * ensure that any `migrationPropertyHashes` for disabled plugins aren't lost. - * - * Right now we don't use these `migrationPropertyHashes` but it could be used - * in the future to detect if mappings were changed. If mappings weren't - * changed we don't need to reindex but can clone the index to save disk space. + * for the `_meta` key. By merging the `_meta` from the existing target index + * into the targetMappings we ensure that any versions for disabled plugins aren't lost. * * @param targetMappings * @param indexMappings */ -export function mergeMigrationMappingPropertyHashes( - targetMappings: IndexMapping, - indexMappings: IndexMapping -) { +export function mergeMappingMeta(targetMappings: IndexMapping, indexMappings: IndexMapping) { + const mappingVersions = { + ...indexMappings._meta?.mappingVersions, + ...targetMappings._meta?.mappingVersions, + }; + + const migrationMappingPropertyHashes = { + ...indexMappings._meta?.migrationMappingPropertyHashes, + ...targetMappings._meta?.migrationMappingPropertyHashes, + }; + return { ...targetMappings, _meta: { ...targetMappings._meta, - migrationMappingPropertyHashes: { - ...indexMappings._meta?.migrationMappingPropertyHashes, - ...targetMappings._meta?.migrationMappingPropertyHashes, - }, + ...(Object.keys(mappingVersions).length && { mappingVersions }), + ...(Object.keys(migrationMappingPropertyHashes).length && { migrationMappingPropertyHashes }), }, }; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts index 29f7f95ba65db..5dba20c7826a8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts @@ -115,7 +115,16 @@ describe('migrations v2 model', () => { ], }, }, + indexTypes: ['config'], knownTypes: ['dashboard', 'config'], + latestMappingsVersions: { + config: '10.3.0', + dashboard: '10.3.0', + }, + hashToVersionMap: { + 'config|someHash': '10.1.0', + 'dashboard|anotherHash': '10.2.0', + }, excludeFromUpgradeFilterHooks: {}, migrationDocLinks: { resolveMigrationFailures: 'https://someurl.co/', @@ -2602,7 +2611,7 @@ describe('migrations v2 model', () => { describe('reindex migration', () => { it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES if origin mappings did not exist', () => { const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({ - type: 'actual_mappings_incomplete' as const, + type: 'index_mappings_incomplete' as const, }); const newState = model( checkTargetMappingsState, @@ -2616,8 +2625,8 @@ describe('migrations v2 model', () => { describe('compatible migration', () => { it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES if core fields have been updated', () => { const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({ - type: 'compared_mappings_changed' as const, - updatedHashes: ['dashboard', 'lens', 'namespaces'], + type: 'root_fields_changed' as const, + updatedFields: ['references'], }); const newState = model( checkTargetMappingsState, @@ -2631,8 +2640,8 @@ describe('migrations v2 model', () => { it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES if only SO types have changed', () => { const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({ - type: 'compared_mappings_changed' as const, - updatedHashes: ['dashboard', 'lens'], + type: 'types_changed' as const, + updatedTypes: ['dashboard', 'lens'], }); const newState = model( checkTargetMappingsState, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 563b138fdb45f..b232a7d750925 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -37,7 +37,7 @@ import { getMigrationType, indexBelongsToLaterVersion, indexVersion, - mergeMigrationMappingPropertyHashes, + mergeMappingMeta, throwBadControlState, throwBadResponse, versionMigrationCompleted, @@ -54,7 +54,6 @@ import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON, FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, } from '../common/constants'; -import { getBaseMappings } from '../core'; import { buildPickupMappingsQuery } from '../core/build_pickup_mappings_query'; export const model = (currentState: State, resW: ResponseType): State => { @@ -518,7 +517,7 @@ export const model = (currentState: State, resW: ResponseType): // in this scenario, a .kibana_X.Y.Z_001 index exists that matches the current kibana version // aka we are NOT upgrading to a newer version // we inject the source index's current mappings in the state, to check them later - targetIndexMappings: mergeMigrationMappingPropertyHashes( + targetIndexMappings: mergeMappingMeta( stateP.targetIndexMappings, stateP.sourceIndexMappings.value ), @@ -574,7 +573,7 @@ export const model = (currentState: State, resW: ResponseType): controlState: 'PREPARE_COMPATIBLE_MIGRATION', mustRefresh: stateP.mustRefresh || typeof res.right.deleted === 'undefined' || res.right.deleted > 0, - targetIndexMappings: mergeMigrationMappingPropertyHashes( + targetIndexMappings: mergeMappingMeta( stateP.targetIndexMappings, stateP.sourceIndexMappings.value ), @@ -1430,14 +1429,14 @@ export const model = (currentState: State, resW: ResponseType): } else if (stateP.controlState === 'CHECK_TARGET_MAPPINGS') { const res = resW as ResponseType; if (Either.isRight(res)) { - // The md5 of ALL mappings match, so there's no need to update target mappings + // The mappings have NOT changed, no need to pick up changes in any documents return { ...stateP, controlState: 'CHECK_VERSION_INDEX_READY_ACTIONS', }; } else { const left = res.left; - if (isTypeof(left, 'actual_mappings_incomplete')) { + if (isTypeof(left, 'index_mappings_incomplete')) { // reindex migration // some top-level properties have changed, e.g. 'dynamic' or '_meta' (see checkTargetMappings()) // we must "pick-up" all documents on the index (by not providing a query) @@ -1446,42 +1445,38 @@ export const model = (currentState: State, resW: ResponseType): controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES', updatedTypesQuery: Option.none, }; - } else if (isTypeof(left, 'compared_mappings_changed')) { - const rootFields = Object.keys(getBaseMappings().properties); - const updatedRootFields = left.updatedHashes.filter((field) => rootFields.includes(field)); - const updatedTypesQuery = Option.fromNullable(buildPickupMappingsQuery(left.updatedHashes)); + } else if (isTypeof(left, 'root_fields_changed')) { + // compatible migration: some core fields have been updated + return { + ...stateP, + controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES', + // we must "pick-up" all documents on the index (by not providing a query) + updatedTypesQuery: Option.none, + logs: [ + ...stateP.logs, + { + level: 'info', + message: `Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: ${left.updatedFields}.`, + }, + ], + }; + } else if (isTypeof(left, 'types_changed')) { + // compatible migration: some fields have been updated, and they all correspond to SO types + const updatedTypesQuery = Option.fromNullable(buildPickupMappingsQuery(left.updatedTypes)); - if (updatedRootFields.length) { - // compatible migration: some core fields have been updated - return { - ...stateP, - controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES', - // we must "pick-up" all documents on the index (by not providing a query) - updatedTypesQuery, - logs: [ - ...stateP.logs, - { - level: 'info', - message: `Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: ${updatedRootFields}.`, - }, - ], - }; - } else { - // compatible migration: some fields have been updated, and they all correspond to SO types - return { - ...stateP, - controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES', - // we can "pick-up" only the SO types that have changed - updatedTypesQuery, - logs: [ - ...stateP.logs, - { - level: 'info', - message: `Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings: ${left.updatedHashes}.`, - }, - ], - }; - } + return { + ...stateP, + controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES', + // we can "pick-up" only the SO types that have changed + updatedTypesQuery, + logs: [ + ...stateP.logs, + { + level: 'info', + message: `Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: ${left.updatedTypes}.`, + }, + ], + }; } else { throwBadResponse(stateP, res as never); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts index 0eedfc620a96e..4df6e59cae22f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts @@ -85,16 +85,15 @@ export const nextActionMap = ( Actions.fetchIndices({ client, indices: [state.currentAlias, state.versionAlias] }), WAIT_FOR_YELLOW_SOURCE: (state: WaitForYellowSourceState) => Actions.waitForIndexStatus({ client, index: state.sourceIndex.value, status: 'yellow' }), - UPDATE_SOURCE_MAPPINGS_PROPERTIES: ({ - sourceIndex, - sourceIndexMappings, - targetIndexMappings, - }: UpdateSourceMappingsPropertiesState) => + UPDATE_SOURCE_MAPPINGS_PROPERTIES: (state: UpdateSourceMappingsPropertiesState) => Actions.updateSourceMappingsProperties({ client, - sourceIndex: sourceIndex.value, - sourceMappings: sourceIndexMappings.value, - targetMappings: targetIndexMappings, + indexTypes: state.indexTypes, + sourceIndex: state.sourceIndex.value, + indexMappings: state.sourceIndexMappings.value, + appMappings: state.targetIndexMappings, + latestMappingsVersions: state.latestMappingsVersions, + hashToVersionMap: state.hashToVersionMap, }), CLEANUP_UNKNOWN_AND_EXCLUDED: (state: CleanupUnknownAndExcluded) => Actions.cleanupUnknownAndExcluded({ @@ -206,8 +205,11 @@ export const nextActionMap = ( Actions.refreshIndex({ client, index: state.targetIndex }), CHECK_TARGET_MAPPINGS: (state: CheckTargetMappingsState) => Actions.checkTargetMappings({ - actualMappings: Option.toUndefined(state.sourceIndexMappings), - expectedMappings: state.targetIndexMappings, + indexTypes: state.indexTypes, + indexMappings: Option.toUndefined(state.sourceIndexMappings), + appMappings: state.targetIndexMappings, + latestMappingsVersions: state.latestMappingsVersions, + hashToVersionMap: state.hashToVersionMap, }), UPDATE_TARGET_MAPPINGS_PROPERTIES: (state: UpdateTargetMappingsPropertiesState) => Actions.updateAndPickupMappings({ diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.fixtures.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.fixtures.ts index c7a70296f35ba..3090991ea2f17 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.fixtures.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.fixtures.ts @@ -34,6 +34,13 @@ export const indexTypesMapMock = { '.complementary_index': ['testtype3'], }; +export const hashToVersionMapMock = { + 'testtype|someHash': '10.1.0', + 'testtype2|anotherHash': '10.2.0', + 'testtasktype|hashesAreCool': '10.1.0', + 'testtype3|yetAnotherHash': '10.1.0', +}; + export const savedObjectTypeRegistryMock = createRegistry([ // typeRegistry depicts an updated index map: // .my_index: ['testtype', 'testtype3'], diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.test.ts index b6111b1f5f6ed..06693177481c0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.test.ts @@ -18,7 +18,11 @@ import { waitGroup } from './kibana_migrator_utils'; import { migrationStateActionMachine } from './migrations_state_action_machine'; import { next } from './next'; import { runResilientMigrator, type RunResilientMigratorParams } from './run_resilient_migrator'; -import { indexTypesMapMock, savedObjectTypeRegistryMock } from './run_resilient_migrator.fixtures'; +import { + hashToVersionMapMock, + indexTypesMapMock, + savedObjectTypeRegistryMock, +} from './run_resilient_migrator.fixtures'; import type { InitState, State } from './state'; import type { Next } from './state_action_machine'; @@ -70,8 +74,10 @@ describe('runResilientMigrator', () => { kibanaVersion: options.kibanaVersion, waitForMigrationCompletion: options.waitForMigrationCompletion, mustRelocateDocuments: options.mustRelocateDocuments, + indexTypes: options.indexTypes, indexTypesMap: options.indexTypesMap, - targetMappings: options.targetMappings, + hashToVersionMap: options.hashToVersionMap, + targetIndexMappings: options.targetIndexMappings, preMigrationScript: options.preMigrationScript, migrationVersionPerType: options.migrationVersionPerType, coreMigrationVersionPerType: options.coreMigrationVersionPerType, @@ -117,8 +123,10 @@ const mockOptions = (): RunResilientMigratorParams => { kibanaVersion: '8.8.0', waitForMigrationCompletion: false, mustRelocateDocuments: true, + indexTypes: ['a', 'c'], indexTypesMap: indexTypesMapMock, - targetMappings: { + hashToVersionMap: hashToVersionMapMock, + targetIndexMappings: { properties: { a: { type: 'keyword' }, c: { type: 'long' }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts index b799b3e179a7f..5973728e2a99a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts @@ -49,8 +49,10 @@ export interface RunResilientMigratorParams { kibanaVersion: string; waitForMigrationCompletion: boolean; mustRelocateDocuments: boolean; + indexTypes: string[]; indexTypesMap: IndexTypesMap; - targetMappings: IndexMapping; + targetIndexMappings: IndexMapping; + hashToVersionMap: Record; preMigrationScript?: string; readyToReindex: WaitGroup; doneReindexing: WaitGroup; @@ -76,8 +78,10 @@ export async function runResilientMigrator({ kibanaVersion, waitForMigrationCompletion, mustRelocateDocuments, + indexTypes, indexTypesMap, - targetMappings, + targetIndexMappings, + hashToVersionMap, logger, preMigrationScript, readyToReindex, @@ -96,8 +100,10 @@ export async function runResilientMigrator({ kibanaVersion, waitForMigrationCompletion, mustRelocateDocuments, + indexTypes, indexTypesMap, - targetMappings, + hashToVersionMap, + targetIndexMappings, preMigrationScript, coreMigrationVersionPerType, migrationVersionPerType, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.test.ts index 3369071879e85..fc4d37cd78596 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.test.ts @@ -27,7 +27,11 @@ import { waitGroup, } from './kibana_migrator_utils'; import { runResilientMigrator } from './run_resilient_migrator'; -import { indexTypesMapMock, savedObjectTypeRegistryMock } from './run_resilient_migrator.fixtures'; +import { + hashToVersionMapMock, + indexTypesMapMock, + savedObjectTypeRegistryMock, +} from './run_resilient_migrator.fixtures'; jest.mock('./core', () => { const actual = jest.requireActual('./core'); @@ -248,6 +252,7 @@ const mockOptions = (kibanaVersion = '8.2.3'): RunV2MigrationOpts => { typeRegistry, kibanaIndexPrefix: '.my_index', defaultIndexTypesMap: indexTypesMapMock, + hashToVersionMap: hashToVersionMapMock, migrationConfig: { algorithm: 'v2' as const, batchSize: 20, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.ts index ff79ebb2ec7fb..de66aa147af48 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_v2_migration.ts @@ -17,13 +17,16 @@ import type { ISavedObjectsSerializer, SavedObjectsRawDoc, } from '@kbn/core-saved-objects-server'; -import type { - IndexTypesMap, - MigrationResult, - SavedObjectsMigrationConfigType, - SavedObjectsTypeMappingDefinitions, +import { + getVirtualVersionMap, + type IndexMappingMeta, + type IndexTypesMap, + type MigrationResult, + type SavedObjectsMigrationConfigType, + type SavedObjectsTypeMappingDefinitions, } from '@kbn/core-saved-objects-base-server-internal'; import Semver from 'semver'; +import { pick } from 'lodash'; import type { DocumentMigrator } from './document_migrator'; import { buildActiveMappings, createIndexMap } from './core'; import { @@ -43,6 +46,8 @@ export interface RunV2MigrationOpts { typeRegistry: ISavedObjectTypeRegistry; /** The map of indices => types to use as a default / baseline state */ defaultIndexTypesMap: IndexTypesMap; + /** A map that holds [last md5 used => modelVersion] for each of the SO types */ + hashToVersionMap: Record; /** Logger to use for migration output */ logger: Logger; /** The document migrator to use to convert the document */ @@ -113,6 +118,9 @@ export const runV2Migration = async (options: RunV2MigrationOpts): Promise migratorIndices.add(index)); + // we will store model versions instead of hashes (to be FIPS compliant) + const appVersions = getVirtualVersionMap(options.typeRegistry.getAllTypes()); + const migrators = Array.from(migratorIndices).map((indexName, i) => { return { migrate: (): Promise => { @@ -122,14 +130,27 @@ export const runV2Migration = async (options: RunV2MigrationOpts): Promise modelVersion] equivalence + */ + readonly hashToVersionMap: Record; /** * All exclude filter hooks registered for types on this index. Keyed by type name. */ @@ -170,14 +186,12 @@ export interface BaseState extends ControlState { */ readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects']; readonly waitForMigrationCompletion: boolean; - /** * This flag tells the migrator that SO documents must be redistributed, * i.e. stored in different system indices, compared to where they are currently stored. * This requires reindexing documents. */ readonly mustRelocateDocuments: boolean; - /** * This object holds a relation of all the types that are stored in each index, e.g.: * { @@ -187,7 +201,6 @@ export interface BaseState extends ControlState { * } */ readonly indexTypesMap: IndexTypesMap; - /** Capabilities of the ES cluster we're using */ readonly esCapabilities: ElasticsearchCapabilities; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index 7aaf44a1329a5..ceb97cb045987 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -116,7 +116,7 @@ export const init: ModelStage< }); // cloning as we may be mutating it in later stages. let currentIndexMeta = cloneDeep(currentMappings._meta!); - if (currentAlgo === 'v2-compatible') { + if (currentAlgo === 'v2-compatible' || currentAlgo === 'v2-partially-migrated') { currentIndexMeta = removePropertiesFromV2(currentIndexMeta); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts index 6c2f90155ac76..91a37ee7e9919 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts @@ -150,64 +150,33 @@ describe('actions', () => { }); describe('UPDATE_INDEX_MAPPINGS', () => { - describe('when only SO types have been updated', () => { - it('calls updateAndPickupMappings with the correct parameters', () => { - const state: UpdateIndexMappingsState = { - ...createPostDocInitState(), - controlState: 'UPDATE_INDEX_MAPPINGS', - additiveMappingChanges: { - someToken: {}, - }, - }; - const action = actionMap.UPDATE_INDEX_MAPPINGS; - - action(state); - - expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledTimes(1); - expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledWith({ - client: context.elasticsearchClient, - index: state.currentIndex, - mappings: { - properties: { - someToken: {}, - }, - }, - batchSize: context.batchSize, - query: { - bool: { - should: [{ term: { type: 'someToken' } }], - }, - }, - }); - }); - }); + it('calls updateAndPickupMappings with the correct parameters', () => { + const state: UpdateIndexMappingsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_INDEX_MAPPINGS', + additiveMappingChanges: { + someToken: {}, + }, + }; + const action = actionMap.UPDATE_INDEX_MAPPINGS; + + action(state); - describe('when core properties have been updated', () => { - it('calls updateAndPickupMappings with the correct parameters', () => { - const state: UpdateIndexMappingsState = { - ...createPostDocInitState(), - controlState: 'UPDATE_INDEX_MAPPINGS', - additiveMappingChanges: { - managed: {}, // this is a root field + expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + mappings: { + properties: { someToken: {}, }, - }; - const action = actionMap.UPDATE_INDEX_MAPPINGS; - - action(state); - - expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledTimes(1); - expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledWith({ - client: context.elasticsearchClient, - index: state.currentIndex, - mappings: { - properties: { - managed: {}, - someToken: {}, - }, + }, + batchSize: context.batchSize, + query: { + bool: { + should: [{ term: { type: 'someToken' } }], }, - batchSize: context.batchSize, - }); + }, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.test.ts index ba776110ca2d7..c81d2f3843a10 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.test.ts @@ -28,12 +28,12 @@ describe('checkIndexCurrentAlgorithm', () => { expect(checkIndexCurrentAlgorithm(mapping)).toEqual('unknown'); }); - it('returns `unknown` if both v2 and zdt metas are present', () => { + it('returns `zdt` if all zdt metas are present', () => { const mapping: IndexMapping = { properties: {}, _meta: { - migrationMappingPropertyHashes: { - foo: 'someHash', + docVersions: { + foo: '8.8.0', }, mappingVersions: { foo: '8.8.0', @@ -41,29 +41,29 @@ describe('checkIndexCurrentAlgorithm', () => { }, }; - expect(checkIndexCurrentAlgorithm(mapping)).toEqual('unknown'); + expect(checkIndexCurrentAlgorithm(mapping)).toEqual('zdt'); }); - it('returns `zdt` if all zdt metas are present', () => { + it('returns `v2-partially-migrated` if only mappingVersions is present', () => { const mapping: IndexMapping = { properties: {}, _meta: { - docVersions: { - foo: '8.8.0', - }, mappingVersions: { foo: '8.8.0', }, }, }; - expect(checkIndexCurrentAlgorithm(mapping)).toEqual('zdt'); + expect(checkIndexCurrentAlgorithm(mapping)).toEqual('v2-partially-migrated'); }); - it('returns `v2-partially-migrated` if only mappingVersions is present', () => { + it('returns `unknown` if if mappingVersions and v2 hashes are present', () => { const mapping: IndexMapping = { properties: {}, _meta: { + migrationMappingPropertyHashes: { + foo: 'someHash', + }, mappingVersions: { foo: '8.8.0', }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.ts index 74e269c828291..68551bf820112 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_index_algorithm.ts @@ -36,19 +36,17 @@ export const checkIndexCurrentAlgorithm = ( return 'unknown'; } - const hasV2Meta = !!meta.migrationMappingPropertyHashes; const hasZDTMeta = !!meta.mappingVersions; + const hasV2Meta = !!meta.migrationMappingPropertyHashes; - if (hasV2Meta && hasZDTMeta) { - return 'unknown'; + if (hasZDTMeta) { + const isFullZdt = !!meta.docVersions; + return isFullZdt ? 'zdt' : 'v2-partially-migrated'; } if (hasV2Meta) { const isCompatible = !!meta.indexTypesMap; return isCompatible ? 'v2-compatible' : 'v2-incompatible'; } - if (hasZDTMeta) { - const isFullZdt = !!meta.docVersions; - return isFullZdt ? 'zdt' : 'v2-partially-migrated'; - } + return 'unknown'; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json index 7bd1917e526b0..c1d993b0f4761 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json @@ -9,6 +9,7 @@ }, "include": [ "**/*.ts", + "src/hash_to_version_map.json" ], "kbn_references": [ "@kbn/logging", diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts b/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts index f334603007b61..8f50176e89eb6 100644 --- a/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts +++ b/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts @@ -43,6 +43,7 @@ import { type SavedObjectsMigrationConfigType, type IKibanaMigrator, DEFAULT_INDEX_TYPES_MAP, + HASH_TO_VERSION_MAP, } from '@kbn/core-saved-objects-base-server-internal'; import { SavedObjectsClient, @@ -389,6 +390,7 @@ export class SavedObjectsService soMigrationsConfig, kibanaIndex: MAIN_SAVED_OBJECT_INDEX, defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + hashToVersionMap: HASH_TO_VERSION_MAP, client, docLinks, waitForMigrationCompletion, diff --git a/packages/core/test-helpers/core-test-helpers-model-versions/src/test_bed/test_kit.ts b/packages/core/test-helpers/core-test-helpers-model-versions/src/test_bed/test_kit.ts index f986bb185cc4f..321cc13f9f6f3 100644 --- a/packages/core/test-helpers/core-test-helpers-model-versions/src/test_bed/test_kit.ts +++ b/packages/core/test-helpers/core-test-helpers-model-versions/src/test_bed/test_kit.ts @@ -82,6 +82,7 @@ export const prepareModelVersionTestKit = async ({ loggerFactory, kibanaIndex, defaultIndexTypesMap: {}, + hashToVersionMap: {}, kibanaVersion, kibanaBranch, nodeRoles: defaultNodeRoles, @@ -213,6 +214,7 @@ const getMigrator = async ({ kibanaIndex, typeRegistry, defaultIndexTypesMap, + hashToVersionMap, loggerFactory, kibanaVersion, kibanaBranch, @@ -224,6 +226,7 @@ const getMigrator = async ({ kibanaIndex: string; typeRegistry: ISavedObjectTypeRegistry; defaultIndexTypesMap: IndexTypesMap; + hashToVersionMap: Record; loggerFactory: LoggerFactory; kibanaVersion: string; kibanaBranch: string; @@ -250,6 +253,7 @@ const getMigrator = async ({ kibanaIndex, typeRegistry, defaultIndexTypesMap, + hashToVersionMap, soMigrationsConfig: soConfig.migration, kibanaVersion, logger: loggerFactory.get('savedobjects-service'), diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/check_hash_to_version_map.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group4/check_hash_to_version_map.test.ts new file mode 100644 index 0000000000000..278d108b830f9 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/check_hash_to_version_map.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * This file contains logic to build and diff the index mappings for a migration. + */ + +import crypto from 'crypto'; +import { mapValues } from 'lodash'; +import { + getLatestMappingsVirtualVersionMap, + HASH_TO_VERSION_MAP, +} from '@kbn/core-saved-objects-base-server-internal'; +import { buildTypesMappings } from '@kbn/core-saved-objects-migration-server-internal'; +import { getCurrentVersionTypeRegistry } from '../kibana_migrator_test_kit'; + +describe('transition from md5 hashes to model versions', () => { + // this short-lived test is here to ensure no changes are introduced after the creation of the HASH_TO_VERSION_MAP + it('ensures the hashToVersionMap does not miss any mappings changes', async () => { + const typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); + const mappingProperties = buildTypesMappings(typeRegistry.getAllTypes()); + const hashes = md5Values(mappingProperties); + const versions = getLatestMappingsVirtualVersionMap(typeRegistry.getAllTypes()); + + const currentHashToVersionMap = Object.entries(hashes).reduce>( + (acc, [type, hash]) => { + acc[`${type}|${hash}`] = versions[type]; + return acc; + }, + {} + ); + + expect(currentHashToVersionMap).toEqual(HASH_TO_VERSION_MAP); + }); +}); + +// Convert an object to an md5 hash string, using a stable serialization (canonicalStringify) +function md5Object(obj: any) { + return crypto.createHash('md5').update(canonicalStringify(obj)).digest('hex'); +} + +// JSON.stringify is non-canonical, meaning the same object may produce slightly +// different JSON, depending on compiler optimizations (e.g. object keys +// are not guaranteed to be sorted). This function consistently produces the same +// string, if passed an object of the same shape. If the outpuf of this function +// changes from one release to another, migrations will run, so it's important +// that this function remains stable across releases. +function canonicalStringify(obj: any): string { + if (Array.isArray(obj)) { + return `[${obj.map(canonicalStringify)}]`; + } + + if (!obj || typeof obj !== 'object') { + return JSON.stringify(obj); + } + + const keys = Object.keys(obj); + + // This is important for properly handling Date + if (!keys.length) { + return JSON.stringify(obj); + } + + const sortedObj = keys + .sort((a, b) => a.localeCompare(b)) + .map((k) => `${k}: ${canonicalStringify(obj[k])}`); + + return `{${sortedObj}}`; +} + +// Convert an object's values to md5 hash strings +function md5Values(obj: any) { + return mapValues(obj, md5Object); +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_md5_to_mv.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_md5_to_mv.test.ts new file mode 100644 index 0000000000000..1304044b63eb5 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_md5_to_mv.test.ts @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import type { Metadata } from '@elastic/elasticsearch/lib/api/types'; +import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { + clearLog, + deleteSavedObjectIndices, + getKibanaMigratorTestKit, + readLog, + startElasticsearch, +} from '../kibana_migrator_test_kit'; +import { delay, createType } from '../test_utils'; +import '../jest_matchers'; + +const logFilePath = Path.join(__dirname, 'v2_md5_to_mv.test.log'); + +const SOME_TYPE = createType({ + switchToModelVersionAt: '8.10.0', + name: 'some-type', + modelVersions: { + 1: { + changes: [], + }, + }, + mappings: { + properties: { + field1: { type: 'text' }, + field2: { type: 'text' }, + }, + }, +}); + +const ANOTHER_TYPE = createType({ + switchToModelVersionAt: '8.10.0', + name: 'another-type', + modelVersions: { + '1': { + changes: [], + }, + }, + mappings: { + properties: { + field1: { type: 'integer' }, + field2: { type: 'integer' }, + }, + }, +}); +const ANOTHER_TYPE_UPDATED = createType({ + switchToModelVersionAt: '8.10.0', + name: 'another-type', + modelVersions: { + '1': { + changes: [], + }, + '2': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + field3: { type: 'keyword' }, + }, + }, + ], + }, + }, + mappings: { + properties: { + field1: { type: 'integer' }, + field2: { type: 'integer' }, + field3: { type: 'keyword' }, + }, + }, +}); + +const TYPE_WITHOUT_MODEL_VERSIONS = createType({ + name: 'no-mv-type', + mappings: { + properties: { + title: { type: 'text' }, + }, + }, +}); + +const SOME_TYPE_HASH = 'someLongHashThatWeCanImagineWasCalculatedUsingMd5'; +const ANOTHER_TYPE_HASH = 'differentFromTheOneAboveAsTheRelatedTypeFieldsAreIntegers'; +const A_THIRD_HASH = 'yetAnotherHashUsedByTypeWithoutModelVersions'; +const HASH_TO_VERSION_MAP: Record = {}; +HASH_TO_VERSION_MAP[`some-type|${SOME_TYPE_HASH}`] = '10.1.0'; +// simulate that transition to modelVersion happened before 'another-type' was updated +HASH_TO_VERSION_MAP[`another-type|${ANOTHER_TYPE_HASH}`] = '10.1.0'; +HASH_TO_VERSION_MAP[`no-mv-type|${A_THIRD_HASH}`] = '0.0.0'; + +describe('V2 algorithm', () => { + let esServer: TestElasticsearchUtils['es']; + let esClient: ElasticsearchClient; + let result: MigrationResult[]; + + const getMappingMeta = async () => { + const mapping = await esClient.indices.getMapping({ index: MAIN_SAVED_OBJECT_INDEX }); + return Object.values(mapping)[0].mappings._meta; + }; + + beforeAll(async () => { + // clean ES startup + esServer = await startElasticsearch(); + }); + + describe('when started on a fresh ES deployment', () => { + beforeAll(async () => { + const { runMigrations, client } = await getKibanaMigratorTestKit({ + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, + types: [SOME_TYPE, ANOTHER_TYPE, TYPE_WITHOUT_MODEL_VERSIONS], + logFilePath, + }); + esClient = client; + + // misc cleanup + await clearLog(logFilePath); + await deleteSavedObjectIndices(client); + + result = await runMigrations(); + }); + + it('creates the SO indices, storing modelVersions in meta.mappingVersions', async () => { + expect(result[0].status === 'skipped'); + expect(await getMappingMeta()).toEqual({ + indexTypesMap: { + '.kibana': ['another-type', 'no-mv-type', 'some-type'], + }, + mappingVersions: { + 'another-type': '10.1.0', + 'no-mv-type': '0.0.0', + 'some-type': '10.1.0', + }, + }); + }); + + describe('when upgrading to a more recent version', () => { + let indexMetaAfterMigration: Metadata | undefined; + + beforeAll(async () => { + // start the migrator again, which will update meta with modelVersions + const { runMigrations: restartKibana } = await getKibanaMigratorTestKit({ + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, + // note that we are updating 'another-type' + types: [SOME_TYPE, ANOTHER_TYPE_UPDATED, TYPE_WITHOUT_MODEL_VERSIONS], + hashToVersionMap: HASH_TO_VERSION_MAP, + logFilePath, + }); + + result = await restartKibana(); + + indexMetaAfterMigration = await getMappingMeta(); + }); + + it('performs a compatible (non-reindexing) migration', () => { + expect(result[0].status).toEqual('patched'); + }); + + it('updates the SO indices meta.mappingVersions with the appropriate model versions', () => { + expect(indexMetaAfterMigration?.mappingVersions).toEqual({ + 'some-type': '10.1.0', + 'another-type': '10.2.0', + 'no-mv-type': '0.0.0', + }); + }); + + it('stores a breakdown of indices => types in the meta', () => { + expect(indexMetaAfterMigration?.indexTypesMap).toEqual({ + '.kibana': ['another-type', 'no-mv-type', 'some-type'], + }); + }); + + it('only "picks up" the types that have changed', async () => { + const logs = await readLog(logFilePath); + expect(logs).toMatch( + 'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: another-type.' + ); + }); + }); + }); + + describe('when SO indices still contain md5 hashes', () => { + let indexMetaAfterMigration: Metadata | undefined; + + beforeAll(async () => { + const { runMigrations, client } = await getKibanaMigratorTestKit({ + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, + types: [SOME_TYPE, ANOTHER_TYPE, TYPE_WITHOUT_MODEL_VERSIONS], + logFilePath, + }); + esClient = client; + + // misc cleanup + await clearLog(logFilePath); + await deleteSavedObjectIndices(client); + + await runMigrations(); + + // we update the mappings to mimic an "md5 state" + await client.indices.putMapping({ + index: MAIN_SAVED_OBJECT_INDEX, + _meta: { + migrationMappingPropertyHashes: { + 'some-type': SOME_TYPE_HASH, + 'another-type': ANOTHER_TYPE_HASH, + 'no-mv-type': A_THIRD_HASH, + }, + }, + allow_no_indices: true, + }); + + // we then start the migrator again, which will update meta with modelVersions + const { runMigrations: restartKibana } = await getKibanaMigratorTestKit({ + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, + // note that we are updating 'another-type' + types: [SOME_TYPE, ANOTHER_TYPE_UPDATED, TYPE_WITHOUT_MODEL_VERSIONS], + hashToVersionMap: HASH_TO_VERSION_MAP, + logFilePath, + }); + + result = await restartKibana(); + + indexMetaAfterMigration = await getMappingMeta(); + }); + + it('performs a compatible (non-reindexing) migration', () => { + expect(result[0].status).toEqual('patched'); + }); + + it('preserves the SO indices meta.migrationMappingPropertyHashes (although they are no longer up to date / in use)', () => { + expect(indexMetaAfterMigration?.migrationMappingPropertyHashes).toEqual({ + 'another-type': 'differentFromTheOneAboveAsTheRelatedTypeFieldsAreIntegers', + 'no-mv-type': 'yetAnotherHashUsedByTypeWithoutModelVersions', + 'some-type': 'someLongHashThatWeCanImagineWasCalculatedUsingMd5', + }); + }); + + it('adds the mappingVersions with the current modelVersions', () => { + expect(indexMetaAfterMigration?.mappingVersions).toEqual({ + 'another-type': '10.2.0', + 'no-mv-type': '0.0.0', + 'some-type': '10.1.0', + }); + }); + + it('stores a breakdown of indices => types in the meta', () => { + expect(indexMetaAfterMigration?.indexTypesMap).toEqual({ + '.kibana': ['another-type', 'no-mv-type', 'some-type'], + }); + }); + + it('only "picks up" the types that have changed', async () => { + const logs = await readLog(logFilePath); + expect(logs).toMatch( + 'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: another-type.' + ); + }); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts index 0ec675ab98b12..bfc105ee7d160 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_same_stack_version.test.ts @@ -138,7 +138,9 @@ describe('V2 algorithm - using model versions - upgrade without stack version in indexTypesMap: { '.kibana': ['test_mv'], }, - migrationMappingPropertyHashes: expect.any(Object), + mappingVersions: { + test_mv: '10.2.0', + }, }); const { saved_objects: testMvDocs } = await savedObjectsRepository.find({ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts index 65e1b07bfac3c..7a5d8a686f084 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group4/v2_with_mv_stack_version_bump.test.ts @@ -206,7 +206,10 @@ describe('V2 algorithm - using model versions - stack version bump scenario', () indexTypesMap: { '.kibana': ['test_mv', 'test_switch'], }, - migrationMappingPropertyHashes: expect.any(Object), + mappingVersions: { + test_mv: '10.2.0', + test_switch: '10.1.0', + }, }); const { saved_objects: testSwitchDocs } = await savedObjectsRepository.find({ diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index 25c548d4e0d6b..0c0717d6e7278 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -134,7 +134,7 @@ describe('split .kibana index into multiple system indices', () => { mappings: { dynamic: 'strict', _meta: { - migrationMappingPropertyHashes: expect.any(Object), + mappingVersions: expect.any(Object), indexTypesMap: expect.any(Object), }, properties: expect.any(Object), @@ -149,7 +149,7 @@ describe('split .kibana index into multiple system indices', () => { mappings: { dynamic: 'strict', _meta: { - migrationMappingPropertyHashes: expect.any(Object), + mappingVersions: expect.any(Object), indexTypesMap: expect.any(Object), }, properties: expect.any(Object), @@ -164,7 +164,7 @@ describe('split .kibana index into multiple system indices', () => { mappings: { dynamic: 'strict', _meta: { - migrationMappingPropertyHashes: expect.any(Object), + mappingVersions: expect.any(Object), indexTypesMap: expect.any(Object), }, properties: expect.any(Object), diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts index 5b2ab045ca2f0..bb216b401e885 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/pickup_updated_types_only.test.ts @@ -11,7 +11,6 @@ import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import { clearLog, createBaseline, - currentVersion, defaultKibanaIndex, defaultLogFilePath, getCompatibleMappingsMigrator, @@ -20,7 +19,6 @@ import { } from '../kibana_migrator_test_kit'; import '../jest_matchers'; import { delay, parseLogFile } from '../test_utils'; -import { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; export const logFilePath = Path.join(__dirname, 'pickup_updated_types_only.test.log'); @@ -45,7 +43,7 @@ describe('pickupUpdatedMappings', () => { const logs = await parseLogFile(defaultLogFilePath); expect(logs).not.toContainLogEntry( - 'Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings' + 'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings' ); }); }); @@ -59,7 +57,7 @@ describe('pickupUpdatedMappings', () => { const logs = await parseLogFile(defaultLogFilePath); expect(logs).toContainLogEntry( - 'Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings: complex.' + 'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: complex.' ); }); @@ -68,21 +66,21 @@ describe('pickupUpdatedMappings', () => { // we tamper the baseline mappings to simulate some root fields changes const baselineMappings = await client.indices.getMapping({ index: defaultKibanaIndex }); - const _meta = baselineMappings[`${defaultKibanaIndex}_${currentVersion}_001`].mappings - ._meta as IndexMappingMeta; - _meta.migrationMappingPropertyHashes!.namespace = - _meta.migrationMappingPropertyHashes!.namespace + '_tampered'; - await client.indices.putMapping({ index: defaultKibanaIndex, _meta }); + const properties = Object.values(baselineMappings)[0].mappings.properties!; + (properties.references as any).properties.description = { + type: 'text', + }; + await client.indices.putMapping({ index: defaultKibanaIndex, properties }); await runMigrations(); const logs = await parseLogFile(defaultLogFilePath); expect(logs).toContainLogEntry( - 'Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: namespace.' + 'Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: references.' ); expect(logs).not.toContainLogEntry( - 'Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings' + 'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings' ); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts index 7d605cf116341..fbba07aec849c 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.fixtures.ts @@ -18,6 +18,12 @@ const defaultType: SavedObjectsType = { name: { type: 'keyword' }, }, }, + modelVersions: { + 1: { + changes: [], + }, + }, + switchToModelVersionAt: '8.10.0', migrations: {}, }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts index e88a876edbc63..d748cfd16755b 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts @@ -76,6 +76,7 @@ export interface KibanaMigratorTestKitParams { settings?: Record; types?: Array>; defaultIndexTypesMap?: IndexTypesMap; + hashToVersionMap?: Record; logFilePath?: string; clientWrapperFactory?: ElasticsearchClientWrapperFactory; } @@ -131,6 +132,7 @@ export const getKibanaMigratorTestKit = async ({ settings = {}, kibanaIndex = defaultKibanaIndex, defaultIndexTypesMap = {}, // do NOT assume any types are stored in any index by default + hashToVersionMap = {}, // allows testing the md5 => modelVersion transition kibanaVersion = currentVersion, kibanaBranch = currentBranch, types = [], @@ -163,6 +165,7 @@ export const getKibanaMigratorTestKit = async ({ loggerFactory, kibanaIndex, defaultIndexTypesMap, + hashToVersionMap, kibanaVersion, kibanaBranch, nodeRoles, @@ -275,6 +278,7 @@ interface GetMigratorParams { kibanaIndex: string; typeRegistry: ISavedObjectTypeRegistry; defaultIndexTypesMap: IndexTypesMap; + hashToVersionMap: Record; loggerFactory: LoggerFactory; kibanaVersion: string; kibanaBranch: string; @@ -288,6 +292,7 @@ const getMigrator = async ({ kibanaIndex, typeRegistry, defaultIndexTypesMap, + hashToVersionMap, loggerFactory, kibanaVersion, kibanaBranch, @@ -314,6 +319,7 @@ const getMigrator = async ({ kibanaIndex, typeRegistry, defaultIndexTypesMap, + hashToVersionMap, soMigrationsConfig: soConfig.migration, kibanaVersion, logger: loggerFactory.get('savedobjects-service'), @@ -324,6 +330,17 @@ const getMigrator = async ({ }); }; +export const deleteSavedObjectIndices = async ( + client: ElasticsearchClient, + index: string[] = ALL_SAVED_OBJECT_INDICES +) => { + const indices = await client.indices.get({ index, allow_no_indices: true }, { ignore: [404] }); + return await client.indices.delete( + { index: Object.keys(indices), allow_no_indices: true }, + { ignore: [404] } + ); +}; + export const getAggregatedTypesCount = async ( client: ElasticsearchClient, index: string @@ -458,11 +475,23 @@ export const getCompatibleMappingsMigrator = async ({ ...type, mappings: { properties: { - name: { type: 'text' }, - value: { type: 'integer' }, + ...type.mappings.properties, createdAt: { type: 'date' }, }, }, + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + createdAt: { type: 'date' }, + }, + }, + ], + }, + }, }; } else { return type; @@ -486,11 +515,28 @@ export const getIncompatibleMappingsMigrator = async ({ ...type, mappings: { properties: { - name: { type: 'keyword' }, - value: { type: 'long' }, + ...type.mappings.properties, + value: { type: 'text' }, // we're forcing an incompatible udpate (number => text) createdAt: { type: 'date' }, }, }, + modelVersions: { + ...type.modelVersions, + 2: { + changes: [ + { + type: 'data_removal', // not true (we're testing reindex migrations, and modelVersions do not support breaking changes) + removedAttributePaths: ['complex.properties.value'], + }, + { + type: 'mappings_addition', + addedMappings: { + createdAt: { type: 'date' }, + }, + }, + ], + }, + }, }; } else { return type; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zdt_2/v2_to_zdt_switch.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zdt_2/v2_to_zdt_switch.test.ts index 4e437a5350cdf..4a92a9d14eef4 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zdt_2/v2_to_zdt_switch.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zdt_2/v2_to_zdt_switch.test.ts @@ -103,8 +103,9 @@ describe('ZDT upgrades - switching from v2 algorithm', () => { const records = await parseLogFile(logFilePath); expect(records).toContainLogEntries( [ - 'INIT: current algo check result: v2-compatible', - 'INIT -> UPDATE_INDEX_MAPPINGS', + 'INIT: current algo check result: v2-partially-migrated', + 'INIT: mapping version check result: equal', + 'INIT -> INDEX_STATE_UPDATE_DONE', 'INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT', 'Migration completed', ], @@ -117,13 +118,17 @@ describe('ZDT upgrades - switching from v2 algorithm', () => { it('fails and throws an explicit error', async () => { const { client } = await createBaseline({ kibanaVersion: '8.7.0' }); - // even when specifying an older version, the `indexTypeMap` will be present on the index's meta, - // so we have to manually remove it there. + // even when specifying an older version, `indexTypeMap` and `mappingVersions` will be present on the index's meta, + // so we have to manually remove them. const indices = await client.indices.get({ index: '.kibana_8.7.0_001', }); const meta = indices['.kibana_8.7.0_001'].mappings!._meta! as IndexMappingMeta; delete meta.indexTypesMap; + delete meta.mappingVersions; + meta.migrationMappingPropertyHashes = { + sample_a: 'sampleAHash', + }; await client.indices.putMapping({ index: '.kibana_8.7.0_001', _meta: meta,