diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index f93602bd99350..bfe48e43491a0 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -15,6 +15,7 @@ export { type IndexMapping, type IndexMappingMeta, type SavedObjectsTypeMappingDefinitions, + type IndexMappingMigrationStateMeta, } from './src/mappings'; export { SavedObjectsSerializer } from './src/serialization'; export { SavedObjectsTypeValidator } from './src/validation'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts index b7869bd12337d..7b2bb933fab3f 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts @@ -7,4 +7,9 @@ */ export { getTypes, getProperty, getRootProperties, getRootPropertiesObjects } from './lib'; -export type { SavedObjectsTypeMappingDefinitions, IndexMappingMeta, IndexMapping } from './types'; +export type { + SavedObjectsTypeMappingDefinitions, + IndexMappingMeta, + IndexMapping, + IndexMappingMigrationStateMeta, +} from './types'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index 0267f2ce27c1a..10faa1b03d31d 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -77,4 +77,19 @@ export interface IndexMappingMeta { * @remark: Only defined for indices using the zdt migration algorithm. */ docVersions?: { [k: string]: number }; + /** + * Info about the current state of the migration. + * Should only be present if a migration is in progress or was interrupted. + * + * @remark: Only defined for indices using the zdt migration algorithm. + */ + migrationState?: IndexMappingMigrationStateMeta; +} + +/** @internal */ +export interface IndexMappingMigrationStateMeta { + /** + * Indicates that the algorithm is currently converting the documents. + */ + convertingDocuments: boolean; } diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts index 8e7816a12fb53..01fc57d46462b 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -10,16 +10,21 @@ import type { IndexMapping, IndexMappingMeta } from '../mappings'; import type { ModelVersionMap } from './version_map'; import { assertValidModelVersion } from './conversion'; +export interface GetModelVersionsFromMappingsOpts { + mappings: IndexMapping; + source: 'mappingVersions' | 'docVersions'; + /** if specified, will filter the types with the provided list */ + knownTypes?: string[]; +} + /** * Build the version map from the specified source of the provided mappings. */ export const getModelVersionsFromMappings = ({ mappings, source, -}: { - mappings: IndexMapping; - source: 'mappingVersions' | 'docVersions'; -}): ModelVersionMap | undefined => { + knownTypes, +}: GetModelVersionsFromMappingsOpts): ModelVersionMap | undefined => { if (!mappings._meta) { return undefined; } @@ -27,25 +32,35 @@ export const getModelVersionsFromMappings = ({ return getModelVersionsFromMappingMeta({ meta: mappings._meta, source, + knownTypes, }); }; +export interface GetModelVersionsFromMappingMetaOpts { + meta: IndexMappingMeta; + source: 'mappingVersions' | 'docVersions'; + /** if specified, will filter the types with the provided list */ + knownTypes?: string[]; +} + /** * Build the version map from the specified source of the provided mappings meta. */ export const getModelVersionsFromMappingMeta = ({ meta, source, -}: { - meta: IndexMappingMeta; - source: 'mappingVersions' | 'docVersions'; -}): ModelVersionMap | undefined => { + knownTypes, +}: GetModelVersionsFromMappingMetaOpts): ModelVersionMap | undefined => { const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; if (!indexVersions) { return undefined; } + const typeSet = knownTypes ? new Set(knownTypes) : undefined; + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { - map[type] = assertValidModelVersion(rawVersion); + if (!typeSet || typeSet.has(type)) { + map[type] = assertValidModelVersion(rawVersion); + } return map; }, {}); }; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts index c4d48d66a6a06..6d1c0e3df4ce0 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts @@ -32,6 +32,9 @@ const migrationSchema = schema.object({ pollInterval: schema.number({ defaultValue: 1_500 }), skip: schema.boolean({ defaultValue: false }), retryAttempts: schema.number({ defaultValue: 15 }), + zdt: schema.object({ + metaPickupSyncDelaySec: schema.number({ min: 1, defaultValue: 120 }), + }), }); export type SavedObjectsMigrationConfigType = TypeOf; @@ -60,6 +63,7 @@ export const savedObjectsConfig: ServiceConfigDescriptor path: 'savedObjects', schema: soSchema, }; + export class SavedObjectConfig { public maxImportPayloadBytes: number; public maxImportExportSize: number; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts index 5dfdb05a0bca8..8b8e0f830261d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts @@ -7,3 +7,5 @@ */ export const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; + +export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts new file mode 100644 index 0000000000000..1181eb992be35 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { BulkOperation } from '../model/create_batches'; + +export const redactBulkOperationBatches = ( + bulkOperationBatches: BulkOperation[][] +): BulkOperationContainer[][] => { + return bulkOperationBatches.map((batch) => + batch.map((operation) => (Array.isArray(operation) ? operation[0] : operation)) + ); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts index 22c58ae62adcb..c83cab1c1d0bc 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts @@ -15,8 +15,8 @@ import type { SavedObjectSanitizedDoc, SavedObjectsRawDoc, SavedObjectUnsanitizedDoc, + ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import { SavedObjectsSerializer } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrateAndConvertFn } from '../document_migrator/document_migrator'; import { TransformSavedObjectDocumentError } from '.'; @@ -65,7 +65,7 @@ export class CorruptSavedObjectError extends Error { * @returns {SavedObjectsRawDoc[]} */ export async function migrateRawDocs( - serializer: SavedObjectsSerializer, + serializer: ISavedObjectsSerializer, migrateDoc: MigrateAndConvertFn, rawDocs: SavedObjectsRawDoc[] ): Promise { @@ -86,7 +86,7 @@ export async function migrateRawDocs( } interface MigrateRawDocsSafelyDeps { - serializer: SavedObjectsSerializer; + serializer: ISavedObjectsSerializer; migrateDoc: MigrateAndConvertFn; rawDocs: SavedObjectsRawDoc[]; } @@ -181,7 +181,7 @@ function transformNonBlocking( async function migrateMapToRawDoc( migrateMethod: MigrateFn, savedObject: SavedObjectSanitizedDoc, - serializer: SavedObjectsSerializer + serializer: ISavedObjectsSerializer ): Promise { return [...(await migrateMethod(savedObject))].map((attrs) => serializer.savedObjectToRaw({ @@ -201,7 +201,7 @@ async function migrateMapToRawDoc( function convertToRawAddMigrationVersion( rawDoc: SavedObjectsRawDoc, options: { namespaceTreatment: 'lax' }, - serializer: SavedObjectsSerializer + serializer: ISavedObjectsSerializer ): SavedObjectSanitizedDoc { const savedObject = serializer.rawToSavedObject(rawDoc, options); if (!savedObject.migrationVersion && !savedObject.typeMigrationVersion) { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index f5a462ebcedac..e6855a1256b54 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -288,6 +288,9 @@ const mockOptions = () => { scrollDuration: '10m', skip: false, retryAttempts: 20, + zdt: { + metaPickupSyncDelaySec: 120, + }, }, client: mockedClient, docLinks: docLinksServiceMock.createSetupContract(), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts index 48a3bad0d0960..d03b4f7378da4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts @@ -51,6 +51,9 @@ describe('migrationsStateActionMachine', () => { scrollDuration: '0s', skip: false, retryAttempts: 5, + zdt: { + metaPickupSyncDelaySec: 120, + }, }, typeRegistry, docLinks, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts index 1b5caf3c4e75d..92b0054bd47c3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts @@ -15,12 +15,11 @@ import { getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; import { logActionResponse, logStateTransition } from './common/utils/logs'; import { type Model, type Next, stateActionMachine } from './state_action_machine'; import { cleanup } from './migrations_state_machine_cleanup'; import type { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './state'; -import type { BulkOperation } from './model/create_batches'; +import { redactBulkOperationBatches } from './common/redact_state'; /** * A specialized migrations-specific state-action machine that: @@ -159,11 +158,3 @@ export async function migrationStateActionMachine({ } } } - -const redactBulkOperationBatches = ( - bulkOperationBatches: BulkOperation[][] -): BulkOperationContainer[][] => { - return bulkOperationBatches.map((batch) => - batch.map((operation) => (Array.isArray(operation) ? operation[0] : operation)) - ); -}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts index ec19f834d9ceb..008d074b2cd6f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts @@ -14,6 +14,7 @@ import type { TransformErrorObjects } from '../core'; export type BulkIndexOperationTuple = [BulkOperationContainer, SavedObjectsRawDocSource]; export type BulkOperation = BulkIndexOperationTuple | BulkOperationContainer; +export type BulkOperationBatch = BulkOperation[]; export interface CreateBatchesParams { documents: SavedObjectsRawDoc[]; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 2f78d8745e55a..23ddee5043261 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -46,9 +46,10 @@ import { } from './helpers'; import { createBatches } from './create_batches'; import type { MigrationLog } from '../types'; -import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../common/constants'; - -export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; +import { + CLUSTER_SHARD_LIMIT_EXCEEDED_REASON, + FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, +} from '../common/constants'; export const model = (currentState: State, resW: ResponseType): State => { // The action response `resW` is weakly typed, the type includes all action diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index bb135c115ce92..a3db45a3748cc 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -6,17 +6,7 @@ * Side Public License, v 1. */ -import type { - IncompatibleClusterRoutingAllocation, - RetryableEsClientError, - WaitForTaskCompletionTimeout, - IndexNotYellowTimeout, - IndexNotGreenTimeout, - ClusterShardLimitExceeded, - IndexNotFound, - AliasNotFound, - IncompatibleMappingException, -} from '../../actions'; +import type { ActionErrorTypeMap as BaseActionErrorTypeMap } from '../../actions'; export { initAction as init, @@ -25,7 +15,16 @@ export { updateAliases, updateMappings, updateAndPickupMappings, + cleanupUnknownAndExcluded, + waitForDeleteByQueryTask, waitForPickupUpdatedMappingsTask, + refreshIndex, + openPit, + readWithPit, + closePit, + transformDocs, + bulkOverwriteTransformedDocuments, + noop, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -33,17 +32,11 @@ export { type IndexNotFound, } from '../../actions'; -export interface ActionErrorTypeMap { - wait_for_task_completion_timeout: WaitForTaskCompletionTimeout; - incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation; - retryable_es_client_error: RetryableEsClientError; - index_not_found_exception: IndexNotFound; - index_not_green_timeout: IndexNotGreenTimeout; - index_not_yellow_timeout: IndexNotYellowTimeout; - cluster_shard_limit_exceeded: ClusterShardLimitExceeded; - alias_not_found_exception: AliasNotFound; - incompatible_mapping_exception: IncompatibleMappingException; -} +export { updateIndexMeta, type UpdateIndexMetaParams } from './update_index_meta'; +export { waitForDelay, type WaitForDelayParams } from './wait_for_delay'; + +// alias in case we need to extend it with zdt specific actions/errors +export type ActionErrorTypeMap = BaseActionErrorTypeMap; /** Type guard for narrowing the type of a left */ export function isTypeof( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts new file mode 100644 index 0000000000000..8f209365a285e --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const updateMappingsMock = jest.fn(); + +jest.doMock('../../actions/update_mappings', () => { + const actual = jest.requireActual('../../actions/update_mappings'); + return { + ...actual, + updateMappings: updateMappingsMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts new file mode 100644 index 0000000000000..6ed55ccb49ebe --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { updateMappingsMock } from './update_index_meta.test.mocks'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { updateIndexMeta } from './update_index_meta'; + +describe('updateIndexMeta', () => { + it('calls updateMappings with the correct parameters', () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + const index = '.kibana_1'; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + }, + }; + + updateIndexMeta({ client, index, meta }); + + expect(updateMappingsMock).toHaveBeenCalledTimes(1); + expect(updateMappingsMock).toHaveBeenCalledWith({ + client, + index, + mappings: { + properties: {}, + _meta: meta, + }, + }); + }); + + it('returns the response from updateMappings', () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + const index = '.kibana_1'; + const meta: IndexMappingMeta = {}; + + const expected = Symbol(); + updateMappingsMock.mockReturnValue(expected); + + const actual = updateIndexMeta({ client, index, meta }); + + expect(actual).toBe(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts new file mode 100644 index 0000000000000..195e282dce5aa --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { updateMappings } from '../../actions'; + +export interface UpdateIndexMetaParams { + client: ElasticsearchClient; + index: string; + meta: IndexMappingMeta; +} + +export const updateIndexMeta = ({ + client, + index, + meta, +}: UpdateIndexMetaParams): ReturnType => { + return updateMappings({ + client, + index, + mappings: { + properties: {}, + _meta: meta, + }, + }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts new file mode 100644 index 0000000000000..fbfc144bd8150 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { waitForDelay } from './wait_for_delay'; + +const nextTick = () => new Promise((resolve) => resolve()); +const aFewTicks = () => nextTick().then(nextTick).then(nextTick); + +describe('waitForDelay', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('resolves after the specified amount of time', async () => { + const handler = jest.fn(); + + waitForDelay({ delayInSec: 5 })().then(handler); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(5000); + await aFewTicks(); + + expect(handler).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts new file mode 100644 index 0000000000000..302a702331de4 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import * as TaskEither from 'fp-ts/lib/TaskEither'; + +export interface WaitForDelayParams { + delayInSec: number; +} + +export const waitForDelay = ({ + delayInSec, +}: WaitForDelayParams): TaskEither.TaskEither => { + return () => { + return delay(delayInSec) + .then(() => Either.right('wait_succeeded' as const)) + .catch((err) => { + // will never happen + throw err; + }); + }; +}; + +const delay = (delayInSec: number) => + new Promise((resolve) => setTimeout(resolve, delayInSec * 1000)); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index 7a660ea470443..c31d4bc799b3b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -21,12 +21,15 @@ export const createContext = ({ types, docLinks, migrationConfig, + documentMigrator, elasticsearchClient, indexPrefix, typeRegistry, serializer, }: CreateContextOps): MigratorContext => { return { + migrationConfig, + documentMigrator, kibanaVersion, indexPrefix, types, @@ -37,5 +40,6 @@ export const createContext = ({ maxRetryAttempts: migrationConfig.retryAttempts, migrationDocLinks: docLinks.links.kibanaUpgradeSavedObjects, deletedTypes: REMOVED_TYPES, + discardCorruptObjects: Boolean(migrationConfig.discardCorruptObjects), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 5b6d4b2fe27e9..95ca7282daf57 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -11,13 +11,19 @@ import type { ISavedObjectTypeRegistry, ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import type { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { + ModelVersionMap, + SavedObjectsMigrationConfigType, +} from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; +import { VersionedTransformer } from '../../document_migrator'; /** * The set of static, precomputed values and services used by the ZDT migration */ export interface MigratorContext { + /** The migration configuration */ + readonly migrationConfig: SavedObjectsMigrationConfigType; /** The current Kibana version */ readonly kibanaVersion: string; /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ @@ -34,8 +40,12 @@ export interface MigratorContext { readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects']; /** SO serializer to use for migration */ readonly serializer: ISavedObjectsSerializer; + /** The doc migrator to use */ + readonly documentMigrator: VersionedTransformer; /** The SO type registry to use for the migration */ readonly typeRegistry: ISavedObjectTypeRegistry; /** List of types that are no longer registered */ readonly deletedTypes: string[]; + /** If true, corrupted objects will be discarded instead of failing the migration */ + readonly discardCorruptObjects: boolean; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts index 8982a1a9c6c7e..cb6a30ce50c6f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts @@ -12,11 +12,17 @@ import { getErrorMessage, getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; +import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; import { logStateTransition, logActionResponse } from '../common/utils'; import { type Next, stateActionMachine } from '../state_action_machine'; import { cleanup } from '../migrations_state_machine_cleanup'; -import type { State } from './state'; +import type { + State, + OutdatedDocumentsSearchTransformState, + OutdatedDocumentsSearchBulkIndexState, +} from './state'; import type { MigratorContext } from './context'; +import { redactBulkOperationBatches } from '../common/redact_state'; /** * A specialized migrations-specific state-action machine that: @@ -60,23 +66,12 @@ export async function migrationStateActionMachine({ // the _id's of documents const redactedNewState = { ...newState, - /* TODO: commented until we have model stages that process outdated docs. (attrs not on model atm) - ...{ - outdatedDocuments: ( - (newState as ReindexSourceToTempTransform).outdatedDocuments ?? [] - ).map( - (doc) => - ({ - _id: doc._id, - } as SavedObjectsRawDoc) - ), - }, - ...{ - transformedDocBatches: ( - (newState as ReindexSourceToTempIndexBulk).transformedDocBatches ?? [] - ).map((batches) => batches.map((doc) => ({ _id: doc._id }))) as [SavedObjectsRawDoc[]], - }, - */ + outdatedDocuments: ( + (newState as OutdatedDocumentsSearchTransformState).outdatedDocuments ?? [] + ).map((doc) => ({ _id: doc._id } as SavedObjectsRawDoc)), + bulkOperationBatches: redactBulkOperationBatches( + (newState as OutdatedDocumentsSearchBulkIndexState).bulkOperationBatches ?? [[]] + ), }; const now = Date.now(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts index c0316b954e5f3..bef92e1986748 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts @@ -6,13 +6,13 @@ * Side Public License, v 1. */ -import { StageMocks } from './model.test.mocks'; +import './model.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { createContextMock, MockedMigratorContext } from '../test_helpers'; import type { RetryableEsClientError } from '../../actions'; import type { State, BaseState, FatalState, AllActionStates } from '../state'; import type { StateActionResponse } from './types'; -import { model } from './model'; +import { model, modelStageMap } from './model'; describe('model', () => { let context: MockedMigratorContext; @@ -128,16 +128,7 @@ describe('model', () => { }, }); - const stageMapping: Record = { - INIT: StageMocks.init, - CREATE_TARGET_INDEX: StageMocks.createTargetIndex, - UPDATE_INDEX_MAPPINGS: StageMocks.updateIndexMappings, - UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: StageMocks.updateIndexMappingsWaitForTask, - UPDATE_MAPPING_MODEL_VERSIONS: StageMocks.updateMappingModelVersion, - UPDATE_ALIASES: StageMocks.updateAliases, - }; - - Object.entries(stageMapping).forEach(([stage, handler]) => { + Object.entries(modelStageMap).forEach(([stage, handler]) => { test(`dispatch ${stage} state`, () => { const state = createStubState(stage as AllActionStates); const res = createStubResponse(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 62971c3a614aa..483e97e3a7f85 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -12,9 +12,44 @@ import type { ResponseType } from '../next'; import { delayRetryState, resetRetryState } from '../../model/retry_state'; import { throwBadControlState } from '../../model/helpers'; import { isTypeof } from '../actions'; -import { MigratorContext } from '../context'; +import type { MigratorContext } from '../context'; +import type { ModelStage } from './types'; import * as Stages from './stages'; -import { StateActionResponse } from './types'; + +type ModelStageMap = { + [K in AllActionStates]: ModelStage; +}; + +type AnyModelStageHandler = ( + state: State, + response: Either.Either, + ctx: MigratorContext +) => State; + +export const modelStageMap: ModelStageMap = { + INIT: Stages.init, + CREATE_TARGET_INDEX: Stages.createTargetIndex, + UPDATE_ALIASES: Stages.updateAliases, + UPDATE_INDEX_MAPPINGS: Stages.updateIndexMappings, + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: Stages.updateIndexMappingsWaitForTask, + UPDATE_MAPPING_MODEL_VERSIONS: Stages.updateMappingModelVersion, + INDEX_STATE_UPDATE_DONE: Stages.indexStateUpdateDone, + DOCUMENTS_UPDATE_INIT: Stages.documentsUpdateInit, + SET_DOC_MIGRATION_STARTED: Stages.setDocMigrationStarted, + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: Stages.setDocMigrationStartedWaitForInstances, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: Stages.cleanupUnknownAndExcludedDocs, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: Stages.cleanupUnknownAndExcludedDocsWaitForTask, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: Stages.cleanupUnknownAndExcludedDocsRefresh, + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: Stages.outdatedDocumentsSearchOpenPit, + OUTDATED_DOCUMENTS_SEARCH_READ: Stages.outdatedDocumentsSearchRead, + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: Stages.outdatedDocumentsSearchTransform, + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: Stages.outdatedDocumentsSearchBulkIndex, + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: Stages.outdatedDocumentsSearchClosePit, + OUTDATED_DOCUMENTS_SEARCH_REFRESH: Stages.outdatedDocumentsSearchRefresh, + UPDATE_DOCUMENT_MODEL_VERSIONS: Stages.updateDocumentModelVersion, + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: + Stages.updateDocumentModelVersionWaitForInstances, +}; export const model = ( current: State, @@ -29,44 +64,14 @@ export const model = ( current = resetRetryState(current); } - switch (current.controlState) { - case 'INIT': - return Stages.init(current, response as StateActionResponse<'INIT'>, context); - case 'CREATE_TARGET_INDEX': - return Stages.createTargetIndex( - current, - response as StateActionResponse<'CREATE_TARGET_INDEX'>, - context - ); - case 'UPDATE_ALIASES': - return Stages.updateAliases( - current, - response as StateActionResponse<'UPDATE_ALIASES'>, - context - ); - case 'UPDATE_INDEX_MAPPINGS': - return Stages.updateIndexMappings( - current, - response as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, - context - ); - case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': - return Stages.updateIndexMappingsWaitForTask( - current, - response as StateActionResponse<'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'>, - context - ); - case 'UPDATE_MAPPING_MODEL_VERSIONS': - return Stages.updateMappingModelVersion( - current, - response as StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'>, - context - ); - case 'DONE': - case 'FATAL': - // The state-action machine will never call the model in the terminating states - return throwBadControlState(current as never); - default: - return throwBadControlState(current); + if (current.controlState === 'DONE' || current.controlState === 'FATAL') { + return throwBadControlState(current as never); + } + + const stageHandler = modelStageMap[current.controlState] as AnyModelStageHandler; + if (!stageHandler) { + return throwBadControlState(current as never); } + + return stageHandler(current, response, context); }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts new file mode 100644 index 0000000000000..29730ab1ec789 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocs } from './cleanup_unknown_and_excluded_docs'; + +describe('Stage: cleanupUnknownAndExcludedDocs', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'> = Either.right({ + type: 'cleanup_started', + taskId: '42', + errorsByType: {}, + unknownDocs: [], + }); + + const newState = cleanupUnknownAndExcludedDocs(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: '42', + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> FATAL when unsuccessful', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'> = Either.left({ + type: 'unknown_docs_found', + unknownDocs: [], + }); + + const newState = cleanupUnknownAndExcludedDocs(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts new file mode 100644 index 0000000000000..6aca449a73de5 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { extractUnknownDocFailureReason } from '../../../model/extract_errors'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocs: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + return { + ...state, + controlState: 'FATAL', + reason: extractUnknownDocFailureReason( + context.migrationDocLinks.resolveMigrationFailures, + res.left.unknownDocs + ), + }; + } + + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: res.right.taskId, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts new file mode 100644 index 0000000000000..10d7ecc6a3841 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsRefreshState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocsRefresh } from './cleanup_unknown_and_excluded_docs_refresh'; + +describe('Stage: cleanupUnknownAndExcludedDocsRefresh', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsRefreshState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT when successful', () => { + const state = createState(); + const res = Either.right({ + refreshed: true, + }) as StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH'>; + + const newState = cleanupUnknownAndExcludedDocsRefresh(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts new file mode 100644 index 0000000000000..32c4dc5bf6634 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocsRefresh: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts new file mode 100644 index 0000000000000..e6deae5590770 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsWaitForTaskState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocsWaitForTask } from './cleanup_unknown_and_excluded_docs_wait_for_task'; + +describe('Stage: cleanupUnknownAndExcludedDocsWaitForTask', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsWaitForTaskState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: '42', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK in case of wait_for_task_completion_timeout', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'wait_for_task_completion_timeout', + message: 'woups', + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + retryCount: 1, + retryDelay: expect.any(Number), + logs: expect.any(Array), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS in case of cleanup_failed', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'cleanup_failed', + failures: [], + versionConflicts: 42, + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + hasDeletedDocs: true, + retryCount: 1, + retryDelay: expect.any(Number), + logs: expect.any(Array), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> FATAL in case of cleanup_failed when exceeding retry count', () => { + const state = createState({ + retryCount: context.maxRetryAttempts + 1, + }); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'cleanup_failed', + failures: [], + versionConflicts: 42, + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH when successful and docs were deleted', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = + Either.right({ + type: 'cleanup_successful', + deleted: 9000, + }); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT when successful and no docs were deleted', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = + Either.right({ + type: 'cleanup_successful', + deleted: 0, + }); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts new file mode 100644 index 0000000000000..83baaccea9921 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocsWaitForTask: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + | 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH' + | 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' + | 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' + | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + if (isTypeof(res.left, 'wait_for_task_completion_timeout')) { + // After waiting for the specified timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticsearch task succeeds or fails. + return delayRetryState(state, res.left.message, Number.MAX_SAFE_INTEGER); + } else { + if (state.retryCount < context.maxRetryAttempts) { + const retryCount = state.retryCount + 1; + const retryDelay = 1500 + 1000 * Math.random(); + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + hasDeletedDocs: true, + retryCount, + retryDelay, + logs: [ + ...state.logs, + { + level: 'warning', + message: `Errors occurred whilst deleting unwanted documents. Retrying attempt ${retryCount}.`, + }, + ], + }; + } else { + const failures = res.left.failures.length; + const versionConflicts = res.left.versionConflicts ?? 0; + let reason = `Migration failed because it was unable to delete unwanted documents from the ${state.currentIndex} system index (${failures} failures and ${versionConflicts} conflicts)`; + if (failures) { + reason += `:\n` + res.left.failures.map((failure: string) => `- ${failure}\n`).join(''); + } + return { + ...state, + controlState: 'FATAL', + reason, + }; + } + } + } + + const mustRefresh = + state.hasDeletedDocs || typeof res.right.deleted === 'undefined' || res.right.deleted > 0; + + if (mustRefresh) { + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + }; + } else { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts index cd15594d32b29..4aaee33a8f974 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts @@ -84,7 +84,7 @@ describe('Stage: createTargetIndex', () => { }); }); - it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful', () => { + it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful and alias actions are not empty', () => { const state = createState(); const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.right('create_index_succeeded'); @@ -101,6 +101,27 @@ describe('Stage: createTargetIndex', () => { currentIndexMeta: state.indexMappings._meta, aliases: [], aliasActions, + newIndexCreation: true, + }); + }); + + it('CREATE_TARGET_INDEX -> INDEX_STATE_UPDATE_DONE when successful and alias actions are empty', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + getAliasActionsMock.mockReturnValue([]); + + const newState = createTargetIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'INDEX_STATE_UPDATE_DONE', + previousMappings: state.indexMappings, + currentIndexMeta: state.indexMappings._meta, + aliases: [], + aliasActions: [], + newIndexCreation: true, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts index bb0697a70cf51..ab0f421822675 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts @@ -15,11 +15,10 @@ import { isTypeof } from '../../actions'; import { getAliasActions } from '../../utils'; import type { ModelStage } from '../types'; -export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASES' | 'FATAL'> = ( - state, - res, - context -) => { +export const createTargetIndex: ModelStage< + 'CREATE_TARGET_INDEX', + 'UPDATE_ALIASES' | 'INDEX_STATE_UPDATE_DONE' | 'FATAL' +> = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'index_not_green_timeout')) { @@ -48,10 +47,11 @@ export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASE return { ...state, - controlState: 'UPDATE_ALIASES', + controlState: aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', previousMappings: state.indexMappings, currentIndexMeta, aliases: [], aliasActions, + newIndexCreation: true, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts new file mode 100644 index 0000000000000..a16bed57fec65 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getOutdatedDocumentsQueryMock = jest.fn(); +export const createDocumentTransformFnMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getOutdatedDocumentsQuery: getOutdatedDocumentsQueryMock, + createDocumentTransformFn: createDocumentTransformFnMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts new file mode 100644 index 0000000000000..abf8a689ad621 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + getOutdatedDocumentsQueryMock, + createDocumentTransformFnMock, +} from './documents_update_init.test.mocks'; +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { DocumentsUpdateInitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { documentsUpdateInit } from './documents_update_init'; +import { createType } from '../../test_helpers'; + +describe('Stage: documentsUpdateInit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): DocumentsUpdateInitState => ({ + ...createPostInitState(), + controlState: 'DOCUMENTS_UPDATE_INIT', + ...parts, + }); + + beforeEach(() => { + getOutdatedDocumentsQueryMock.mockReset(); + createDocumentTransformFnMock.mockReset(); + + context = createContextMock(); + context.typeRegistry.registerType(createType({ name: 'foo' })); + context.typeRegistry.registerType(createType({ name: 'bar' })); + }); + + it('calls getOutdatedDocumentsQuery with the correct parameters', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + documentsUpdateInit(state, res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, context); + + expect(getOutdatedDocumentsQueryMock).toHaveBeenCalledTimes(1); + expect(getOutdatedDocumentsQueryMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + }); + }); + + it('calls createDocumentTransformFn with the correct parameters', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + documentsUpdateInit(state, res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, context); + + expect(createDocumentTransformFnMock).toHaveBeenCalledTimes(1); + expect(createDocumentTransformFnMock).toHaveBeenCalledWith({ + serializer: context.serializer, + documentMigrator: context.documentMigrator, + }); + }); + + it('DOCUMENTS_UPDATE_INIT -> SET_DOC_MIGRATION_STARTED when successful', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + const newState = documentsUpdateInit( + state, + res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, + context + ); + expect(newState.controlState).toEqual('SET_DOC_MIGRATION_STARTED'); + }); + + it('updates the state with the expected properties', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + const transformRawDocs = jest.fn(); + createDocumentTransformFnMock.mockReturnValue(transformRawDocs); + + const outdatedDocumentsQuery = Symbol(); + getOutdatedDocumentsQueryMock.mockReturnValue(outdatedDocumentsQuery); + + const newState = documentsUpdateInit( + state, + res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED', + transformRawDocs, + outdatedDocumentsQuery, + excludeFromUpgradeFilterHooks: expect.any(Object), + excludeOnUpgradeQuery: expect.any(Object), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts new file mode 100644 index 0000000000000..a4f94e3cbe7b2 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { excludeUnusedTypesQuery } from '../../../core'; +import type { ModelStage } from '../types'; +import { getOutdatedDocumentsQuery, createDocumentTransformFn } from '../../utils'; + +export const documentsUpdateInit: ModelStage< + 'DOCUMENTS_UPDATE_INIT', + 'SET_DOC_MIGRATION_STARTED' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const excludeFilterHooks = Object.fromEntries( + context.types + .map((name) => context.typeRegistry.getType(name)!) + .filter((type) => !!type.excludeOnUpgrade) + .map((type) => [type.name, type.excludeOnUpgrade!]) + ); + + const types = context.types.map((type) => context.typeRegistry.getType(type)!); + const outdatedDocumentsQuery = getOutdatedDocumentsQuery({ types }); + + const transformRawDocs = createDocumentTransformFn({ + serializer: context.serializer, + documentMigrator: context.documentMigrator, + }); + + return { + ...state, + excludeOnUpgradeQuery: excludeUnusedTypesQuery, + excludeFromUpgradeFilterHooks: excludeFilterHooks, + outdatedDocumentsQuery, + transformRawDocs, + controlState: 'SET_DOC_MIGRATION_STARTED', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index 0322f92eb35aa..532f15b5417ed 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -12,3 +12,18 @@ export { updateAliases } from './update_aliases'; export { updateIndexMappings } from './update_index_mappings'; export { updateIndexMappingsWaitForTask } from './update_index_mappings_wait_for_task'; export { updateMappingModelVersion } from './update_mapping_model_version'; +export { indexStateUpdateDone } from './index_state_update_done'; +export { documentsUpdateInit } from './documents_update_init'; +export { setDocMigrationStarted } from './set_doc_migration_started'; +export { setDocMigrationStartedWaitForInstances } from './set_doc_migration_started_wait_for_instances'; +export { cleanupUnknownAndExcludedDocs } from './cleanup_unknown_and_excluded_docs'; +export { cleanupUnknownAndExcludedDocsWaitForTask } from './cleanup_unknown_and_excluded_docs_wait_for_task'; +export { cleanupUnknownAndExcludedDocsRefresh } from './cleanup_unknown_and_excluded_docs_refresh'; +export { outdatedDocumentsSearchOpenPit } from './outdated_documents_search_open_pit'; +export { outdatedDocumentsSearchRead } from './outdated_documents_search_read'; +export { outdatedDocumentsSearchTransform } from './outdated_documents_search_transform'; +export { outdatedDocumentsSearchBulkIndex } from './outdated_documents_search_bulk_index'; +export { outdatedDocumentsSearchClosePit } from './outdated_documents_search_close_pit'; +export { outdatedDocumentsSearchRefresh } from './outdated_documents_search_refresh'; +export { updateDocumentModelVersion } from './update_document_model_version'; +export { updateDocumentModelVersionWaitForInstances } from './update_document_model_version_wait_for_instances'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts new file mode 100644 index 0000000000000..2b523887109b8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { IndexStateUpdateDoneState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { indexStateUpdateDone } from './index_state_update_done'; + +describe('Stage: indexStateUpdateDone', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): IndexStateUpdateDoneState => ({ + ...createPostDocInitState(), + controlState: 'INDEX_STATE_UPDATE_DONE', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT when successful and newIndexCreation is false', () => { + const state = createState({ + newIndexCreation: false, + }); + const res = Either.right('noop') as StateActionResponse<'INDEX_STATE_UPDATE_DONE'>; + + const newState = indexStateUpdateDone(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DOCUMENTS_UPDATE_INIT', + }); + }); + + it('INDEX_STATE_UPDATE_DONE -> DONE when successful and newIndexCreation is true', () => { + const state = createState({ + newIndexCreation: true, + }); + const res = Either.right('noop') as StateActionResponse<'INDEX_STATE_UPDATE_DONE'>; + + const newState = indexStateUpdateDone(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts new file mode 100644 index 0000000000000..2ca41886c8b6c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const indexStateUpdateDone: ModelStage< + 'INDEX_STATE_UPDATE_DONE', + 'DOCUMENTS_UPDATE_INIT' | 'DONE' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + if (state.newIndexCreation) { + // we created the index, so we can safely skip the whole document migration + // and go directly to DONE + return { + ...state, + controlState: 'DONE', + }; + } else { + return { + ...state, + controlState: 'DOCUMENTS_UPDATE_INIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts index 8f773fe951171..89c513fa2a66d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -10,6 +10,7 @@ export const getCurrentIndexMock = jest.fn(); export const checkVersionCompatibilityMock = jest.fn(); export const buildIndexMappingsMock = jest.fn(); export const generateAdditiveMappingDiffMock = jest.fn(); +export const getAliasActionsMock = jest.fn(); jest.doMock('../../utils', () => { const realModule = jest.requireActual('../../utils'); @@ -19,5 +20,6 @@ jest.doMock('../../utils', () => { checkVersionCompatibility: checkVersionCompatibilityMock, buildIndexMappings: buildIndexMappingsMock, generateAdditiveMappingDiff: generateAdditiveMappingDiffMock, + getAliasActions: getAliasActionsMock, }; }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index ea6f4424404ef..d8c176af4be0a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -11,6 +11,7 @@ import { checkVersionCompatibilityMock, buildIndexMappingsMock, generateAdditiveMappingDiffMock, + getAliasActionsMock, } from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { FetchIndexResponse } from '../../../actions'; @@ -49,6 +50,7 @@ describe('Stage: init', () => { status: 'equal', }); generateAdditiveMappingDiffMock.mockReset().mockReturnValue({}); + getAliasActionsMock.mockReset().mockReturnValue([]); context = createContextMock({ indexPrefix: '.kibana', types: ['foo', 'bar'] }); context.typeRegistry.registerType({ @@ -65,7 +67,7 @@ describe('Stage: init', () => { }); }); - it('loops to INIT when cluster routing allocation is incompatible', () => { + it('INIT -> INIT when cluster routing allocation is incompatible', () => { const state = createState(); const res: StateActionResponse<'INIT'> = Either.left({ type: 'incompatible_cluster_routing_allocation', @@ -124,7 +126,7 @@ describe('Stage: init', () => { }); }); - it('forwards to CREATE_TARGET_INDEX', () => { + it('INIT -> CREATE_TARGET_INDEX', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -164,7 +166,7 @@ describe('Stage: init', () => { }); }); - it('forwards to UPDATE_INDEX_MAPPINGS', () => { + it('INIT -> UPDATE_INDEX_MAPPINGS', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -182,6 +184,7 @@ describe('Stage: init', () => { currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, additiveMappingChanges: { someToken: {} }, + newIndexCreation: false, }) ); }); @@ -203,7 +206,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `equal`', () => { - it('forwards to UPDATE_ALIASES', () => { + it('INIT -> UPDATE_ALIASES if alias actions are not empty', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -211,6 +214,7 @@ describe('Stage: init', () => { checkVersionCompatibilityMock.mockReturnValue({ status: 'equal', }); + getAliasActionsMock.mockReturnValue([{ add: { index: '.kibana_1', alias: '.kibana' } }]); const newState = init(state, res, context); @@ -219,6 +223,29 @@ describe('Stage: init', () => { controlState: 'UPDATE_ALIASES', currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, + newIndexCreation: false, + }) + ); + }); + + it('INIT -> INDEX_STATE_UPDATE_DONE if alias actions are empty', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + getAliasActionsMock.mockReturnValue([]); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'INDEX_STATE_UPDATE_DONE', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + newIndexCreation: false, }) ); }); @@ -240,7 +267,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `lesser`', () => { - it('forwards to FATAL', () => { + it('INIT -> FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -276,7 +303,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `conflict`', () => { - it('forwards to FATAL', () => { + it('INIT -> FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index da138730364f4..19538cc1a4a85 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -23,7 +23,11 @@ import type { ModelStage } from '../types'; export const init: ModelStage< 'INIT', - 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_ALIASES' | 'FATAL' + | 'CREATE_TARGET_INDEX' + | 'UPDATE_INDEX_MAPPINGS' + | 'UPDATE_ALIASES' + | 'INDEX_STATE_UPDATE_DONE' + | 'FATAL' > = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; @@ -78,6 +82,15 @@ export const init: ModelStage< // cloning as we may be mutating it in later stages. const currentIndexMeta = cloneDeep(currentMappings._meta!); + const commonState = { + logs, + currentIndex, + currentIndexMeta, + aliases, + aliasActions, + previousMappings: currentMappings, + }; + switch (versionCheck.status) { // app version is greater than the index mapping version. // scenario of an upgrade: we need to update the mappings @@ -90,13 +103,9 @@ export const init: ModelStage< return { ...state, controlState: 'UPDATE_INDEX_MAPPINGS', - logs, - currentIndex, - currentIndexMeta, - aliases, - aliasActions, - previousMappings: currentMappings, + ...commonState, additiveMappingChanges, + newIndexCreation: false, }; // app version and index mapping version are the same. // either application upgrade without model change, or a simple reboot on the same version. @@ -104,13 +113,9 @@ export const init: ModelStage< case 'equal': return { ...state, - controlState: 'UPDATE_ALIASES', - logs, - currentIndex, - currentIndexMeta, - aliases, - aliasActions, - previousMappings: currentMappings, + controlState: aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', + ...commonState, + newIndexCreation: false, }; // app version is lower than the index mapping version. // likely a rollback scenario - unsupported for the initial implementation diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts new file mode 100644 index 0000000000000..323a8ba646875 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchBulkIndexState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchBulkIndex } from './outdated_documents_search_bulk_index'; + +describe('Stage: outdatedDocumentsSearchBulkIndex', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchBulkIndexState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + bulkOperationBatches: [], + currentBatch: 0, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX when there are remaining batches', () => { + const state = createState({ + currentBatch: 0, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res = Either.right( + 'bulk_index_succeeded' + ) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'>; + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: 1, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_READ when there are no remaining batches', () => { + const state = createState({ + currentBatch: 1, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'> = + Either.right('bulk_index_succeeded'); + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: true, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> FATAL in case of request_entity_too_large_exception', () => { + const state = createState({ + currentBatch: 1, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'> = Either.left({ + type: 'request_entity_too_large_exception', + }); + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts new file mode 100644 index 0000000000000..4d6bba19c5154 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { FATAL_REASON_REQUEST_ENTITY_TOO_LARGE } from '../../../common/constants'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchBulkIndex: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + 'OUTDATED_DOCUMENTS_SEARCH_READ' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + if (isTypeof(res.left, 'request_entity_too_large_exception')) { + return { + ...state, + controlState: 'FATAL', + reason: FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, + }; + } else if ( + isTypeof(res.left, 'target_index_had_write_block') || + isTypeof(res.left, 'index_not_found_exception') + ) { + // we fail on these errors since the target index will never get + // deleted and should only have a write block if a newer version of + // Kibana started an upgrade + throwBadResponse(state, res.left as never); + } else { + throwBadResponse(state, res.left); + } + } + + if (state.currentBatch + 1 < state.bulkOperationBatches.length) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: state.currentBatch + 1, + }; + } + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: true, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts new file mode 100644 index 0000000000000..a5db9fd119a80 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchClosePitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchClosePit } from './outdated_documents_search_close_pit'; + +describe('Stage: outdatedDocumentsSearchClosePit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchClosePitState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_SEARCH_REFRESH when documents were transformed', () => { + const state = createState({ + hasTransformedDocs: true, + }); + const res = Either.right({}) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'>; + + const newState = outdatedDocumentsSearchClosePit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> UPDATE_DOCUMENT_MODEL_VERSIONS when no documents were transformed', () => { + const state = createState({ + hasTransformedDocs: false, + }); + const res = Either.right({}) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'>; + + const newState = outdatedDocumentsSearchClosePit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts new file mode 100644 index 0000000000000..a467a4a7d90fa --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchClosePit: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH' | 'UPDATE_DOCUMENT_MODEL_VERSIONS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const { hasTransformedDocs } = state; + if (hasTransformedDocs) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + }; + } else { + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts new file mode 100644 index 0000000000000..90417d66f1a3d --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchOpenPitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchOpenPit } from './outdated_documents_search_open_pit'; + +describe('Stage: outdatedDocumentsSearchOpenPit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchOpenPitState => ({ + ...createPostDocInitState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ when successful', () => { + const state = createState(); + const res = Either.right({ + pitId: '42', + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'>; + + const newState = outdatedDocumentsSearchOpenPit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + pitId: '42', + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + progress: { + processed: undefined, + total: undefined, + }, + hasTransformedDocs: false, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts new file mode 100644 index 0000000000000..f7a76e57779c3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { createInitialProgress } from '../../../model/progress'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchOpenPit: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + 'OUTDATED_DOCUMENTS_SEARCH_READ' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const pitId = res.right.pitId; + + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + pitId, + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + progress: createInitialProgress(), + hasTransformedDocs: false, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts new file mode 100644 index 0000000000000..de312c966dbda --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + createSavedObjectRawDoc, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchReadState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchRead } from './outdated_documents_search_read'; + +describe('Stage: outdatedDocumentsSearchRead', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchReadState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_TRANSFORM when outdated documents are found', () => { + const state = createState({ + progress: { + total: 300, + processed: 0, + }, + }); + const outdatedDocuments = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + const res = Either.right({ + outdatedDocuments, + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments, + lastHitSortValue: [12, 24], + logs: expect.any(Array), + progress: { + total: 9000, + processed: 0, + }, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when no more outdated documents', () => { + const state = createState({ + progress: { + total: 300, + processed: 0, + }, + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL when corrupt ids are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + corruptDocumentIds: ['foo_1', 'bar_2'], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL when transform errors are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + transformErrors: [{ rawId: 'foo_1', err: new Error('woups') }], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + //// + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when corrupt ids are are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: true, + }); + const state = createState({ + corruptDocumentIds: ['foo_1', 'bar_2'], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when transform errors are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: true, + }); + const state = createState({ + transformErrors: [{ rawId: 'foo_1', err: new Error('woups') }], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts new file mode 100644 index 0000000000000..d3e8b2b5b9018 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; +import { logProgress, setProgressTotal } from '../../../model/progress'; +import { + extractDiscardedCorruptDocs, + extractTransformFailuresReason, +} from '../../../model/extract_errors'; + +export const outdatedDocumentsSearchRead: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_READ', + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM' | 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + let logs = state.logs; + + if (res.right.outdatedDocuments.length > 0) { + // search returned outdated documents, so we process them + const progress = setProgressTotal(state.progress, res.right.totalHits); + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments: res.right.outdatedDocuments, + lastHitSortValue: res.right.lastHitSortValue, + logs: logProgress(state.logs, progress), + progress, + }; + } else { + // no more outdated documents , we need to move on + if (state.corruptDocumentIds.length > 0 || state.transformErrors.length > 0) { + if (!context.discardCorruptObjects) { + const transformFailureReason = extractTransformFailuresReason( + context.migrationDocLinks.resolveMigrationFailures, + state.corruptDocumentIds, + state.transformErrors + ); + return { + ...state, + controlState: 'FATAL', + reason: transformFailureReason, + }; + } + + // at this point, users have configured kibana to discard corrupt objects + // thus, we can ignore corrupt documents and transform errors and proceed with the migration + logs = [ + ...state.logs, + { + level: 'warning', + message: extractDiscardedCorruptDocs(state.corruptDocumentIds, state.transformErrors), + }, + ]; + } + + // If there are no more results we have transformed all outdated + // documents and we didn't encounter any corrupt documents or transformation errors + // and can proceed to the next step + return { + ...state, + logs, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts new file mode 100644 index 0000000000000..9584d97b2ace1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchRefreshState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchRefresh } from './outdated_documents_search_refresh'; + +describe('Stage: outdatedDocumentsSearchRefresh', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchRefreshState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_REFRESH -> UPDATE_DOCUMENT_MODEL_VERSIONS when successful', () => { + const state = createState({}); + const res = Either.right({ + refreshed: true, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_REFRESH'>; + + const newState = outdatedDocumentsSearchRefresh(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts new file mode 100644 index 0000000000000..0816174c5b585 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchRefresh: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + 'UPDATE_DOCUMENT_MODEL_VERSIONS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts new file mode 100644 index 0000000000000..6c08e34678214 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + createSavedObjectRawDoc, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchTransformState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchTransform } from './outdated_documents_search_transform'; + +describe('Stage: outdatedDocumentsSearchTransform', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchTransformState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments: [], + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX when outdated documents were converted', () => { + const state = createState({ + progress: { + processed: 0, + total: 100, + }, + outdatedDocuments: [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ], + }); + const processedDocs = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + const res = Either.right({ + processedDocs, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'>; + + const newState = outdatedDocumentsSearchTransform(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: 0, + hasTransformedDocs: true, + bulkOperationBatches: expect.any(Array), + progress: { + processed: 2, + total: 100, + }, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ in case of documents_transform_failed when discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + progress: { + processed: 0, + total: 100, + }, + outdatedDocuments: [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ], + corruptDocumentIds: ['init_1'], + }); + const processedDocs = [ + createSavedObjectRawDoc({ _id: '3' }), + createSavedObjectRawDoc({ _id: '4' }), + ]; + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'> = Either.left({ + type: 'documents_transform_failed', + processedDocs, + corruptDocumentIds: ['foo_1', 'bar_2'], + transformErrors: [], + }); + + const newState = outdatedDocumentsSearchTransform(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: ['init_1', 'foo_1', 'bar_2'], + transformErrors: [], + hasTransformedDocs: false, + progress: { + processed: 2, + total: 100, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts new file mode 100644 index 0000000000000..b174a57096e17 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; +import { incrementProcessedProgress } from '../../../model/progress'; +import { fatalReasonDocumentExceedsMaxBatchSizeBytes } from '../../../model/extract_errors'; +import { createBatches } from '../../../model/create_batches'; +import { isTypeof } from '../../actions'; + +export const outdatedDocumentsSearchTransform: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX' | 'OUTDATED_DOCUMENTS_SEARCH_READ' | 'FATAL' +> = (state, res, context) => { + // Increment the processed documents, no matter what the results are. + // Otherwise the progress might look off when there are errors. + const progress = incrementProcessedProgress(state.progress, state.outdatedDocuments.length); + const discardCorruptObjects = context.discardCorruptObjects; + if ( + Either.isRight(res) || + (isTypeof(res.left, 'documents_transform_failed') && discardCorruptObjects) + ) { + // we might have some transformation errors, but user has chosen to discard them + if ( + (state.corruptDocumentIds.length === 0 && state.transformErrors.length === 0) || + discardCorruptObjects + ) { + const documents = Either.isRight(res) ? res.right.processedDocs : res.left.processedDocs; + + let corruptDocumentIds = state.corruptDocumentIds; + let transformErrors = state.transformErrors; + + if (Either.isLeft(res)) { + corruptDocumentIds = [...state.corruptDocumentIds, ...res.left.corruptDocumentIds]; + transformErrors = [...state.transformErrors, ...res.left.transformErrors]; + } + + const batches = createBatches({ + documents, + corruptDocumentIds, + transformErrors, + maxBatchSizeBytes: context.migrationConfig.maxBatchSizeBytes.getValueInBytes(), + }); + if (Either.isRight(batches)) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + bulkOperationBatches: batches.right, + currentBatch: 0, + hasTransformedDocs: true, + progress, + }; + } else { + return { + ...state, + controlState: 'FATAL', + reason: fatalReasonDocumentExceedsMaxBatchSizeBytes({ + _id: batches.left.documentId, + docSizeBytes: batches.left.docSizeBytes, + maxBatchSizeBytes: batches.left.maxBatchSizeBytes, + }), + }; + } + } else { + // We have seen corrupt documents and/or transformation errors + // skip indexing and go straight to reading and transforming more docs + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + progress, + }; + } + } else { + if (isTypeof(res.left, 'documents_transform_failed')) { + // continue to build up any more transformation errors before failing the migration. + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [...state.corruptDocumentIds, ...res.left.corruptDocumentIds], + transformErrors: [...state.transformErrors, ...res.left.transformErrors], + hasTransformedDocs: false, + progress, + }; + } else { + throwBadResponse(state, res as never); + } + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts new file mode 100644 index 0000000000000..fca84ab858a2f --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { SetDocMigrationStartedState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { setDocMigrationStarted } from './set_doc_migration_started'; + +describe('Stage: setDocMigrationStarted', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): SetDocMigrationStartedState => ({ + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('SET_DOC_MIGRATION_STARTED -> SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES when successful', () => { + const state = createState(); + const res: StateActionResponse<'SET_DOC_MIGRATION_STARTED'> = Either.right( + 'update_mappings_succeeded' as const + ); + + const newState = setDocMigrationStarted(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + currentIndexMeta: { + ...state.currentIndexMeta, + migrationState: { + ...state.currentIndexMeta.migrationState, + convertingDocuments: true, + }, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts new file mode 100644 index 0000000000000..f1c24dbba3861 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { setMetaDocMigrationStarted } from '../../utils'; +import type { ModelStage } from '../types'; + +export const setDocMigrationStarted: ModelStage< + 'SET_DOC_MIGRATION_STARTED', + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + currentIndexMeta: setMetaDocMigrationStarted({ + meta: state.currentIndexMeta, + }), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts new file mode 100644 index 0000000000000..530430d79586e --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { SetDocMigrationStartedWaitForInstancesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { setDocMigrationStartedWaitForInstances } from './set_doc_migration_started_wait_for_instances'; + +describe('Stage: setDocMigrationStartedWaitForInstances', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): SetDocMigrationStartedWaitForInstancesState => ({ + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS when successful', () => { + const state = createState(); + const res = Either.right( + 'wait_succeeded' as const + ) as StateActionResponse<'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'>; + + const newState = setDocMigrationStartedWaitForInstances(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts new file mode 100644 index 0000000000000..7053f3149579a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const setDocMigrationStartedWaitForInstances: ModelStage< + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts index 4fac3d02db044..9849e6b464032 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts @@ -60,7 +60,7 @@ describe('Stage: updateAliases', () => { }); }); - it('UPDATE_ALIASES -> DONE if successful', () => { + it('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE if successful', () => { const state = createState(); const res: StateActionResponse<'UPDATE_ALIASES'> = Either.right('update_aliases_succeeded'); @@ -68,7 +68,7 @@ describe('Stage: updateAliases', () => { expect(newState).toEqual({ ...state, - controlState: 'DONE', + controlState: 'INDEX_STATE_UPDATE_DONE', }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts index 4d91eb116871b..5d7f0914d0d52 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts @@ -11,7 +11,7 @@ import { throwBadResponse } from '../../../model/helpers'; import { isTypeof } from '../../actions'; import type { ModelStage } from '../types'; -export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( +export const updateAliases: ModelStage<'UPDATE_ALIASES', 'INDEX_STATE_UPDATE_DONE' | 'FATAL'> = ( state, res, context @@ -41,6 +41,6 @@ export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( return { ...state, - controlState: 'DONE', + controlState: 'INDEX_STATE_UPDATE_DONE', }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts new file mode 100644 index 0000000000000..c3d3fd67422b9 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateDocumentModelVersionsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateDocumentModelVersion } from './update_document_model_version'; + +describe('Stage: updateDocumentModelVersion', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateDocumentModelVersionsState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_DOCUMENT_MODEL_VERSIONS -> UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES when successful', () => { + const state = createState({}); + const res = Either.right( + 'update_mappings_succeeded' + ) as StateActionResponse<'UPDATE_DOCUMENT_MODEL_VERSIONS'>; + + const newState = updateDocumentModelVersion(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + currentIndexMeta: expect.any(Object), + }); + }); + + it('updates state.currentIndexMeta when successful', () => { + const state = createState({ + currentIndexMeta: { + mappingVersions: { foo: 1, bar: 2 }, + docVersions: { foo: 0, bar: 0 }, + migrationState: { + convertingDocuments: true, + }, + }, + }); + const res = Either.right( + 'update_mappings_succeeded' + ) as StateActionResponse<'UPDATE_DOCUMENT_MODEL_VERSIONS'>; + + const newState = updateDocumentModelVersion(state, res, context); + + expect(newState.currentIndexMeta).toEqual({ + mappingVersions: { foo: 1, bar: 2 }, + docVersions: { foo: 1, bar: 2 }, + migrationState: { + convertingDocuments: false, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts new file mode 100644 index 0000000000000..ea3ea142cbd19 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { setMetaDocMigrationComplete } from '../../utils'; +import type { ModelStage } from '../types'; + +export const updateDocumentModelVersion: ModelStage< + 'UPDATE_DOCUMENT_MODEL_VERSIONS', + 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + currentIndexMeta: setMetaDocMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts new file mode 100644 index 0000000000000..b2ea37919418b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateDocumentModelVersionsWaitForInstancesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateDocumentModelVersionWaitForInstances } from './update_document_model_version_wait_for_instances'; + +describe('Stage: updateDocumentModelVersionWaitForInstances', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateDocumentModelVersionsWaitForInstancesState => ({ + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES -> DONE when successful', () => { + const state = createState(); + const res = Either.right( + 'wait_succeeded' as const + ) as StateActionResponse<'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'>; + + const newState = updateDocumentModelVersionWaitForInstances(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts new file mode 100644 index 0000000000000..82e3ce7c10cdd --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateDocumentModelVersionWaitForInstances: ModelStage< + 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + 'DONE' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts index 9856cb0c5a1e5..c47ef54030b3b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts @@ -33,11 +33,5 @@ export const updateIndexMappingsWaitForTask: ModelStage< return { ...state, controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', - currentIndexMeta: { - ...state.currentIndexMeta, - mappingVersions: { - ...context.typeModelVersions, - }, - }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts index 971482d3262b7..b21ec69a531f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -12,20 +12,19 @@ import { createPostInitState, type MockedMigratorContext, } from '../../test_helpers'; -import type { ResponseType } from '../../next'; -import type { UpdateIndexMappingsState } from '../../state'; +import type { UpdateMappingModelVersionState } from '../../state'; import type { StateActionResponse } from '../types'; -import { updateIndexMappings } from './update_index_mappings'; +import { updateMappingModelVersion } from './update_mapping_model_version'; +import { setMetaMappingMigrationComplete } from '../../utils'; -describe('Stage: updateIndexMappings', () => { +describe('Stage: updateMappingModelVersion', () => { let context: MockedMigratorContext; const createState = ( - parts: Partial = {} - ): UpdateIndexMappingsState => ({ + parts: Partial = {} + ): UpdateMappingModelVersionState => ({ ...createPostInitState(), - controlState: 'UPDATE_INDEX_MAPPINGS', - additiveMappingChanges: {}, + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', ...parts, }); @@ -33,21 +32,50 @@ describe('Stage: updateIndexMappings', () => { context = createContextMock(); }); - it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { - const state = createState(); - const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ - taskId: '42', + it('updates state.currentIndexMeta', () => { + const state = createState({}); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' + ); + + const newState = updateMappingModelVersion(state, res, context); + expect(newState.currentIndexMeta).toEqual( + setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }) + ); + }); + + it('UPDATE_MAPPING_MODEL_VERSIONS -> UPDATE_ALIASES when at least one aliasActions', () => { + const state = createState({ + aliasActions: [{ add: { alias: '.kibana', index: '.kibana_1' } }], }); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' + ); - const newState = updateIndexMappings( - state, - res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, - context + const newState = updateMappingModelVersion(state, res, context); + expect(newState).toEqual({ + ...state, + currentIndexMeta: expect.any(Object), + controlState: 'UPDATE_ALIASES', + }); + }); + + it('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE when no aliasActions', () => { + const state = createState({ + aliasActions: [], + }); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' ); + + const newState = updateMappingModelVersion(state, res, context); expect(newState).toEqual({ ...state, - controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', - updateTargetMappingsTaskId: '42', + currentIndexMeta: expect.any(Object), + controlState: 'INDEX_STATE_UPDATE_DONE', }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts index 8b4df56fc83a7..946c4a4ab1ef3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts @@ -9,10 +9,11 @@ import * as Either from 'fp-ts/lib/Either'; import { throwBadResponse } from '../../../model/helpers'; import type { ModelStage } from '../types'; +import { setMetaMappingMigrationComplete } from '../../utils'; export const updateMappingModelVersion: ModelStage< 'UPDATE_MAPPING_MODEL_VERSIONS', - 'DONE' | 'FATAL' + 'UPDATE_ALIASES' | 'INDEX_STATE_UPDATE_DONE' > = (state, res, context) => { if (Either.isLeft(res)) { throwBadResponse(state, res as never); @@ -20,6 +21,10 @@ export const updateMappingModelVersion: ModelStage< return { ...state, - controlState: 'DONE', + controlState: state.aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', + currentIndexMeta: setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts new file mode 100644 index 0000000000000..fc274bcfd1214 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const setMetaMappingMigrationCompleteMock = jest.fn(); +export const setMetaDocMigrationCompleteMock = jest.fn(); +export const setMetaDocMigrationStartedMock = jest.fn(); + +jest.doMock('./utils', () => { + const actual = jest.requireActual('./utils'); + return { + ...actual, + setMetaDocMigrationStarted: setMetaDocMigrationStartedMock, + setMetaMappingMigrationComplete: setMetaMappingMigrationCompleteMock, + setMetaDocMigrationComplete: setMetaDocMigrationCompleteMock, + }; +}); + +const realActions = jest.requireActual('./actions'); + +export const ActionMocks = Object.keys(realActions).reduce((mocks, key) => { + mocks[key] = jest.fn().mockImplementation((state: unknown) => state); + return mocks; +}, {} as Record>); + +jest.doMock('./actions', () => ActionMocks); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts new file mode 100644 index 0000000000000..d9135fff65a3e --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + ActionMocks, + setMetaDocMigrationStartedMock, + setMetaDocMigrationCompleteMock, + setMetaMappingMigrationCompleteMock, +} from './next.test.mocks'; +import { nextActionMap, type ActionMap } from './next'; +import { + createContextMock, + type MockedMigratorContext, + createPostDocInitState, +} from './test_helpers'; +import type { + SetDocMigrationStartedState, + UpdateMappingModelVersionState, + UpdateDocumentModelVersionsState, +} from './state'; + +describe('actions', () => { + let context: MockedMigratorContext; + let actionMap: ActionMap; + + beforeEach(() => { + jest.clearAllMocks(); + + context = createContextMock(); + actionMap = nextActionMap(context); + }); + + describe('SET_DOC_MIGRATION_STARTED', () => { + it('calls setMetaDocMigrationStarted with the correct parameters', () => { + const state: SetDocMigrationStartedState = { + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + }; + const action = actionMap.SET_DOC_MIGRATION_STARTED; + + action(state); + + expect(setMetaDocMigrationStartedMock).toHaveBeenCalledTimes(1); + expect(setMetaDocMigrationStartedMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: SetDocMigrationStartedState = { + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + }; + const action = actionMap.SET_DOC_MIGRATION_STARTED; + + const someMeta = { some: 'meta' }; + setMetaDocMigrationStartedMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); + + describe('UPDATE_MAPPING_MODEL_VERSIONS', () => { + it('calls setMetaMappingMigrationComplete with the correct parameters', () => { + const state: UpdateMappingModelVersionState = { + ...createPostDocInitState(), + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_MAPPING_MODEL_VERSIONS; + + action(state); + + expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledTimes(1); + expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: UpdateMappingModelVersionState = { + ...createPostDocInitState(), + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_MAPPING_MODEL_VERSIONS; + + const someMeta = { some: 'meta' }; + setMetaMappingMigrationCompleteMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); + + describe('UPDATE_DOCUMENT_MODEL_VERSIONS', () => { + it('calls setMetaDocMigrationComplete with the correct parameters', () => { + const state: UpdateDocumentModelVersionsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_DOCUMENT_MODEL_VERSIONS; + + action(state); + + expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledTimes(1); + expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: UpdateDocumentModelVersionsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_DOCUMENT_MODEL_VERSIONS; + + const someMeta = { some: 'meta' }; + setMetaDocMigrationCompleteMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index a85e9bfde6b56..cb3e1b5b5ad27 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -15,10 +15,30 @@ import type { UpdateIndexMappingsWaitForTaskState, UpdateMappingModelVersionState, UpdateAliasesState, + CleanupUnknownAndExcludedDocsState, + CleanupUnknownAndExcludedDocsWaitForTaskState, + DocumentsUpdateInitState, + IndexStateUpdateDoneState, + OutdatedDocumentsSearchBulkIndexState, + OutdatedDocumentsSearchClosePitState, + OutdatedDocumentsSearchOpenPitState, + OutdatedDocumentsSearchReadState, + OutdatedDocumentsSearchTransformState, + CleanupUnknownAndExcludedDocsRefreshState, + SetDocMigrationStartedState, + SetDocMigrationStartedWaitForInstancesState, + OutdatedDocumentsSearchRefreshState, + UpdateDocumentModelVersionsState, + UpdateDocumentModelVersionsWaitForInstancesState, } from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; import { createDelayFn } from '../common/utils'; +import { + setMetaMappingMigrationComplete, + setMetaDocMigrationComplete, + setMetaDocMigrationStarted, +} from './utils'; export type ActionMap = ReturnType; @@ -59,19 +79,108 @@ export const nextActionMap = (context: MigratorContext) => { timeout: '60s', }), UPDATE_MAPPING_MODEL_VERSIONS: (state: UpdateMappingModelVersionState) => - Actions.updateMappings({ + Actions.updateIndexMeta({ client, index: state.currentIndex, - mappings: { - properties: {}, - _meta: state.currentIndexMeta, - }, + meta: setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), }), UPDATE_ALIASES: (state: UpdateAliasesState) => Actions.updateAliases({ client, aliasActions: state.aliasActions, }), + INDEX_STATE_UPDATE_DONE: (state: IndexStateUpdateDoneState) => () => Actions.noop(), + DOCUMENTS_UPDATE_INIT: (state: DocumentsUpdateInitState) => () => Actions.noop(), + SET_DOC_MIGRATION_STARTED: (state: SetDocMigrationStartedState) => + Actions.updateIndexMeta({ + client, + index: state.currentIndex, + meta: setMetaDocMigrationStarted({ + meta: state.currentIndexMeta, + }), + }), + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: ( + state: SetDocMigrationStartedWaitForInstancesState + ) => + Actions.waitForDelay({ + delayInSec: context.migrationConfig.zdt.metaPickupSyncDelaySec, + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: (state: CleanupUnknownAndExcludedDocsState) => + Actions.cleanupUnknownAndExcluded({ + client, + indexName: state.currentIndex, + discardUnknownDocs: true, + excludeOnUpgradeQuery: state.excludeOnUpgradeQuery, + excludeFromUpgradeFilterHooks: state.excludeFromUpgradeFilterHooks, + knownTypes: context.types, + removedTypes: context.deletedTypes, + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: ( + state: CleanupUnknownAndExcludedDocsWaitForTaskState + ) => + Actions.waitForDeleteByQueryTask({ + client, + taskId: state.deleteTaskId, + timeout: '120s', + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: (state: CleanupUnknownAndExcludedDocsRefreshState) => + Actions.refreshIndex({ + client, + index: state.currentIndex, + }), + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: (state: OutdatedDocumentsSearchOpenPitState) => + Actions.openPit({ + client, + index: state.currentIndex, + }), + OUTDATED_DOCUMENTS_SEARCH_READ: (state: OutdatedDocumentsSearchReadState) => + Actions.readWithPit({ + client, + pitId: state.pitId, + searchAfter: state.lastHitSortValue, + batchSize: context.migrationConfig.batchSize, + query: state.outdatedDocumentsQuery, + }), + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: (state: OutdatedDocumentsSearchTransformState) => + Actions.transformDocs({ + outdatedDocuments: state.outdatedDocuments, + transformRawDocs: state.transformRawDocs, + }), + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: (state: OutdatedDocumentsSearchBulkIndexState) => + Actions.bulkOverwriteTransformedDocuments({ + client, + index: state.currentIndex, + operations: state.bulkOperationBatches[state.currentBatch], + refresh: false, + }), + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: (state: OutdatedDocumentsSearchClosePitState) => + Actions.closePit({ + client, + pitId: state.pitId, + }), + OUTDATED_DOCUMENTS_SEARCH_REFRESH: (state: OutdatedDocumentsSearchRefreshState) => + Actions.refreshIndex({ + client, + index: state.currentIndex, + }), + UPDATE_DOCUMENT_MODEL_VERSIONS: (state: UpdateDocumentModelVersionsState) => + Actions.updateIndexMeta({ + client, + index: state.currentIndex, + meta: setMetaDocMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), + }), + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: ( + state: UpdateDocumentModelVersionsWaitForInstancesState + ) => + Actions.waitForDelay({ + delayInSec: context.migrationConfig.zdt.metaPickupSyncDelaySec, + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index 0f7d28507bb4a..45d958720a912 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -21,5 +21,20 @@ export type { AllControlStates, StateFromActionState, StateFromControlState, + IndexStateUpdateDoneState, + DocumentsUpdateInitState, + SetDocMigrationStartedState, + SetDocMigrationStartedWaitForInstancesState, + CleanupUnknownAndExcludedDocsState, + CleanupUnknownAndExcludedDocsWaitForTaskState, + CleanupUnknownAndExcludedDocsRefreshState, + OutdatedDocumentsSearchOpenPitState, + OutdatedDocumentsSearchReadState, + OutdatedDocumentsSearchTransformState, + OutdatedDocumentsSearchBulkIndexState, + OutdatedDocumentsSearchClosePitState, + OutdatedDocumentsSearchRefreshState, + UpdateDocumentModelVersionsState, + UpdateDocumentModelVersionsWaitForInstancesState, } from './types'; export { createInitialState } from './create_initial_state'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index d43c6e49dd5e5..123df0455c9bf 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -6,11 +6,18 @@ * Side Public License, v 1. */ -import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { + SavedObjectsRawDoc, + SavedObjectsMappingProperties, + SavedObjectTypeExcludeFromUpgradeFilterHook, +} from '@kbn/core-saved-objects-server'; import type { IndexMapping, IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; -import type { MigrationLog } from '../../types'; +import type { MigrationLog, Progress, TransformRawDocs } from '../../types'; import type { ControlState } from '../../state_action_machine'; +import type { BulkOperationBatch } from '../../model/create_batches'; import type { AliasAction } from '../../actions'; +import { TransformErrorObjects } from '../../core'; export interface BaseState extends ControlState { readonly retryCount: number; @@ -23,6 +30,9 @@ export interface InitState extends BaseState { readonly controlState: 'INIT'; } +/** + * Common state properties available after the `INIT` stage + */ export interface PostInitState extends BaseState { /** * The index we're currently migrating. @@ -46,6 +56,35 @@ export interface PostInitState extends BaseState { * All operations updating this field will update in the state accordingly. */ readonly currentIndexMeta: IndexMappingMeta; + /** + * When true, will fully skip document migration after the INDEX_STATE_UPDATE_DONE stage. + * Used when 'upgrading' a fresh cluster (via CREATE_TARGET_INDEX), as we create + * the index with the correct meta and because we're sure we don't need to migrate documents + * in that case. + */ + readonly newIndexCreation: boolean; +} + +/** + * Common state properties available after the `DOCUMENTS_UPDATE_INIT` stage + */ +export interface PostDocInitState extends PostInitState { + readonly excludeOnUpgradeQuery: QueryDslQueryContainer; + readonly excludeFromUpgradeFilterHooks: Record< + string, + SavedObjectTypeExcludeFromUpgradeFilterHook + >; + readonly outdatedDocumentsQuery: QueryDslQueryContainer; + readonly transformRawDocs: TransformRawDocs; +} + +export interface OutdatedDocumentsSearchState extends PostDocInitState { + readonly pitId: string; + readonly lastHitSortValue: number[] | undefined; + readonly corruptDocumentIds: string[]; + readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; + readonly hasTransformedDocs: boolean; } export interface CreateTargetIndexState extends BaseState { @@ -72,6 +111,72 @@ export interface UpdateAliasesState extends PostInitState { readonly controlState: 'UPDATE_ALIASES'; } +export interface IndexStateUpdateDoneState extends PostInitState { + readonly controlState: 'INDEX_STATE_UPDATE_DONE'; +} + +export interface DocumentsUpdateInitState extends PostInitState { + readonly controlState: 'DOCUMENTS_UPDATE_INIT'; +} + +export interface SetDocMigrationStartedState extends PostDocInitState { + readonly controlState: 'SET_DOC_MIGRATION_STARTED'; +} + +export interface SetDocMigrationStartedWaitForInstancesState extends PostDocInitState { + readonly controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'; +} + +export interface CleanupUnknownAndExcludedDocsState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'; + readonly hasDeletedDocs?: boolean; +} + +export interface CleanupUnknownAndExcludedDocsWaitForTaskState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'; + readonly deleteTaskId: string; + readonly hasDeletedDocs?: boolean; +} + +export interface CleanupUnknownAndExcludedDocsRefreshState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH'; +} + +export interface OutdatedDocumentsSearchOpenPitState extends PostDocInitState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'; +} + +export interface OutdatedDocumentsSearchReadState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ'; +} + +export interface OutdatedDocumentsSearchTransformState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'; + readonly outdatedDocuments: SavedObjectsRawDoc[]; +} + +export interface OutdatedDocumentsSearchBulkIndexState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'; + readonly bulkOperationBatches: BulkOperationBatch[]; + readonly currentBatch: number; +} + +export interface OutdatedDocumentsSearchClosePitState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'; +} + +export interface OutdatedDocumentsSearchRefreshState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH'; +} + +export interface UpdateDocumentModelVersionsState extends PostDocInitState { + readonly controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS'; +} + +export interface UpdateDocumentModelVersionsWaitForInstancesState extends PostInitState { + readonly controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES'; +} + /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; @@ -92,7 +197,22 @@ export type State = | UpdateIndexMappingsState | UpdateIndexMappingsWaitForTaskState | UpdateMappingModelVersionState - | UpdateAliasesState; + | UpdateAliasesState + | IndexStateUpdateDoneState + | DocumentsUpdateInitState + | SetDocMigrationStartedState + | SetDocMigrationStartedWaitForInstancesState + | CleanupUnknownAndExcludedDocsState + | CleanupUnknownAndExcludedDocsWaitForTaskState + | CleanupUnknownAndExcludedDocsRefreshState + | OutdatedDocumentsSearchOpenPitState + | OutdatedDocumentsSearchReadState + | OutdatedDocumentsSearchTransformState + | OutdatedDocumentsSearchBulkIndexState + | OutdatedDocumentsSearchClosePitState + | UpdateDocumentModelVersionsState + | UpdateDocumentModelVersionsWaitForInstancesState + | OutdatedDocumentsSearchRefreshState; export type AllControlStates = State['controlState']; @@ -110,6 +230,21 @@ export interface ControlStateMap { UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; UPDATE_MAPPING_MODEL_VERSIONS: UpdateMappingModelVersionState; UPDATE_ALIASES: UpdateAliasesState; + INDEX_STATE_UPDATE_DONE: IndexStateUpdateDoneState; + DOCUMENTS_UPDATE_INIT: DocumentsUpdateInitState; + SET_DOC_MIGRATION_STARTED: SetDocMigrationStartedState; + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: SetDocMigrationStartedWaitForInstancesState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: CleanupUnknownAndExcludedDocsState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: CleanupUnknownAndExcludedDocsWaitForTaskState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: CleanupUnknownAndExcludedDocsRefreshState; + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: OutdatedDocumentsSearchOpenPitState; + OUTDATED_DOCUMENTS_SEARCH_READ: OutdatedDocumentsSearchReadState; + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: OutdatedDocumentsSearchTransformState; + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: OutdatedDocumentsSearchBulkIndexState; + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: OutdatedDocumentsSearchClosePitState; + OUTDATED_DOCUMENTS_SEARCH_REFRESH: OutdatedDocumentsSearchRefreshState; + UPDATE_DOCUMENT_MODEL_VERSIONS: UpdateDocumentModelVersionsState; + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: UpdateDocumentModelVersionsWaitForInstancesState; } /** diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index faf9f9c89c9f5..0dafc36108b93 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { ByteSizeValue } from '@kbn/config-schema'; import { ElasticsearchClientMock, elasticsearchClientMock, @@ -14,6 +15,7 @@ import { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-int import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import type { MigratorContext } from '../context'; +import { createDocumentMigrator } from './document_migrator'; export type MockedMigratorContext = Omit & { elasticsearchClient: ElasticsearchClientMock; @@ -33,12 +35,26 @@ export const createContextMock = ( foo: 1, bar: 2, }, + documentMigrator: createDocumentMigrator(), + migrationConfig: { + algorithm: 'zdt', + batchSize: 1000, + maxBatchSizeBytes: new ByteSizeValue(1e8), + pollInterval: 0, + scrollDuration: '0s', + skip: false, + retryAttempts: 5, + zdt: { + metaPickupSyncDelaySec: 120, + }, + }, elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(), maxRetryAttempts: 15, migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects, typeRegistry, serializer: serializerMock.create(), deletedTypes: ['deleted-type'], + discardCorruptObjects: false, ...parts, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts new file mode 100644 index 0000000000000..524da05952055 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { VersionedTransformer } from '../../document_migrator'; + +export const createDocumentMigrator = (): jest.Mocked => { + return { + migrationVersion: {}, + migrate: jest.fn().mockImplementation((doc: unknown) => doc), + migrateAndConvert: jest.fn().mockImplementation((doc: unknown) => [doc]), + prepareMigrations: jest.fn(), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts index 5658828fc2e0c..8b79eef0069f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts @@ -7,5 +7,11 @@ */ export { createContextMock, type MockedMigratorContext } from './context'; -export { createPostInitState } from './state'; +export { + createPostInitState, + createPostDocInitState, + createOutdatedDocumentSearchState, +} from './state'; export { createType } from './saved_object_type'; +export { createDocumentMigrator } from './document_migrator'; +export { createSavedObjectRawDoc } from './saved_object'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts new file mode 100644 index 0000000000000..9b5520a8508db --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; + +export const createSavedObjectRawDoc = ( + parts: Partial +): SavedObjectsRawDoc => ({ + _id: '42', + _source: { + type: 'some-type', + }, + ...parts, +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts index bd95881abbba4..b91f2482326cd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { PostInitState } from '../state/types'; +import { PostInitState, PostDocInitState, OutdatedDocumentsSearchState } from '../state/types'; export const createPostInitState = (): PostInitState => ({ controlState: 'INIT', @@ -18,4 +18,26 @@ export const createPostInitState = (): PostInitState => ({ aliasActions: [], previousMappings: { properties: {} }, currentIndexMeta: {}, + newIndexCreation: false, +}); + +export const createPostDocInitState = (): PostDocInitState => ({ + ...createPostInitState(), + excludeOnUpgradeQuery: { bool: {} }, + excludeFromUpgradeFilterHooks: {}, + outdatedDocumentsQuery: { bool: {} }, + transformRawDocs: jest.fn(), +}); + +export const createOutdatedDocumentSearchState = (): OutdatedDocumentsSearchState => ({ + ...createPostDocInitState(), + pitId: '42', + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: false, + progress: { + processed: undefined, + total: undefined, + }, }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts index f4536cf1c75b0..35354001f6803 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts @@ -78,6 +78,9 @@ describe('buildIndexMeta', () => { bar: 1, dolly: 3, }, + migrationState: { + convertingDocuments: false, + }, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts index 6221221ab993c..a75ebd4dbdc1e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -53,5 +53,8 @@ export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta return { mappingVersions: modelVersions, docVersions: modelVersions, + migrationState: { + convertingDocuments: false, + }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts index 6ad12656229fc..8430f1f898426 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts @@ -62,6 +62,7 @@ describe('checkVersionCompatibility', () => { expect(getModelVersionsFromMappingsMock).toHaveBeenCalledWith({ mappings, source: 'mappingVersions', + knownTypes: ['foo', 'bar'], }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts index 4499ce419d34a..c231645fb7993 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -29,7 +29,11 @@ export const checkVersionCompatibility = ({ deletedTypes, }: CheckVersionCompatibilityOpts): CompareModelVersionResult => { const appVersions = getModelVersionMapForTypes(types); - const indexVersions = getModelVersionsFromMappings({ mappings, source }); + const indexVersions = getModelVersionsFromMappings({ + mappings, + source, + knownTypes: types.map((type) => type.name), + }); if (!indexVersions) { throw new Error(`Cannot check version: ${source} not present in the mapping meta`); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts index f23b1e84a87ea..400e01e999797 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -36,7 +36,11 @@ export const generateAdditiveMappingDiff = ({ deletedTypes, }: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { const typeVersions = getModelVersionMapForTypes(types); - const mappingVersion = getModelVersionsFromMappingMeta({ meta, source: 'mappingVersions' }); + const mappingVersion = getModelVersionsFromMappingMeta({ + meta, + source: 'mappingVersions', + knownTypes: types.map((type) => type.name), + }); if (!mappingVersion) { // should never occur given we checked previously in the flow but better safe than sorry. throw new Error( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index ebc22e623f600..76c66a9fc9bd0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -12,3 +12,10 @@ export { checkVersionCompatibility } from './check_version_compatibility'; export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; export { getAliasActions } from './get_alias_actions'; export { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; +export { getOutdatedDocumentsQuery } from './outdated_documents_query'; +export { createDocumentTransformFn } from './transform_raw_docs'; +export { + setMetaMappingMigrationComplete, + setMetaDocMigrationStarted, + setMetaDocMigrationComplete, +} from './update_index_meta'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts new file mode 100644 index 0000000000000..f39016b7d86a1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { getOutdatedDocumentsQuery } from './outdated_documents_query'; +import { createType } from '../test_helpers/saved_object_type'; + +const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +describe('getOutdatedDocumentsQuery', () => { + it('generates the correct query', () => { + const fooType = createType({ + name: 'foo', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + }, + }); + const barType = createType({ + name: 'bar', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + 3: dummyModelVersion, + }, + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.foo": "10.2.0", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + Object { + "term": Object { + "typeMigrationVersion": "10.2.0", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.bar": "10.3.0", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + Object { + "term": Object { + "typeMigrationVersion": "10.3.0", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + } + `); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts new file mode 100644 index 0000000000000..e15a2e447b554 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + getModelVersionMapForTypes, + modelVersionToVirtualVersion, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface GetOutdatedDocumentsQueryOps { + types: SavedObjectsType[]; +} + +export const getOutdatedDocumentsQuery = ({ + types, +}: GetOutdatedDocumentsQueryOps): QueryDslQueryContainer => { + // Note: in theory, we could check the difference of model version with the index's + // and narrow the search filter only on the type that have different versions. + // however, it feels safer to just search for all outdated document, just in case. + const modelVersions = getModelVersionMapForTypes(types); + return { + bool: { + should: types.map((type) => { + const virtualVersion = modelVersionToVirtualVersion(modelVersions[type.name]); + return { + bool: { + must: [ + { term: { type: type.name } }, + { + bool: { + should: [ + { + bool: { + must: { exists: { field: 'migrationVersion' } }, + must_not: { term: { [`migrationVersion.${type.name}`]: virtualVersion } }, + }, + }, + { + bool: { + must_not: [ + { exists: { field: 'migrationVersion' } }, + { term: { typeMigrationVersion: virtualVersion } }, + ], + }, + }, + ], + }, + }, + ], + }, + }; + }), + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts new file mode 100644 index 0000000000000..ab6e3f0c5773c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const migrateRawDocsSafelyMock = jest.fn(); + +jest.doMock('../../core/migrate_raw_docs', () => { + const actual = jest.requireActual('../../core/migrate_raw_docs'); + return { + ...actual, + migrateRawDocsSafely: migrateRawDocsSafelyMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts new file mode 100644 index 0000000000000..91a1f9983f330 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { migrateRawDocsSafelyMock } from './transform_raw_docs.test.mocks'; +import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; +import { createDocumentMigrator, createSavedObjectRawDoc } from '../test_helpers'; +import { createDocumentTransformFn } from './transform_raw_docs'; + +describe('createDocumentTransformFn', () => { + let serializer: ReturnType; + let documentMigrator: ReturnType; + + beforeEach(() => { + migrateRawDocsSafelyMock.mockReset(); + serializer = serializerMock.create(); + documentMigrator = createDocumentMigrator(); + }); + + it('returns a function calling migrateRawDocsSafely', () => { + const transformFn = createDocumentTransformFn({ + serializer, + documentMigrator, + }); + + expect(migrateRawDocsSafelyMock).not.toHaveBeenCalled(); + + const documents = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + transformFn(documents); + + expect(migrateRawDocsSafelyMock).toHaveBeenCalledTimes(1); + expect(migrateRawDocsSafelyMock).toHaveBeenCalledWith({ + rawDocs: documents, + serializer, + migrateDoc: documentMigrator.migrateAndConvert, + }); + }); + + it('forward the return from migrateRawDocsSafely', () => { + const transformFn = createDocumentTransformFn({ + serializer, + documentMigrator, + }); + + const documents = [createSavedObjectRawDoc({ _id: '1' })]; + + const expected = Symbol(); + migrateRawDocsSafelyMock.mockReturnValue(expected); + + const result = transformFn(documents); + + expect(result).toBe(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts new file mode 100644 index 0000000000000..1af1cee9d3b84 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ISavedObjectsSerializer, SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; +import { VersionedTransformer } from '../../document_migrator'; +import { TransformRawDocs } from '../../types'; +import { migrateRawDocsSafely } from '../../core/migrate_raw_docs'; + +export interface CreateDocumentTransformFnOpts { + serializer: ISavedObjectsSerializer; + documentMigrator: VersionedTransformer; +} + +export const createDocumentTransformFn = ({ + documentMigrator, + serializer, +}: CreateDocumentTransformFnOpts): TransformRawDocs => { + return (documents: SavedObjectsRawDoc[]) => + migrateRawDocsSafely({ + rawDocs: documents, + migrateDoc: documentMigrator.migrateAndConvert, + serializer, + }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts new file mode 100644 index 0000000000000..4298c6b027072 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndexMappingMeta, + ModelVersionMap, +} from '@kbn/core-saved-objects-base-server-internal'; +import { + setMetaDocMigrationStarted, + setMetaDocMigrationComplete, + setMetaMappingMigrationComplete, +} from './update_index_meta'; + +const getDefaultMeta = (): IndexMappingMeta => ({ + mappingVersions: { + foo: 1, + bar: 1, + }, + docVersions: { + foo: 1, + bar: 1, + }, + migrationState: { + convertingDocuments: false, + }, +}); + +describe('setMetaMappingMigrationComplete', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = getDefaultMeta(); + const versions: ModelVersionMap = { foo: 3, bar: 2 }; + + const updated = setMetaMappingMigrationComplete({ meta, versions }); + + expect(updated).toEqual({ + ...meta, + mappingVersions: versions, + }); + }); +}); + +describe('setMetaDocMigrationStarted', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = getDefaultMeta(); + + const updated = setMetaDocMigrationStarted({ meta }); + + expect(updated).toEqual({ + ...meta, + migrationState: { + convertingDocuments: true, + }, + }); + }); +}); + +describe('setMetaDocMigrationComplete', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = { + ...getDefaultMeta(), + migrationState: { + convertingDocuments: true, + }, + }; + const versions: ModelVersionMap = { foo: 3, bar: 2 }; + + const updated = setMetaDocMigrationComplete({ meta, versions }); + + expect(updated).toEqual({ + ...meta, + docVersions: versions, + migrationState: { + convertingDocuments: false, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts new file mode 100644 index 0000000000000..73d693d4a1f1a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndexMappingMeta, + ModelVersionMap, +} from '@kbn/core-saved-objects-base-server-internal'; + +export const setMetaMappingMigrationComplete = ({ + meta, + versions, +}: { + meta: IndexMappingMeta; + versions: ModelVersionMap; +}): IndexMappingMeta => { + return { + ...meta, + mappingVersions: { + ...versions, + }, + }; +}; + +export const setMetaDocMigrationStarted = ({ + meta, +}: { + meta: IndexMappingMeta; +}): IndexMappingMeta => { + return { + ...meta, + migrationState: { + convertingDocuments: true, + }, + }; +}; + +export const setMetaDocMigrationComplete = ({ + meta, + versions, +}: { + meta: IndexMappingMeta; + versions: ModelVersionMap; +}): IndexMappingMeta => { + return { + ...meta, + docVersions: { + ...versions, + }, + migrationState: { + convertingDocuments: false, + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts index 3094a1ccb3e09..44e8c348778ec 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts @@ -37,10 +37,7 @@ export interface SavedObjectsModelVersion { * * @public */ -export interface SavedObjectsModelExpansionChange< - PreviousAttributes = unknown, - NewAttributes = unknown -> { +export interface SavedObjectsModelExpansionChange { /** * The type of {@link SavedObjectsModelChange | change}, used to identify them internally. */ diff --git a/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts b/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts new file mode 100644 index 0000000000000..2065dcf27576d --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MatcherFunction } from 'expect'; +import { LogRecord } from '@kbn/logging'; + +const toContainLogEntry: MatcherFunction<[entry: string]> = (actual, entry) => { + if (!Array.isArray(actual)) { + throw new Error('actual must be an array'); + } + const logEntries = actual as LogRecord[]; + if (logEntries.find((item) => item.message.includes(entry))) { + return { + pass: true, + message: () => `Entry "${entry}" found in log file`, + }; + } else { + return { + pass: false, + message: () => `Entry "${entry}" not found in log file`, + }; + } +}; + +expect.extend({ + toContainLogEntry, +}); + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toContainLogEntry(entry: string): R; + } + } +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts index 0a84b9bc4b7e8..610981bab56ab 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts @@ -8,9 +8,12 @@ import { Env } from '@kbn/config'; import { getDocLinksMeta, getDocLinks } from '@kbn/doc-links'; +import { LogRecord } from '@kbn/logging'; import { REPO_ROOT } from '@kbn/repo-info'; import { getEnvOptions } from '@kbn/config-mocks'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; export const getDocVersion = () => { const env = Env.createDefault(REPO_ROOT, getEnvOptions()); @@ -33,3 +36,11 @@ export const createType = (parts: Partial): SavedObjectsType = mappings: { properties: {} }, ...parts, }); + +export const parseLogFile = async (filePath: string): Promise => { + const logFileContent = await fs.readFile(filePath, 'utf-8'); + return logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts new file mode 100644 index 0000000000000..3cf5e499eaa6e --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { createType } from '../test_utils'; +import { type KibanaMigratorTestKitParams } from '../kibana_migrator_test_kit'; + +export const getBaseMigratorParams = (): KibanaMigratorTestKitParams => ({ + kibanaIndex: '.kibana', + kibanaVersion: '8.8.0', + settings: { + migrations: { + algorithm: 'zdt', + zdt: { + metaPickupSyncDelaySec: 5, + }, + }, + }, +}); + +export const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +export const getFooType = () => { + return createType({ + name: 'foo', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + '2': dummyModelVersion, + }, + }); +}; + +export const getBarType = () => { + return createType({ + name: 'bar', + mappings: { + properties: { + aKeyword: { type: 'keyword' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getSampleAType = () => { + return createType({ + name: 'sample_a', + mappings: { + properties: { + keyword: { type: 'keyword' }, + boolean: { type: 'boolean' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getSampleBType = () => { + return createType({ + name: 'sample_b', + mappings: { + properties: { + text: { type: 'text' }, + text2: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getDeletedType = () => { + return createType({ + // we cant' easily introduce a deleted type, so we're using an existing one + name: 'server', + mappings: { + properties: { + text: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getExcludedType = () => { + return createType({ + // we cant' easily introduce a deleted type, so we're using an existing one + name: 'excluded', + mappings: { + properties: { + value: { type: 'integer' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + excludeOnUpgrade: () => { + return { + bool: { + must: [{ term: { type: 'excluded' } }, { range: { 'excluded.value': { lte: 1 } } }], + }, + }; + }, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts deleted file mode 100644 index 9bfac3ac5fd49..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; -import { createType } from '../test_utils'; - -export const dummyModelVersion: SavedObjectsModelVersion = { - modelChange: { - type: 'expansion', - }, -}; - -export const getFooType = () => { - return createType({ - name: 'foo', - mappings: { - properties: { - someField: { type: 'text' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - '2': dummyModelVersion, - }, - }); -}; - -export const getBarType = () => { - return createType({ - name: 'bar', - mappings: { - properties: { - aKeyword: { type: 'keyword' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - }, - }); -}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts new file mode 100644 index 0000000000000..418adb0a7894e --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); + +describe('ZDT upgrades - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { + text: `i am number ${number}`, + text2: `some static text`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + // typeA -> we add a new field and bump the model version by one with a migration + + typeA.mappings.properties = { + ...typeA.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + someAddedField: `${doc.attributes.keyword}-mig`, + }, + }, + }; + }, + down: jest.fn(), + }, + addedMappings: { + someAddedField: { type: 'keyword' }, + }, + }, + }, + }; + + // typeB -> we add two new model version with migrations + + typeB.modelVersions = { + ...typeB.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + text2: `${doc.attributes.text2} - mig2`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + '3': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + text2: `${doc.attributes.text2} - mig3`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + }; + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, typeB], + }); + + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + sample_a: typeA.mappings, + sample_b: typeB.mappings, + }) + ); + + expect(mappingMeta.docVersions).toEqual({ + sample_a: 2, + sample_b: 3, + }); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ type: 'sample_a' }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ type: 'sample_b' }); + + expect(sampleADocs).toHaveLength(5); + expect(sampleBDocs).toHaveLength(5); + + const sampleAData = sortBy(sampleADocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleAData).toEqual([ + { + id: 'a-0', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_0', someAddedField: 'a_0-mig' }, + }, + { + id: 'a-1', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_1', someAddedField: 'a_1-mig' }, + }, + { + id: 'a-2', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_2', someAddedField: 'a_2-mig' }, + }, + { + id: 'a-3', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_3', someAddedField: 'a_3-mig' }, + }, + { + id: 'a-4', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_4', someAddedField: 'a_4-mig' }, + }, + ]); + + const sampleBData = sortBy(sampleBDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleBData).toEqual([ + { + id: 'b-0', + type: 'sample_b', + attributes: { text: 'i am number 0', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-1', + type: 'sample_b', + attributes: { text: 'i am number 1', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-2', + type: 'sample_b', + attributes: { text: 'i am number 2', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-3', + type: 'sample_b', + attributes: { text: 'i am number 3', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-4', + type: 'sample_b', + attributes: { text: 'i am number 4', text2: 'some static text - mig2 - mig3' }, + }, + ]); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('Starting to process 10 documents'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts new file mode 100644 index 0000000000000..4f1b0a4bfe468 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'conversion_failures.test.log'); + +describe('ZDT upgrades - encountering conversion failures', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + beforeEach(async () => { + await fs.unlink(logFilePath).catch(() => {}); + }); + + describe('when discardCorruptObjects is true', () => { + it('completes the migration and discard the documents', async () => { + const { runMigrations, savedObjectsRepository } = await prepareScenario({ + discardCorruptObjects: true, + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('-> DONE'); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ + type: 'sample_a', + }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ + type: 'sample_b', + }); + + expect(sampleADocs).toHaveLength(0); + expect(sampleBDocs.map((doc) => doc.id).sort()).toEqual(['b-1', 'b-2', 'b-3', 'b-4']); + }); + }); + + describe('when discardCorruptObjects is false', () => { + it('fails the migration with an explicit message and keep the documents', async () => { + const { runMigrations, savedObjectsRepository } = await prepareScenario({ + discardCorruptObjects: false, + }); + + try { + await runMigrations(); + fail('migration should have failed'); + } catch (err) { + const errorMessage = err.message; + expect(errorMessage).toMatch('6 transformation errors were encountered'); + expect(errorMessage).toMatch('error from a-0'); + expect(errorMessage).toMatch('error from a-1'); + expect(errorMessage).toMatch('error from a-2'); + expect(errorMessage).toMatch('error from a-3'); + expect(errorMessage).toMatch('error from a-4'); + expect(errorMessage).toMatch('error from b-0'); + } + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL'); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ + type: 'sample_a', + }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ + type: 'sample_b', + }); + + expect(sampleADocs).toHaveLength(5); + expect(sampleBDocs).toHaveLength(5); + }); + }); + + const prepareScenario = async ({ discardCorruptObjects }: { discardCorruptObjects: boolean }) => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + // typeA -> migration failing all the documents + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + throw new Error(`error from ${doc.id}`); + }, + down: jest.fn(), + }, + }, + }, + }; + + // typeB -> migration failing the first doc + typeB.modelVersions = { + ...typeB.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + if (doc.id === 'b-0') { + throw new Error(`error from ${doc.id}`); + } + return { document: doc }; + }, + down: jest.fn(), + }, + }, + }, + }; + + const baseParams = getBaseMigratorParams(); + if (discardCorruptObjects) { + baseParams!.settings!.migrations!.discardCorruptObjects = '8.7.0'; + } + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...baseParams, + logFilePath, + types: [typeA, typeB], + }); + + return { runMigrations, client, savedObjectsRepository }; + }; + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + + try { + await client.indices.delete({ index: '.kibana_1' }); + } catch (e) { + /* index wasn't created, that's fine */ + } + + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { + text: `i am number ${number}`, + text2: `some static text`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts index e3f45bb561cf6..760b0f113a8ff 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -8,12 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; +import '../jest_matchers'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType } from './base_types.fixtures'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'create_index.test.log'); @@ -47,15 +46,9 @@ describe('ZDT upgrades - running on a fresh cluster', () => { const barType = getBarType(); const { runMigrations, client } = await getKibanaMigratorTestKit({ - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', + ...getBaseMigratorParams(), logFilePath, types: [fooType, barType], - settings: { - migrations: { - algorithm: 'zdt', - }, - }, }); const result = await runMigrations(); @@ -77,7 +70,7 @@ describe('ZDT upgrades - running on a fresh cluster', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappings.properties).toEqual( expect.objectContaining({ @@ -95,21 +88,17 @@ describe('ZDT upgrades - running on a fresh cluster', () => { foo: 2, bar: 1, }, + migrationState: expect.objectContaining({ + convertingDocuments: false, + }), }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; + const records = await parseLogFile(logFilePath); - expectLogsContains('INIT -> CREATE_TARGET_INDEX'); - expectLogsContains('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); - expectLogsContains('UPDATE_ALIASES -> DONE'); - expectLogsContains('Migration completed'); + expect(records).toContainLogEntry('INIT -> CREATE_TARGET_INDEX'); + expect(records).toContainLogEntry('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); + expect(records).toContainLogEntry('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DONE'); + expect(records).toContainLogEntry('Migration completed'); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts new file mode 100644 index 0000000000000..b321a400684b7 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { + getBaseMigratorParams, + getDeletedType, + getExcludedType, + getFooType, + getBarType, + dummyModelVersion, +} from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'document_cleanup.test.log'); + +describe('ZDT upgrades - document cleanup', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getFooType(), getBarType(), getDeletedType(), getExcludedType()], + }); + await runMigrations(); + + const fooObjs = range(5).map((number) => ({ + id: `foo-${number}`, + type: 'foo', + attributes: { + someField: `foo_${number}`, + }, + })); + + const barObjs = range(5).map((number) => ({ + id: `bar-${number}`, + type: 'bar', + attributes: { + aKeyword: `bar_${number}`, + }, + })); + + const deletedObjs = range(5).map((number) => ({ + id: `server-${number}`, + type: 'server', + attributes: { + text: `some text`, + }, + })); + + const excludedObjs = range(5).map((number) => ({ + id: `excluded-${number}`, + type: 'excluded', + attributes: { + value: number, + }, + })); + + await savedObjectsRepository.bulkCreate([ + ...fooObjs, + ...barObjs, + ...deletedObjs, + ...excludedObjs, + ]); + }; + + it('deletes the documents', async () => { + await createBaseline(); + + const fooType = getFooType(); + const excludedType = getExcludedType(); + + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, excludedType], + }); + + await runMigrations(); + + const indexContent = await client.search<{ type: string }>({ index: '.kibana_1', size: 100 }); + + // normal type + expect(countResultsByType(indexContent, 'foo')).toEqual(5); + // unknown type + expect(countResultsByType(indexContent, 'bar')).toEqual(0); + // deleted type + expect(countResultsByType(indexContent, 'server')).toEqual(0); + // excludeOnUpgrade type + expect(countResultsByType(indexContent, 'excluded')).toEqual(3); + }); +}); + +const countResultsByType = ( + indexContents: SearchResponse<{ type: string }>, + type: string +): number => { + return indexContents.hits.hits.filter((result) => result._source?.type === type).length; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts index eadd24bf447ac..1674bf747f7b1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts @@ -8,15 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - getKibanaMigratorTestKit, - type KibanaMigratorTestKitParams, -} from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log'); @@ -35,15 +31,7 @@ describe('ZDT upgrades - mapping model version conflict', () => { return await startES(); }; - const baseMigratorParams: KibanaMigratorTestKitParams = { - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', - settings: { - migrations: { - algorithm: 'zdt', - }, - }, - }; + const baseMigratorParams = getBaseMigratorParams(); beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); @@ -115,25 +103,16 @@ describe('ZDT upgrades - mapping model version conflict', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappingMeta.mappingVersions).toEqual({ foo: 2, bar: 2, }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; + const records = await parseLogFile(logFilePath); - // - expectLogsContains('Mappings model version check result: conflict'); - expectLogsContains('INIT -> FATAL'); + expect(records).toContainLogEntry('Mappings model version check result: conflict'); + expect(records).toContainLogEntry('INIT -> FATAL'); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts new file mode 100644 index 0000000000000..2be7c93965693 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'rerun_same_version.test.log'); + +describe('ZDT upgrades - rerun migration on same version', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('should perform a no-op upgrade', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, barType], + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT'); + expect(records).toContainLogEntry('-> DONE'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts new file mode 100644 index 0000000000000..2fbbe7e090267 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { + getBaseMigratorParams, + getSampleAType, + getSampleBType, + dummyModelVersion, +} from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'standard_workflow.test.log'); + +describe('ZDT upgrades - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { keyword: `a_${number}`, boolean: true }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { text: `i am number ${number}`, text2: `some static text` }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; + + it('follows the expected stages and transitions', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': dummyModelVersion, + }; + + typeB.modelVersions = { + ...typeB.modelVersions, + '2': dummyModelVersion, + }; + + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, typeB], + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT'); + expect(records).toContainLogEntry('DOCUMENTS_UPDATE_INIT -> SET_DOC_MIGRATION_STARTED'); + expect(records).toContainLogEntry( + 'SET_DOC_MIGRATION_STARTED -> SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES' + ); + expect(records).toContainLogEntry( + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' + ); + expect(records).toContainLogEntry( + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_TRANSFORM' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_READ' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_SEARCH_REFRESH' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH -> UPDATE_DOCUMENT_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry( + 'UPDATE_DOCUMENT_MODEL_VERSIONS -> UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES' + ); + expect(records).toContainLogEntry('UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES -> DONE'); + + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts index 5d27c32c1ee6a..9829da66a965e 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -8,15 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - getKibanaMigratorTestKit, - type KibanaMigratorTestKitParams, -} from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); @@ -35,16 +31,6 @@ describe('ZDT upgrades - basic mapping update', () => { return await startES(); }; - const baseMigratorParams: KibanaMigratorTestKitParams = { - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', - settings: { - migrations: { - algorithm: 'zdt', - }, - }, - }; - beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); esServer = await startElasticsearch(); @@ -59,7 +45,7 @@ describe('ZDT upgrades - basic mapping update', () => { const fooType = getFooType(); const barType = getBarType(); const { runMigrations } = await getKibanaMigratorTestKit({ - ...baseMigratorParams, + ...getBaseMigratorParams(), types: [fooType, barType], }); await runMigrations(); @@ -91,7 +77,7 @@ describe('ZDT upgrades - basic mapping update', () => { }; const { runMigrations, client } = await getKibanaMigratorTestKit({ - ...baseMigratorParams, + ...getBaseMigratorParams(), logFilePath, types: [fooType, barType], }); @@ -115,7 +101,7 @@ describe('ZDT upgrades - basic mapping update', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappings.properties).toEqual( expect.objectContaining({ @@ -124,32 +110,21 @@ describe('ZDT upgrades - basic mapping update', () => { }) ); - expect(mappingMeta).toEqual({ - // doc migration not implemented yet - docVersions are not bumped. - docVersions: { - foo: 2, - bar: 1, - }, - mappingVersions: { - foo: 3, - bar: 2, - }, + expect(mappingMeta.mappingVersions).toEqual({ + foo: 3, + bar: 2, }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; + const records = await parseLogFile(logFilePath); - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; - - expectLogsContains('INIT -> UPDATE_INDEX_MAPPINGS'); - expectLogsContains('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'); - expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS'); - expectLogsContains('UPDATE_MAPPING_MODEL_VERSIONS -> DONE'); - expectLogsContains('Migration completed'); + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('Migration completed'); }); });