From 050034db9ed12217fa50e415f64e854844494a3d Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Fri, 31 Mar 2023 15:16:50 +0200 Subject: [PATCH 01/17] Allow relocating SO to different indices during migration --- .../index.ts | 1 + .../src/mappings/index.ts | 1 + .../src/mappings/types.ts | 9 + .../src/migration/kibana_migrator.ts | 6 +- ...grations_state_action_machine.test.ts.snap | 186 + .../src/actions/index.ts | 3 + .../src/actions/synchronize_migrators.test.ts | 156 + .../src/actions/synchronize_migrators.ts | 27 + .../src/initial_state.test.ts | 235 +- .../src/initial_state.ts | 46 +- .../src/kibana_migrator.test.ts | 241 +- .../src/kibana_migrator.ts | 75 +- .../src/kibana_migrator_constants.ts | 134 + .../src/kibana_migrator_utils.fixtures.ts | 3413 +++++++++++++++++ .../src/kibana_migrator_utils.test.ts | 257 ++ .../src/kibana_migrator_utils.ts | 171 + .../migrations_state_action_machine.test.ts | 44 +- .../src/migrations_state_action_machine.ts | 35 +- .../src/model/create_batches.test.ts | 189 +- .../src/model/create_batches.ts | 42 +- .../src/model/helpers.test.ts | 48 + .../src/model/helpers.ts | 15 +- .../src/model/model.test.ts | 182 +- .../src/model/model.ts | 138 +- .../src/next.test.ts | 5 +- .../src/next.ts | 34 +- .../src/run_resilient_migrator.ts | 22 +- .../src/state.ts | 39 +- .../tsconfig.json | 1 - .../src/status.test.ts | 8 +- .../src/lib/indices/kibana_index.ts | 20 +- .../group2/batch_size_bytes.test.ts | 4 +- .../migrations/group3/actions/actions.test.ts | 18 +- 33 files changed, 5451 insertions(+), 354 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts create mode 100644 packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index bfe48e43491a0..981783bb05fd5 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -14,6 +14,7 @@ export { getTypes, type IndexMapping, type IndexMappingMeta, + type IndexTypesMap, type SavedObjectsTypeMappingDefinitions, type IndexMappingMigrationStateMeta, } from './src/mappings'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts index 7b2bb933fab3f..ee51fa9aaf4bb 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts @@ -11,5 +11,6 @@ export type { SavedObjectsTypeMappingDefinitions, IndexMappingMeta, IndexMapping, + IndexTypesMap, IndexMappingMigrationStateMeta, } from './types'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index 10faa1b03d31d..c756f0534db67 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -55,6 +55,9 @@ export interface IndexMapping { _meta?: IndexMappingMeta; } +/** @internal */ +export type IndexTypesMap = Record; + /** @internal */ export interface IndexMappingMeta { /** @@ -65,6 +68,12 @@ export interface IndexMappingMeta { * @remark: Only defined for indices using the v2 migration algorithm. */ migrationMappingPropertyHashes?: { [k: string]: string }; + /** + * A map that tells what are the SO types stored in each index + * + * @remark: Only defined for indices using the v2 migration algorithm. + */ + indexTypesMap?: IndexTypesMap; /** * The current model versions of the mapping of the index. * diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/migration/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/migration/kibana_migrator.ts index bb078135c8bcc..de569332ff9ce 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/migration/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/migration/kibana_migrator.ts @@ -69,7 +69,11 @@ export type MigrationStatus = /** @internal */ export type MigrationResult = | { status: 'skipped' } - | { status: 'patched' } + | { + status: 'patched'; + destIndex: string; + elapsedMs: number; + } | { status: 'migrated'; destIndex: string; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap index cd2b218d58dd4..387dbd87bbafe 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/__snapshots__/migrations_state_action_machine.test.ts.snap @@ -138,6 +138,20 @@ Object { }, }, "indexPrefix": ".my-so-index", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "7.11.0", "knownTypes": Array [], "legacyIndex": ".my-so-index", @@ -154,6 +168,7 @@ Object { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { "bool": Object { @@ -167,6 +182,22 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -325,6 +356,20 @@ Object { }, }, "indexPrefix": ".my-so-index", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "7.11.0", "knownTypes": Array [], "legacyIndex": ".my-so-index", @@ -345,6 +390,7 @@ Object { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { "bool": Object { @@ -358,6 +404,22 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -516,6 +578,20 @@ Object { }, }, "indexPrefix": ".my-so-index", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "7.11.0", "knownTypes": Array [], "legacyIndex": ".my-so-index", @@ -540,6 +616,7 @@ Object { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { "bool": Object { @@ -553,6 +630,22 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -711,6 +804,20 @@ Object { }, }, "indexPrefix": ".my-so-index", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "7.11.0", "knownTypes": Array [], "legacyIndex": ".my-so-index", @@ -739,6 +846,7 @@ Object { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { "bool": Object { @@ -752,6 +860,22 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -954,6 +1078,20 @@ Object { }, }, "indexPrefix": ".my-so-index", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "7.11.0", "knownTypes": Array [], "legacyIndex": ".my-so-index", @@ -970,6 +1108,7 @@ Object { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocuments": Array [ Object { "_id": "1234", @@ -988,6 +1127,22 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", @@ -1152,6 +1307,20 @@ Object { }, }, "indexPrefix": ".my-so-index", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "7.11.0", "knownTypes": Array [], "legacyIndex": ".my-so-index", @@ -1172,6 +1341,7 @@ Object { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocuments": Array [ Object { "_id": "1234", @@ -1190,6 +1360,22 @@ Object { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "properties": Object {}, }, "tempIndex": ".my-so-index_7.11.0_reindex_temp", diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts index 652178fd25919..9080e2ce93dbe 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/index.ts @@ -83,6 +83,9 @@ export { cleanupUnknownAndExcluded } from './cleanup_unknown_and_excluded'; export { waitForDeleteByQueryTask } from './wait_for_delete_by_query_task'; export type { CreateIndexParams, ClusterShardLimitExceeded } from './create_index'; + +export { synchronizeMigrators } from './synchronize_migrators'; + export { createIndex } from './create_index'; export { checkTargetMappings } from './check_target_mappings'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.test.ts new file mode 100644 index 0000000000000..a5a8e9c25f929 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { synchronizeMigrators } from './synchronize_migrators'; +import { type Defer, defer } from '../kibana_migrator_utils'; + +describe('synchronizeMigrators', () => { + let defers: Array>; + let allDefersPromise: Promise; + let migratorsDefers: Array>; + + beforeEach(() => { + jest.clearAllMocks(); + + defers = ['.kibana_cases', '.kibana_task_manager', '.kibana'].map(defer); + allDefersPromise = Promise.all(defers.map(({ promise }) => promise)); + + migratorsDefers = defers.map(({ resolve, reject }) => ({ + resolve: jest.fn(resolve), + reject: jest.fn(reject), + promise: allDefersPromise, + })); + }); + + describe('when all migrators reach the synchronization point with a correct state', () => { + it('unblocks all migrators and resolves Right', async () => { + const tasks = migratorsDefers.map((migratorDefer) => synchronizeMigrators(migratorDefer)); + + const res = await Promise.all(tasks.map((task) => task())); + + migratorsDefers.forEach((migratorDefer) => + expect(migratorDefer.resolve).toHaveBeenCalledTimes(1) + ); + migratorsDefers.forEach((migratorDefer) => + expect(migratorDefer.reject).not.toHaveBeenCalled() + ); + + expect(res).toEqual([ + { _tag: 'Right', right: 'synchronized_successfully' }, + { _tag: 'Right', right: 'synchronized_successfully' }, + { _tag: 'Right', right: 'synchronized_successfully' }, + ]); + }); + + it('migrators are not unblocked until the last one reaches the synchronization point', async () => { + let resolved: number = 0; + migratorsDefers.forEach((migratorDefer) => migratorDefer.promise.then(() => ++resolved)); + const [casesDefer, ...otherMigratorsDefers] = migratorsDefers; + + // we simulate that only kibana_task_manager and kibana migrators get to the sync point + const tasks = otherMigratorsDefers.map((migratorDefer) => + synchronizeMigrators(migratorDefer) + ); + // we don't await for them, or we would be locked forever + Promise.all(tasks.map((task) => task())); + + const [taskManagerDefer, kibanaDefer] = otherMigratorsDefers; + expect(taskManagerDefer.resolve).toHaveBeenCalledTimes(1); + expect(kibanaDefer.resolve).toHaveBeenCalledTimes(1); + expect(casesDefer.resolve).not.toHaveBeenCalled(); + expect(resolved).toEqual(0); + + // finally, the last migrator gets to the synchronization point + await synchronizeMigrators(casesDefer)(); + expect(resolved).toEqual(3); + }); + }); + + describe('when one migrator fails and rejects the synchronization defer', () => { + describe('before the rest of the migrators reach the synchronization point', () => { + it('synchronizedMigrators resolves Left for the rest of migrators', async () => { + let resolved: number = 0; + let errors: number = 0; + migratorsDefers.forEach((migratorDefer) => + migratorDefer.promise.then(() => ++resolved).catch(() => ++errors) + ); + const [casesDefer, ...otherMigratorsDefers] = migratorsDefers; + + // we first make one random migrator fail and not reach the sync point + casesDefer.reject('Oops. The cases migrator failed unexpectedly.'); + + // the other migrators then try to synchronize + const tasks = otherMigratorsDefers.map((migratorDefer) => + synchronizeMigrators(migratorDefer) + ); + + expect(Promise.all(tasks.map((task) => task()))).resolves.toEqual([ + { + _tag: 'Left', + left: { + type: 'sync_failed', + error: 'Oops. The cases migrator failed unexpectedly.', + }, + }, + { + _tag: 'Left', + left: { + type: 'sync_failed', + error: 'Oops. The cases migrator failed unexpectedly.', + }, + }, + ]); + + // force next tick (as we did not await for Promises) + await new Promise((resolve) => setImmediate(resolve)); + expect(resolved).toEqual(0); + expect(errors).toEqual(3); + }); + }); + + describe('after the rest of the migrators reach the synchronization point', () => { + it('synchronizedMigrators resolves Left for the rest of migrators', async () => { + let resolved: number = 0; + let errors: number = 0; + migratorsDefers.forEach((migratorDefer) => + migratorDefer.promise.then(() => ++resolved).catch(() => ++errors) + ); + const [casesDefer, ...otherMigratorsDefers] = migratorsDefers; + + // some migrators try to synchronize + const tasks = otherMigratorsDefers.map((migratorDefer) => + synchronizeMigrators(migratorDefer) + ); + + // we then make one random migrator fail and not reach the sync point + casesDefer.reject('Oops. The cases migrator failed unexpectedly.'); + + expect(Promise.all(tasks.map((task) => task()))).resolves.toEqual([ + { + _tag: 'Left', + left: { + type: 'sync_failed', + error: 'Oops. The cases migrator failed unexpectedly.', + }, + }, + { + _tag: 'Left', + left: { + type: 'sync_failed', + error: 'Oops. The cases migrator failed unexpectedly.', + }, + }, + ]); + + // force next tick (as we did not await for Promises) + await new Promise((resolve) => setImmediate(resolve)); + expect(resolved).toEqual(0); + expect(errors).toEqual(3); + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.ts new file mode 100644 index 0000000000000..26763f1f51ae2 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/actions/synchronize_migrators.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import * as TaskEither from 'fp-ts/lib/TaskEither'; +import type { Defer } from '../kibana_migrator_utils'; + +export interface SyncFailed { + type: 'sync_failed'; + error: Error; +} + +export function synchronizeMigrators( + defer: Defer +): TaskEither.TaskEither { + return () => { + defer.resolve(); + return defer.promise + .then(() => Either.right('synchronized_successfully' as const)) + .catch((error) => Either.left({ type: 'sync_failed' as const, error })); + }; +} diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts index 2dcadc9ab0210..d49e784f77633 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts @@ -13,44 +13,63 @@ import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import { type SavedObjectsMigrationConfigType, SavedObjectTypeRegistry, + type IndexMapping, } from '@kbn/core-saved-objects-base-server-internal'; +import type { Logger } from '@kbn/logging'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; -import { createInitialState } from './initial_state'; +import { createInitialState, type CreateInitialStateParams } from './initial_state'; const mockLogger = loggingSystemMock.create(); +const migrationsConfig = { + retryAttempts: 15, + batchSize: 1000, + maxBatchSizeBytes: ByteSizeValue.parse('100mb'), +} as unknown as SavedObjectsMigrationConfigType; + +const createInitialStateCommonParams = { + kibanaVersion: '8.1.0', + waitForMigrationCompletion: false, + mustRelocateDocuments: true, + indexTypesMap: { + '.kibana': ['typeA', 'typeB', 'typeC'], + '.kibana_task_manager': ['task'], + '.kibana_cases': ['typeD', 'typeE'], + }, + targetMappings: { + dynamic: 'strict', + properties: { my_type: { properties: { title: { type: 'text' } } } }, + } as IndexMapping, + migrationVersionPerType: {}, + indexPrefix: '.kibana_task_manager', + migrationsConfig, +}; + describe('createInitialState', () => { let typeRegistry: SavedObjectTypeRegistry; let docLinks: DocLinksServiceSetup; + let logger: Logger; + let createInitialStateParams: CreateInitialStateParams; beforeEach(() => { typeRegistry = new SavedObjectTypeRegistry(); docLinks = docLinksServiceMock.createSetupContract(); + logger = mockLogger.get(); + createInitialStateParams = { + ...createInitialStateCommonParams, + typeRegistry, + docLinks, + logger, + }; }); afterEach(() => jest.clearAllMocks()); - const migrationsConfig = { - retryAttempts: 15, - batchSize: 1000, - maxBatchSizeBytes: ByteSizeValue.parse('100mb'), - } as unknown as SavedObjectsMigrationConfigType; - it('creates the initial state for the model based on the passed in parameters', () => { expect( createInitialState({ - kibanaVersion: '8.1.0', + ...createInitialStateParams, waitForMigrationCompletion: true, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger: mockLogger.get(), }) ).toMatchInlineSnapshot(` Object { @@ -172,6 +191,20 @@ describe('createInitialState', () => { }, }, "indexPrefix": ".kibana_task_manager", + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, "kibanaVersion": "8.1.0", "knownTypes": Array [], "legacyIndex": ".kibana_task_manager", @@ -183,6 +216,7 @@ describe('createInitialState', () => { "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, + "mustRelocateDocuments": true, "outdatedDocumentsQuery": Object { "bool": Object { "should": Array [], @@ -195,6 +229,22 @@ describe('createInitialState', () => { "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { + "_meta": Object { + "indexTypesMap": Object { + ".kibana": Array [ + "typeA", + "typeB", + "typeC", + ], + ".kibana_cases": Array [ + "typeD", + "typeE", + ], + ".kibana_task_manager": Array [ + "task", + ], + }, + }, "dynamic": "strict", "properties": Object { "my_type": Object { @@ -227,22 +277,7 @@ describe('createInitialState', () => { }); it('creates the initial state for the model with waitForMigrationCompletion false,', () => { - expect( - createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger: mockLogger.get(), - }) - ).toMatchObject({ + expect(createInitialState(createInitialStateParams)).toMatchObject({ waitForMigrationCompletion: false, }); }); @@ -262,18 +297,10 @@ describe('createInitialState', () => { }); const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, + ...createInitialStateParams, typeRegistry, docLinks, - logger: mockLogger.get(), + logger, }); expect(initialState.knownTypes).toEqual(['foo', 'bar']); @@ -289,40 +316,15 @@ describe('createInitialState', () => { excludeOnUpgrade: fooExcludeOnUpgradeHook, }); - const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger: mockLogger.get(), - }); - + const initialState = createInitialState(createInitialStateParams); expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ foo: fooExcludeOnUpgradeHook }); }); it('returns state with a preMigration script', () => { const preMigrationScript = "ctx._id = ctx._source.type + ':' + ctx._id"; const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, + ...createInitialStateParams, preMigrationScript, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger: mockLogger.get(), }); expect(Option.isSome(initialState.preMigrationScript)).toEqual(true); @@ -334,19 +336,8 @@ describe('createInitialState', () => { expect( Option.isNone( createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, + ...createInitialStateParams, preMigrationScript: undefined, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger: mockLogger.get(), }).preMigrationScript ) ).toEqual(true); @@ -354,19 +345,9 @@ describe('createInitialState', () => { it('returns state with an outdatedDocumentsQuery', () => { expect( createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, + ...createInitialStateParams, preMigrationScript: "ctx._id = ctx._source.type + ':' + ctx._id", migrationVersionPerType: { my_dashboard: '7.10.1', my_viz: '8.0.0' }, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger: mockLogger.get(), }).outdatedDocumentsQuery ).toMatchInlineSnapshot(` Object { @@ -473,44 +454,19 @@ describe('createInitialState', () => { }); it('initializes the `discardUnknownObjects` flag to false if the flag is not provided in the config', () => { - const logger = mockLogger.get(); - const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', - migrationsConfig, - typeRegistry, - docLinks, - logger, - }); + const initialState = createInitialState(createInitialStateParams); expect(logger.warn).not.toBeCalled(); expect(initialState.discardUnknownObjects).toEqual(false); }); it('initializes the `discardUnknownObjects` flag to false if the value provided in the config does not match the current kibana version', () => { - const logger = mockLogger.get(); const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', + ...createInitialStateParams, migrationsConfig: { ...migrationsConfig, discardUnknownObjects: '8.0.0', }, - typeRegistry, - docLinks, - logger, }); expect(initialState.discardUnknownObjects).toEqual(false); @@ -522,44 +478,23 @@ describe('createInitialState', () => { it('initializes the `discardUnknownObjects` flag to true if the value provided in the config matches the current kibana version', () => { const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', + ...createInitialStateParams, migrationsConfig: { ...migrationsConfig, discardUnknownObjects: '8.1.0', }, - typeRegistry, - docLinks, - logger: mockLogger.get(), }); expect(initialState.discardUnknownObjects).toEqual(true); }); it('initializes the `discardCorruptObjects` flag to false if the value provided in the config does not match the current kibana version', () => { - const logger = mockLogger.get(); const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', + ...createInitialStateParams, migrationsConfig: { ...migrationsConfig, discardCorruptObjects: '8.0.0', }, - typeRegistry, - docLinks, - logger, }); expect(initialState.discardCorruptObjects).toEqual(false); @@ -571,21 +506,11 @@ describe('createInitialState', () => { it('initializes the `discardCorruptObjects` flag to true if the value provided in the config matches the current kibana version', () => { const initialState = createInitialState({ - kibanaVersion: '8.1.0', - waitForMigrationCompletion: false, - targetMappings: { - dynamic: 'strict', - properties: { my_type: { properties: { title: { type: 'text' } } } }, - }, - migrationVersionPerType: {}, - indexPrefix: '.kibana_task_manager', + ...createInitialStateParams, migrationsConfig: { ...migrationsConfig, discardCorruptObjects: '8.1.0', }, - typeRegistry, - docLinks, - logger: mockLogger.get(), }); expect(initialState.discardCorruptObjects).toEqual(true); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts index ff15afa7c691a..6daa8887d5b72 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.ts @@ -14,10 +14,27 @@ import type { SavedObjectsMigrationVersion } from '@kbn/core-saved-objects-commo import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server'; import type { IndexMapping, + IndexTypesMap, SavedObjectsMigrationConfigType, } from '@kbn/core-saved-objects-base-server-internal'; import type { InitState } from './state'; import { excludeUnusedTypesQuery } from './core'; +import { getTempIndexName } from './model/helpers'; + +export interface CreateInitialStateParams { + kibanaVersion: string; + waitForMigrationCompletion: boolean; + mustRelocateDocuments: boolean; + indexTypesMap: IndexTypesMap; + targetMappings: IndexMapping; + preMigrationScript?: string; + migrationVersionPerType: SavedObjectsMigrationVersion; + indexPrefix: string; + migrationsConfig: SavedObjectsMigrationConfigType; + typeRegistry: ISavedObjectTypeRegistry; + docLinks: DocLinksServiceStart; + logger: Logger; +} /** * Construct the initial state for the model @@ -25,6 +42,8 @@ import { excludeUnusedTypesQuery } from './core'; export const createInitialState = ({ kibanaVersion, waitForMigrationCompletion, + mustRelocateDocuments, + indexTypesMap, targetMappings, preMigrationScript, migrationVersionPerType, @@ -33,18 +52,7 @@ export const createInitialState = ({ typeRegistry, docLinks, logger, -}: { - kibanaVersion: string; - waitForMigrationCompletion: boolean; - targetMappings: IndexMapping; - preMigrationScript?: string; - migrationVersionPerType: SavedObjectsMigrationVersion; - indexPrefix: string; - migrationsConfig: SavedObjectsMigrationConfigType; - typeRegistry: ISavedObjectTypeRegistry; - docLinks: DocLinksServiceStart; - logger: Logger; -}): InitState => { +}: CreateInitialStateParams): InitState => { const outdatedDocumentsQuery: QueryDslQueryContainer = { bool: { should: Object.entries(migrationVersionPerType).map(([type, latestVersion]) => ({ @@ -117,18 +125,28 @@ export const createInitialState = ({ ); } + const targetIndexMappings: IndexMapping = { + ...targetMappings, + _meta: { + ...targetMappings._meta, + indexTypesMap, + }, + }; + return { controlState: 'INIT', waitForMigrationCompletion, + mustRelocateDocuments, + indexTypesMap, indexPrefix, legacyIndex: indexPrefix, currentAlias: indexPrefix, versionAlias: `${indexPrefix}_${kibanaVersion}`, versionIndex: `${indexPrefix}_${kibanaVersion}_001`, - tempIndex: `${indexPrefix}_${kibanaVersion}_reindex_temp`, + tempIndex: getTempIndexName(indexPrefix, kibanaVersion), kibanaVersion, preMigrationScript: Option.fromNullable(preMigrationScript), - targetIndexMappings: targetMappings, + targetIndexMappings, tempIndexMappings: reindexTargetMappings, outdatedDocumentsQuery, retryCount: 0, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index e6855a1256b54..f394d4866b54e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -18,6 +18,19 @@ import { DocumentMigrator } from './document_migrator'; import { ByteSizeValue } from '@kbn/config-schema'; import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import { lastValueFrom } from 'rxjs'; +import { runResilientMigrator } from './run_resilient_migrator'; + +jest.mock('./run_resilient_migrator', () => ({ + runResilientMigrator: jest.fn((options) => { + return { + status: 'migrated', + // TODO check the actual value of the MigrationResult for these fields + destIndex: `${options.indexPrefix}_8.2.3_001`, + sourceIndex: `${options.indexPrefix}_8.2.2_001`, + elapsedMs: 28, + }; + }), +})); jest.mock('./document_migrator', () => { return { @@ -29,6 +42,21 @@ jest.mock('./document_migrator', () => { }; }); +const mappingsResponseWithoutIndexTypesMap: estypes.IndicesGetMappingResponse = { + '.kibana_8.7.0_001': { + mappings: { + _meta: { + migrationMappingPropertyHashes: { + references: '7997cf5a56cc02bdc9c93361bde732b0', + // ... + }, + // we do not add a `indexTypesMap` + // simulating a Kibana < 8.8.0 that does not have one yet + }, + }, + }, +}; + const createRegistry = (types: Array>) => { const registry = new SavedObjectTypeRegistry(); types.forEach((type) => @@ -47,6 +75,7 @@ const createRegistry = (types: Array>) => { describe('KibanaMigrator', () => { beforeEach(() => { (DocumentMigrator as jest.Mock).mockClear(); + (runResilientMigrator as jest.MockedFunction).mockClear(); }); describe('getActiveMappings', () => { it('returns full index mappings w/ core properties', () => { @@ -96,21 +125,21 @@ describe('KibanaMigrator', () => { }); describe('runMigrations', () => { - it('throws if prepareMigrations is not called first', async () => { + it('throws if prepareMigrations is not called first', () => { const options = mockOptions(); - - options.client.indices.get.mockResponse({}, { statusCode: 200 }); - const migrator = new KibanaMigrator(options); - await expect(() => migrator.runMigrations()).toThrowErrorMatchingInlineSnapshot( - `"Migrations are not ready. Make sure prepareMigrations is called first."` + expect(migrator.runMigrations()).rejects.toThrowError( + 'Migrations are not ready. Make sure prepareMigrations is called first.' ); }); it('only runs migrations once if called multiple times', async () => { const options = mockOptions(); options.client.indices.get.mockResponse({}, { statusCode: 200 }); + options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { + statusCode: 200, + }); options.client.cluster.getSettings.mockResponse( { @@ -132,6 +161,9 @@ describe('KibanaMigrator', () => { it('emits results on getMigratorResult$()', async () => { const options = mockV2MigrationOptions(); + options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { + statusCode: 200, + }); const migrator = new KibanaMigrator(options); const migratorStatus = lastValueFrom(migrator.getStatus$().pipe(take(3))); migrator.prepareMigrations(); @@ -166,6 +198,9 @@ describe('KibanaMigrator', () => { }, { statusCode: 200 } ); + options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { + statusCode: 200, + }); const migrator = new KibanaMigrator(options); migrator.prepareMigrations(); @@ -181,6 +216,9 @@ describe('KibanaMigrator', () => { error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, task: { description: 'task description' } as any, }); + options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { + statusCode: 200, + }); const migrator = new KibanaMigrator(options); migrator.prepareMigrations(); @@ -193,6 +231,158 @@ describe('KibanaMigrator', () => { {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); }); + + describe('for V2 migrations', () => { + describe('where some SO types must be relocated', () => { + // eslint-disable-next-line jest/no-focused-tests + fit('runs successfully', async () => { + const options = mockV2MigrationOptions(); + options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { + statusCode: 200, + }); + + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + const results = await migrator.runMigrations(); + expect(results).toMatchInlineSnapshot(` + Array [ + Object { + "destIndex": ".my-index_8.2.3_001", + "elapsedMs": 28, + "sourceIndex": ".my-index_8.2.2_001", + "status": "migrated", + }, + Object { + "destIndex": ".other-index_8.2.3_001", + "elapsedMs": 28, + "sourceIndex": ".other-index_8.2.2_001", + "status": "migrated", + }, + Object { + "destIndex": ".my-task-index_8.2.3_001", + "elapsedMs": 28, + "sourceIndex": ".my-task-index_8.2.2_001", + "status": "migrated", + }, + Object { + "destIndex": ".my-complementary-index_8.2.3_001", + "elapsedMs": 28, + "sourceIndex": ".my-complementary-index_8.2.2_001", + "status": "migrated", + }, + ] + `); + + expect(runResilientMigrator).toHaveBeenCalledTimes(4); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + kibanaVersion: '8.2.3', + indexPrefix: '.my-index', + indexTypesMap: { + '.my-index': ['testtype', 'testtype3'], + '.other-index': ['testtype2'], + '.my-task-index': ['testtasktype'], + }, + targetMappings: expect.objectContaining({ + properties: expect.objectContaining({ + testtype: expect.anything(), + testtype3: expect.anything(), + }), + }), + readyToReindex: expect.objectContaining({ + promise: expect.anything(), + resolve: expect.anything(), + reject: expect.anything(), + }), + mustRelocateDocuments: true, + doneReindexing: expect.objectContaining({ + promise: expect.anything(), + resolve: expect.anything(), + reject: expect.anything(), + }), + }) + ); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + kibanaVersion: '8.2.3', + indexPrefix: '.other-index', + indexTypesMap: { + '.my-index': ['testtype', 'testtype3'], + '.other-index': ['testtype2'], + '.my-task-index': ['testtasktype'], + }, + targetMappings: expect.objectContaining({ + properties: expect.objectContaining({ + testtype2: expect.anything(), + }), + }), + readyToReindex: expect.objectContaining({ + promise: expect.anything(), + resolve: expect.anything(), + reject: expect.anything(), + }), + mustRelocateDocuments: true, + doneReindexing: expect.objectContaining({ + promise: expect.anything(), + resolve: expect.anything(), + reject: expect.anything(), + }), + }) + ); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + kibanaVersion: '8.2.3', + indexPrefix: '.my-task-index', + indexTypesMap: { + '.my-index': ['testtype', 'testtype3'], + '.other-index': ['testtype2'], + '.my-task-index': ['testtasktype'], + }, + targetMappings: expect.objectContaining({ + properties: expect.objectContaining({ + testtasktype: expect.anything(), + }), + }), + // this migrator is NOT involved in any relocation, + // thus, it must not synchronize with other migrators + mustRelocateDocuments: false, + readyToReindex: undefined, + doneReindexing: undefined, + }) + ); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 4, + expect.objectContaining({ + kibanaVersion: '8.2.3', + indexPrefix: '.my-complementary-index', + indexTypesMap: { + '.my-index': ['testtype', 'testtype3'], + '.other-index': ['testtype2'], + '.my-task-index': ['testtasktype'], + }, + targetMappings: expect.objectContaining({ + properties: expect.not.objectContaining({ + // this index does no longer have any types associated to it + testtype: expect.anything(), + testtype2: expect.anything(), + testtype3: expect.anything(), + testtasktype: expect.anything(), + }), + }), + mustRelocateDocuments: true, + doneReindexing: expect.objectContaining({ + promise: expect.anything(), + resolve: expect.anything(), + reject: expect.anything(), + }), + }) + ); + }); + }); + }); }); }); @@ -254,7 +444,19 @@ const mockOptions = () => { logger: loggingSystemMock.create().get(), kibanaVersion: '8.2.3', waitForMigrationCompletion: false, + defaultIndexTypesMap: { + '.my-index': ['testtype', 'testtype2'], + '.my-task-index': ['testtasktype'], + // this index no longer has any types registered in typeRegistry + // but we still need a migrator for it, so that 'testtype3' documents + // are moved over to their new index (.my_index) + '.my-complementary-index': ['testtype3'], + }, typeRegistry: createRegistry([ + // typeRegistry depicts an updated index map: + // .my-index: ['testtype', 'testtype3'], + // .my-other-index: ['testtype2'], + // .my-task-index': ['testtasktype'], { name: 'testtype', hidden: false, @@ -270,7 +472,32 @@ const mockOptions = () => { name: 'testtype2', hidden: false, namespaceType: 'single', - indexPattern: 'other-index', + // We are moving 'testtype2' from '.my-index' to '.my-other-index' + indexPattern: '.other-index', + mappings: { + properties: { + name: { type: 'keyword' }, + }, + }, + migrations: {}, + }, + { + name: 'testtasktype', + hidden: false, + namespaceType: 'single', + indexPattern: '.my-task-index', + mappings: { + properties: { + name: { type: 'keyword' }, + }, + }, + migrations: {}, + }, + { + // We are moving 'testtype3' from '.my-complementary-index' to '.my-index' + name: 'testtype3', + hidden: false, + namespaceType: 'single', mappings: { properties: { name: { type: 'keyword' }, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index d63cc69d8f7f4..7f7a1ce94fcb8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -29,17 +29,22 @@ import { type IKibanaMigrator, type KibanaMigratorStatus, type MigrationResult, + type IndexTypesMap, } from '@kbn/core-saved-objects-base-server-internal'; +import { getIndicesInvoledInRelocation } from './kibana_migrator_utils'; import { buildActiveMappings, buildTypesMappings } from './core'; import { DocumentMigrator, type VersionedTransformer } from './document_migrator'; import { createIndexMap } from './core/build_index_map'; import { runResilientMigrator } from './run_resilient_migrator'; import { migrateRawDocsSafely } from './core/migrate_raw_docs'; import { runZeroDowntimeMigration } from './zdt'; - -// ensure plugins don't try to convert SO namespaceTypes after 8.0.0 -// see https://github.com/elastic/kibana/issues/147344 -const ALLOWED_CONVERT_VERSION = '8.0.0'; +import { createMultiPromiseDefer, indexMapToIndexTypesMap } from './kibana_migrator_utils'; +import { + ALLOWED_CONVERT_VERSION, + DEFAULT_INDEX_TYPES_MAP, + LEGACY_SAVED_OBJECT_INDEX, + MAIN_SAVED_OBJECT_INDEX, +} from './kibana_migrator_constants'; export interface KibanaMigratorOptions { client: ElasticsearchClient; @@ -50,6 +55,7 @@ export interface KibanaMigratorOptions { logger: Logger; docLinks: DocLinksServiceStart; waitForMigrationCompletion: boolean; + defaultIndexTypesMap?: IndexTypesMap; } /** @@ -71,6 +77,7 @@ export class KibanaMigrator implements IKibanaMigrator { private readonly soMigrationsConfig: SavedObjectsMigrationConfigType; private readonly docLinks: DocLinksServiceStart; private readonly waitForMigrationCompletion: boolean; + private readonly defaultIndexTypesMap: IndexTypesMap; public readonly kibanaVersion: string; /** @@ -84,6 +91,7 @@ export class KibanaMigrator implements IKibanaMigrator { kibanaVersion, logger, docLinks, + defaultIndexTypesMap = DEFAULT_INDEX_TYPES_MAP, waitForMigrationCompletion, }: KibanaMigratorOptions) { this.client = client; @@ -105,6 +113,7 @@ export class KibanaMigrator implements IKibanaMigrator { // operation so we cache the result this.activeMappings = buildActiveMappings(this.mappingProperties); this.docLinks = docLinks; + this.defaultIndexTypesMap = defaultIndexTypesMap; } public runMigrations({ rerun = false }: { rerun?: boolean } = {}): Promise { @@ -134,12 +143,12 @@ export class KibanaMigrator implements IKibanaMigrator { return this.status$.asObservable(); } - private runMigrationsInternal(): Promise { + private async runMigrationsInternal(): Promise { const migrationAlgorithm = this.soMigrationsConfig.algorithm; if (migrationAlgorithm === 'zdt') { return this.runMigrationZdt(); } else { - return this.runMigrationV2(); + return await this.runMigrationV2(); } } @@ -157,7 +166,7 @@ export class KibanaMigrator implements IKibanaMigrator { }); } - private runMigrationV2(): Promise { + private async runMigrationV2(): Promise { const indexMap = createIndexMap({ kibanaIndexName: this.kibanaIndex, indexMap: this.mappingProperties, @@ -173,16 +182,60 @@ export class KibanaMigrator implements IKibanaMigrator { this.log.debug(`migrationVersion: ${migrationVersion} saved object type: ${type}`); }); - const migrators = Object.keys(indexMap).map((index) => { + // build a indexTypesMap from the info present in tye typeRegistry, e.g.: + // { + // '.kibana': ['typeA', 'typeB', ...] + // '.kibana_task_manager': ['task', ...] + // '.kibana_cases': ['typeC', 'typeD', ...] + // ... + // } + const indexTypesMap = indexMapToIndexTypesMap(indexMap); + + // compare indexTypesMap with the one present (or not) in the .kibana index meta + // and check if some SO types have been moved to different indices + const indicesWithMovingTypes = await getIndicesInvoledInRelocation({ + mainIndex: MAIN_SAVED_OBJECT_INDEX, + legacyIndex: LEGACY_SAVED_OBJECT_INDEX, + client: this.client, + indexTypesMap, + logger: this.log, + defaultIndexTypesMap: this.defaultIndexTypesMap, + }); + + // we create 2 synchronization objects (2 synchronization points) for each of the + // migrators involved in relocations, aka each of the migrators that will: + // A) reindex some documents TO other indices + // B) receive some documents FROM other indices + // C) both + const readyToReindexDefers = createMultiPromiseDefer(indicesWithMovingTypes); + const doneReindexingDefers = createMultiPromiseDefer(indicesWithMovingTypes); + + // build a list of all migrators that must be started + const migratorIndices = new Set(Object.keys(indexMap)); + // indices involved in a relocation might no longer be present in current mappings + // but if their SOs must be relocated to another index, we still need a migrator to do the job + indicesWithMovingTypes.forEach((index) => migratorIndices.add(index)); + + const migrators = Array.from(migratorIndices).map((indexName, i) => { return { migrate: (): Promise => { + const readyToReindex = readyToReindexDefers[indexName]; + const doneReindexing = doneReindexingDefers[indexName]; + // check if this migrator's index is involved in some document redistribution + const mustRelocateDocuments = !!readyToReindex; + return runResilientMigrator({ client: this.client, kibanaVersion: this.kibanaVersion, + mustRelocateDocuments, + indexTypesMap, waitForMigrationCompletion: this.waitForMigrationCompletion, - targetMappings: buildActiveMappings(indexMap[index].typeMappings), + // a migrator's index might no longer have any associated types to it + targetMappings: buildActiveMappings(indexMap[indexName]?.typeMappings ?? {}), logger: this.log, - preMigrationScript: indexMap[index].script, + preMigrationScript: indexMap[indexName]?.script, + readyToReindex, + doneReindexing, transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => migrateRawDocsSafely({ serializer: this.serializer, @@ -190,7 +243,7 @@ export class KibanaMigrator implements IKibanaMigrator { rawDocs, }), migrationVersionPerType: this.documentMigrator.migrationVersion, - indexPrefix: index, + indexPrefix: indexName, migrationsConfig: this.soMigrationsConfig, typeRegistry: this.typeRegistry, docLinks: this.docLinks, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts new file mode 100644 index 0000000000000..be199122879ab --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; + +export enum TypeStatus { + Added = 'added', + Removed = 'removed', + Moved = 'moved', + Untouched = 'untouched', +} + +export interface TypeStatusDetails { + currentIndex?: string; + targetIndex?: string; + status: TypeStatus; +} + +export const MAIN_SAVED_OBJECT_INDEX = '.kibana'; +export const LEGACY_SAVED_OBJECT_INDEX = '.kibana_1'; + +// ensure plugins don't try to convert SO namespaceTypes after 8.0.0 +// see https://github.com/elastic/kibana/issues/147344 +export const ALLOWED_CONVERT_VERSION = '8.0.0'; + +export const DEFAULT_INDEX_TYPES_MAP: IndexTypesMap = { + '.kibana_task_manager': ['task'], + '.kibana': [ + 'core-usage-stats', + 'legacy-url-alias', + 'config', + 'config-global', + 'usage-counters', + 'guided-onboarding-guide-state', + 'guided-onboarding-plugin-state', + 'ui-metric', + 'application_usage_totals', + 'application_usage_daily', + 'event_loop_delays_daily', + 'url', + 'index-pattern', + 'sample-data-telemetry', + 'space', + 'spaces-usage-stats', + 'exception-list-agnostic', + 'exception-list', + 'telemetry', + 'file', + 'fileShare', + 'action', + 'action_task_params', + 'connector_token', + 'query', + 'kql-telemetry', + 'search-session', + 'search-telemetry', + 'file-upload-usage-collection-telemetry', + 'alert', + 'api_key_pending_invalidation', + 'rules-settings', + 'search', + 'tag', + 'graph-workspace', + 'visualization', + 'dashboard', + 'todo', + 'book', + 'searchableList', + 'lens', + 'lens-ui-telemetry', + 'map', + 'cases-comments', + 'cases-configure', + 'cases-connector-mappings', + 'cases', + 'cases-user-actions', + 'cases-telemetry', + 'canvas-element', + 'canvas-workpad', + 'canvas-workpad-template', + 'slo', + 'ingest_manager_settings', + 'ingest-agent-policies', + 'ingest-outputs', + 'ingest-package-policies', + 'epm-packages', + 'epm-packages-assets', + 'fleet-preconfiguration-deletion-record', + 'ingest-download-sources', + 'fleet-fleet-server-host', + 'fleet-proxy', + 'fleet-message-signing-keys', + 'osquery-manager-usage-metric', + 'osquery-saved-query', + 'osquery-pack', + 'osquery-pack-asset', + 'csp-rule-template', + 'ml-job', + 'ml-trained-model', + 'ml-module', + 'uptime-dynamic-settings', + 'synthetics-privates-locations', + 'synthetics-monitor', + 'uptime-synthetics-api-key', + 'synthetics-param', + 'siem-ui-timeline-note', + 'siem-ui-timeline-pinned-event', + 'siem-detection-engine-rule-actions', + 'security-rule', + 'siem-ui-timeline', + 'endpoint:user-artifact', + 'endpoint:user-artifact-manifest', + 'security-solution-signals-migration', + 'infrastructure-ui-source', + 'metrics-explorer-view', + 'inventory-view', + 'infrastructure-monitoring-log-view', + 'upgrade-assistant-reindex-operation', + 'upgrade-assistant-ml-upgrade-operation', + 'monitoring-telemetry', + 'enterprise_search_telemetry', + 'app_search_telemetry', + 'workplace_search_telemetry', + 'apm-indices', + 'apm-telemetry', + 'apm-server-schema', + 'apm-service-group', + ], +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts new file mode 100644 index 0000000000000..c748e57705cea --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts @@ -0,0 +1,3413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IndexMap } from './core'; + +export const INDEX_MAP_8_8_0: IndexMap = { + '.kibana': { + typeMappings: { + 'core-usage-stats': { + dynamic: false, + properties: {}, + }, + 'legacy-url-alias': { + dynamic: false, + properties: { + sourceId: { + type: 'keyword', + }, + targetNamespace: { + type: 'keyword', + }, + targetType: { + type: 'keyword', + }, + targetId: { + type: 'keyword', + }, + resolveCounter: { + type: 'long', + }, + disabled: { + type: 'boolean', + }, + }, + }, + config: { + dynamic: false, + properties: { + buildNum: { + type: 'keyword', + }, + }, + }, + 'config-global': { + dynamic: false, + properties: { + buildNum: { + type: 'keyword', + }, + }, + }, + 'usage-counters': { + dynamic: false, + properties: { + domainId: { + type: 'keyword', + }, + }, + }, + 'guided-onboarding-guide-state': { + dynamic: false, + properties: { + guideId: { + type: 'keyword', + }, + isActive: { + type: 'boolean', + }, + }, + }, + 'guided-onboarding-plugin-state': { + dynamic: false, + properties: {}, + }, + 'ui-metric': { + properties: { + count: { + type: 'integer', + }, + }, + }, + application_usage_totals: { + dynamic: false, + properties: {}, + }, + application_usage_daily: { + dynamic: false, + properties: { + timestamp: { + type: 'date', + }, + }, + }, + event_loop_delays_daily: { + dynamic: false, + properties: { + lastUpdatedAt: { + type: 'date', + }, + }, + }, + url: { + properties: { + slug: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + accessCount: { + type: 'long', + }, + accessDate: { + type: 'date', + }, + createDate: { + type: 'date', + }, + url: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 2048, + }, + }, + }, + locatorJSON: { + type: 'text', + index: false, + }, + }, + }, + 'index-pattern': { + dynamic: false, + properties: { + title: { + type: 'text', + }, + type: { + type: 'keyword', + }, + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + }, + }, + 'sample-data-telemetry': { + properties: { + installCount: { + type: 'long', + }, + unInstallCount: { + type: 'long', + }, + }, + }, + space: { + properties: { + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 2048, + }, + }, + }, + description: { + type: 'text', + }, + initials: { + type: 'keyword', + }, + color: { + type: 'keyword', + }, + disabledFeatures: { + type: 'keyword', + }, + imageUrl: { + type: 'text', + index: false, + }, + _reserved: { + type: 'boolean', + }, + }, + }, + 'spaces-usage-stats': { + dynamic: false, + properties: {}, + }, + 'exception-list-agnostic': { + properties: { + _tags: { + type: 'keyword', + }, + created_at: { + type: 'keyword', + }, + created_by: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + immutable: { + type: 'boolean', + }, + list_id: { + type: 'keyword', + }, + list_type: { + type: 'keyword', + }, + meta: { + type: 'keyword', + }, + name: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + tags: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + tie_breaker_id: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + comments: { + properties: { + comment: { + type: 'keyword', + }, + created_at: { + type: 'keyword', + }, + created_by: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + updated_at: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + }, + }, + entries: { + properties: { + entries: { + properties: { + field: { + type: 'keyword', + }, + operator: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + value: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + field: { + type: 'keyword', + }, + list: { + properties: { + id: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + }, + }, + operator: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + value: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + expire_time: { + type: 'date', + }, + item_id: { + type: 'keyword', + }, + os_types: { + type: 'keyword', + }, + }, + }, + 'exception-list': { + properties: { + _tags: { + type: 'keyword', + }, + created_at: { + type: 'keyword', + }, + created_by: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + immutable: { + type: 'boolean', + }, + list_id: { + type: 'keyword', + }, + list_type: { + type: 'keyword', + }, + meta: { + type: 'keyword', + }, + name: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + tags: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + tie_breaker_id: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + comments: { + properties: { + comment: { + type: 'keyword', + }, + created_at: { + type: 'keyword', + }, + created_by: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + updated_at: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + }, + }, + entries: { + properties: { + entries: { + properties: { + field: { + type: 'keyword', + }, + operator: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + value: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + field: { + type: 'keyword', + }, + list: { + properties: { + id: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + }, + }, + operator: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + value: { + fields: { + text: { + type: 'text', + }, + }, + type: 'keyword', + }, + }, + }, + expire_time: { + type: 'date', + }, + item_id: { + type: 'keyword', + }, + os_types: { + type: 'keyword', + }, + }, + }, + telemetry: { + properties: { + enabled: { + type: 'boolean', + }, + sendUsageFrom: { + type: 'keyword', + }, + lastReported: { + type: 'date', + }, + lastVersionChecked: { + type: 'keyword', + }, + userHasSeenNotice: { + type: 'boolean', + }, + reportFailureCount: { + type: 'integer', + }, + reportFailureVersion: { + type: 'keyword', + }, + allowChangingOptInStatus: { + type: 'boolean', + }, + }, + }, + file: { + dynamic: false, + properties: { + created: { + type: 'date', + }, + Updated: { + type: 'date', + }, + name: { + type: 'text', + }, + user: { + type: 'flattened', + }, + Status: { + type: 'keyword', + }, + mime_type: { + type: 'keyword', + }, + extension: { + type: 'keyword', + }, + size: { + type: 'long', + }, + Meta: { + type: 'flattened', + }, + FileKind: { + type: 'keyword', + }, + }, + }, + fileShare: { + dynamic: false, + properties: { + created: { + type: 'date', + }, + valid_until: { + type: 'long', + }, + token: { + type: 'keyword', + }, + name: { + type: 'keyword', + }, + }, + }, + action: { + properties: { + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + actionTypeId: { + type: 'keyword', + }, + isMissingSecrets: { + type: 'boolean', + }, + config: { + enabled: false, + type: 'object', + }, + secrets: { + type: 'binary', + }, + }, + }, + action_task_params: { + dynamic: false, + properties: { + actionId: { + type: 'keyword', + }, + consumer: { + type: 'keyword', + }, + params: { + enabled: false, + type: 'object', + }, + apiKey: { + type: 'binary', + }, + executionId: { + type: 'keyword', + }, + relatedSavedObjects: { + enabled: false, + type: 'object', + }, + }, + }, + connector_token: { + properties: { + connectorId: { + type: 'keyword', + }, + tokenType: { + type: 'keyword', + }, + token: { + type: 'binary', + }, + expiresAt: { + type: 'date', + }, + createdAt: { + type: 'date', + }, + updatedAt: { + type: 'date', + }, + }, + }, + query: { + properties: { + title: { + type: 'text', + }, + description: { + type: 'text', + }, + query: { + properties: { + language: { + type: 'keyword', + }, + query: { + type: 'keyword', + index: false, + }, + }, + }, + filters: { + dynamic: false, + properties: {}, + }, + timefilter: { + dynamic: false, + properties: {}, + }, + }, + }, + 'kql-telemetry': { + properties: { + optInCount: { + type: 'long', + }, + optOutCount: { + type: 'long', + }, + }, + }, + 'search-session': { + properties: { + sessionId: { + type: 'keyword', + }, + name: { + type: 'keyword', + }, + created: { + type: 'date', + }, + expires: { + type: 'date', + }, + appId: { + type: 'keyword', + }, + locatorId: { + type: 'keyword', + }, + initialState: { + dynamic: false, + properties: {}, + }, + restoreState: { + dynamic: false, + properties: {}, + }, + idMapping: { + dynamic: false, + properties: {}, + }, + realmType: { + type: 'keyword', + }, + realmName: { + type: 'keyword', + }, + username: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + isCanceled: { + type: 'boolean', + }, + }, + }, + 'search-telemetry': { + dynamic: false, + properties: {}, + }, + 'file-upload-usage-collection-telemetry': { + properties: { + file_upload: { + properties: { + index_creation_count: { + type: 'long', + }, + }, + }, + }, + }, + alert: { + properties: { + enabled: { + type: 'boolean', + }, + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + normalizer: 'lowercase', + }, + }, + }, + tags: { + type: 'keyword', + }, + alertTypeId: { + type: 'keyword', + }, + schedule: { + properties: { + interval: { + type: 'keyword', + }, + }, + }, + consumer: { + type: 'keyword', + }, + legacyId: { + type: 'keyword', + }, + actions: { + dynamic: false, + type: 'nested', + properties: { + group: { + type: 'keyword', + }, + actionRef: { + type: 'keyword', + }, + actionTypeId: { + type: 'keyword', + }, + params: { + dynamic: false, + properties: {}, + }, + frequency: { + properties: { + summary: { + index: false, + type: 'boolean', + }, + notifyWhen: { + index: false, + type: 'keyword', + }, + throttle: { + index: false, + type: 'keyword', + }, + }, + }, + }, + }, + params: { + type: 'flattened', + ignore_above: 4096, + }, + mapped_params: { + properties: { + risk_score: { + type: 'float', + }, + severity: { + type: 'keyword', + }, + }, + }, + scheduledTaskId: { + type: 'keyword', + }, + createdBy: { + type: 'keyword', + }, + updatedBy: { + type: 'keyword', + }, + createdAt: { + type: 'date', + }, + updatedAt: { + type: 'date', + }, + apiKey: { + type: 'binary', + }, + apiKeyOwner: { + type: 'keyword', + }, + throttle: { + type: 'keyword', + }, + notifyWhen: { + type: 'keyword', + }, + muteAll: { + type: 'boolean', + }, + mutedInstanceIds: { + type: 'keyword', + }, + meta: { + properties: { + versionApiKeyLastmodified: { + type: 'keyword', + }, + }, + }, + monitoring: { + properties: { + run: { + properties: { + history: { + properties: { + duration: { + type: 'long', + }, + success: { + type: 'boolean', + }, + timestamp: { + type: 'date', + }, + outcome: { + type: 'keyword', + }, + }, + }, + calculated_metrics: { + properties: { + p50: { + type: 'long', + }, + p95: { + type: 'long', + }, + p99: { + type: 'long', + }, + success_ratio: { + type: 'float', + }, + }, + }, + last_run: { + properties: { + timestamp: { + type: 'date', + }, + metrics: { + properties: { + duration: { + type: 'long', + }, + total_search_duration_ms: { + type: 'long', + }, + total_indexing_duration_ms: { + type: 'long', + }, + total_alerts_detected: { + type: 'float', + }, + total_alerts_created: { + type: 'float', + }, + gap_duration_s: { + type: 'float', + }, + }, + }, + }, + }, + }, + }, + }, + }, + snoozeSchedule: { + type: 'nested', + properties: { + id: { + type: 'keyword', + }, + duration: { + type: 'long', + }, + skipRecurrences: { + type: 'date', + format: 'strict_date_time', + }, + rRule: { + type: 'nested', + properties: { + freq: { + type: 'keyword', + }, + dtstart: { + type: 'date', + format: 'strict_date_time', + }, + tzid: { + type: 'keyword', + }, + until: { + type: 'date', + format: 'strict_date_time', + }, + count: { + type: 'long', + }, + interval: { + type: 'long', + }, + wkst: { + type: 'keyword', + }, + byweekday: { + type: 'keyword', + }, + bymonth: { + type: 'short', + }, + bysetpos: { + type: 'long', + }, + bymonthday: { + type: 'short', + }, + byyearday: { + type: 'short', + }, + byweekno: { + type: 'short', + }, + byhour: { + type: 'long', + }, + byminute: { + type: 'long', + }, + bysecond: { + type: 'long', + }, + }, + }, + }, + }, + nextRun: { + type: 'date', + }, + executionStatus: { + properties: { + numberOfTriggeredActions: { + type: 'long', + }, + status: { + type: 'keyword', + }, + lastExecutionDate: { + type: 'date', + }, + lastDuration: { + type: 'long', + }, + error: { + properties: { + reason: { + type: 'keyword', + }, + message: { + type: 'keyword', + }, + }, + }, + warning: { + properties: { + reason: { + type: 'keyword', + }, + message: { + type: 'keyword', + }, + }, + }, + }, + }, + lastRun: { + properties: { + outcome: { + type: 'keyword', + }, + outcomeOrder: { + type: 'float', + }, + warning: { + type: 'text', + }, + outcomeMsg: { + type: 'text', + }, + alertsCount: { + properties: { + active: { + type: 'float', + }, + new: { + type: 'float', + }, + recovered: { + type: 'float', + }, + ignored: { + type: 'float', + }, + }, + }, + }, + }, + running: { + type: 'boolean', + }, + }, + }, + api_key_pending_invalidation: { + properties: { + apiKeyId: { + type: 'keyword', + }, + createdAt: { + type: 'date', + }, + }, + }, + 'rules-settings': { + properties: { + flapping: { + properties: { + enabled: { + type: 'boolean', + index: false, + }, + lookBackWindow: { + type: 'long', + index: false, + }, + statusChangeThreshold: { + type: 'long', + index: false, + }, + createdBy: { + type: 'keyword', + index: false, + }, + updatedBy: { + type: 'keyword', + index: false, + }, + createdAt: { + type: 'date', + index: false, + }, + updatedAt: { + type: 'date', + index: false, + }, + }, + }, + }, + }, + search: { + properties: { + columns: { + type: 'keyword', + index: false, + doc_values: false, + }, + description: { + type: 'text', + }, + viewMode: { + type: 'keyword', + index: false, + doc_values: false, + }, + hideChart: { + type: 'boolean', + index: false, + doc_values: false, + }, + isTextBasedQuery: { + type: 'boolean', + index: false, + doc_values: false, + }, + usesAdHocDataView: { + type: 'boolean', + index: false, + doc_values: false, + }, + hideAggregatedPreview: { + type: 'boolean', + index: false, + doc_values: false, + }, + hits: { + type: 'integer', + index: false, + doc_values: false, + }, + kibanaSavedObjectMeta: { + properties: { + searchSourceJSON: { + type: 'text', + index: false, + }, + }, + }, + sort: { + type: 'keyword', + index: false, + doc_values: false, + }, + title: { + type: 'text', + }, + grid: { + dynamic: false, + properties: {}, + }, + version: { + type: 'integer', + }, + rowHeight: { + type: 'text', + }, + timeRestore: { + type: 'boolean', + index: false, + doc_values: false, + }, + timeRange: { + dynamic: false, + properties: { + from: { + type: 'keyword', + index: false, + doc_values: false, + }, + to: { + type: 'keyword', + index: false, + doc_values: false, + }, + }, + }, + refreshInterval: { + dynamic: false, + properties: { + pause: { + type: 'boolean', + index: false, + doc_values: false, + }, + value: { + type: 'integer', + index: false, + doc_values: false, + }, + }, + }, + rowsPerPage: { + type: 'integer', + index: false, + doc_values: false, + }, + breakdownField: { + type: 'text', + }, + }, + }, + tag: { + properties: { + name: { + type: 'text', + }, + description: { + type: 'text', + }, + color: { + type: 'text', + }, + }, + }, + 'graph-workspace': { + properties: { + description: { + type: 'text', + }, + kibanaSavedObjectMeta: { + properties: { + searchSourceJSON: { + type: 'text', + }, + }, + }, + numLinks: { + type: 'integer', + }, + numVertices: { + type: 'integer', + }, + title: { + type: 'text', + }, + version: { + type: 'integer', + }, + wsState: { + type: 'text', + }, + legacyIndexPatternRef: { + type: 'text', + index: false, + }, + }, + }, + visualization: { + properties: { + description: { + type: 'text', + }, + kibanaSavedObjectMeta: { + properties: { + searchSourceJSON: { + type: 'text', + index: false, + }, + }, + }, + savedSearchRefName: { + type: 'keyword', + index: false, + doc_values: false, + }, + title: { + type: 'text', + }, + uiStateJSON: { + type: 'text', + index: false, + }, + version: { + type: 'integer', + }, + visState: { + type: 'text', + index: false, + }, + }, + }, + dashboard: { + properties: { + description: { + type: 'text', + }, + hits: { + type: 'integer', + index: false, + doc_values: false, + }, + kibanaSavedObjectMeta: { + properties: { + searchSourceJSON: { + type: 'text', + index: false, + }, + }, + }, + optionsJSON: { + type: 'text', + index: false, + }, + panelsJSON: { + type: 'text', + index: false, + }, + refreshInterval: { + properties: { + display: { + type: 'keyword', + index: false, + doc_values: false, + }, + pause: { + type: 'boolean', + index: false, + doc_values: false, + }, + section: { + type: 'integer', + index: false, + doc_values: false, + }, + value: { + type: 'integer', + index: false, + doc_values: false, + }, + }, + }, + controlGroupInput: { + properties: { + controlStyle: { + type: 'keyword', + index: false, + doc_values: false, + }, + chainingSystem: { + type: 'keyword', + index: false, + doc_values: false, + }, + panelsJSON: { + type: 'text', + index: false, + }, + ignoreParentSettingsJSON: { + type: 'text', + index: false, + }, + }, + }, + timeFrom: { + type: 'keyword', + index: false, + doc_values: false, + }, + timeRestore: { + type: 'boolean', + index: false, + doc_values: false, + }, + timeTo: { + type: 'keyword', + index: false, + doc_values: false, + }, + title: { + type: 'text', + }, + version: { + type: 'integer', + }, + }, + }, + todo: { + properties: { + title: { + type: 'keyword', + }, + task: { + type: 'text', + }, + icon: { + type: 'keyword', + }, + }, + }, + book: { + properties: { + title: { + type: 'keyword', + }, + author: { + type: 'keyword', + }, + readIt: { + type: 'boolean', + }, + }, + }, + searchableList: { + properties: { + title: { + type: 'text', + }, + version: { + type: 'integer', + }, + }, + }, + lens: { + properties: { + title: { + type: 'text', + }, + description: { + type: 'text', + }, + visualizationType: { + type: 'keyword', + }, + state: { + type: 'flattened', + }, + expression: { + index: false, + doc_values: false, + type: 'keyword', + }, + }, + }, + 'lens-ui-telemetry': { + properties: { + name: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + date: { + type: 'date', + }, + count: { + type: 'integer', + }, + }, + }, + map: { + properties: { + description: { + type: 'text', + }, + title: { + type: 'text', + }, + version: { + type: 'integer', + }, + mapStateJSON: { + type: 'text', + }, + layerListJSON: { + type: 'text', + }, + uiStateJSON: { + type: 'text', + }, + bounds: { + dynamic: false, + properties: {}, + }, + }, + }, + 'cases-comments': { + dynamic: false, + properties: { + comment: { + type: 'text', + }, + owner: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + actions: { + properties: { + type: { + type: 'keyword', + }, + }, + }, + alertId: { + type: 'keyword', + }, + created_at: { + type: 'date', + }, + created_by: { + properties: { + username: { + type: 'keyword', + }, + }, + }, + externalReferenceAttachmentTypeId: { + type: 'keyword', + }, + persistableStateAttachmentTypeId: { + type: 'keyword', + }, + pushed_at: { + type: 'date', + }, + updated_at: { + type: 'date', + }, + }, + }, + 'cases-configure': { + dynamic: false, + properties: { + created_at: { + type: 'date', + }, + closure_type: { + type: 'keyword', + }, + owner: { + type: 'keyword', + }, + }, + }, + 'cases-connector-mappings': { + dynamic: false, + properties: { + owner: { + type: 'keyword', + }, + }, + }, + cases: { + dynamic: false, + properties: { + assignees: { + properties: { + uid: { + type: 'keyword', + }, + }, + }, + closed_at: { + type: 'date', + }, + closed_by: { + properties: { + username: { + type: 'keyword', + }, + full_name: { + type: 'keyword', + }, + email: { + type: 'keyword', + }, + profile_uid: { + type: 'keyword', + }, + }, + }, + created_at: { + type: 'date', + }, + created_by: { + properties: { + username: { + type: 'keyword', + }, + full_name: { + type: 'keyword', + }, + email: { + type: 'keyword', + }, + profile_uid: { + type: 'keyword', + }, + }, + }, + duration: { + type: 'unsigned_long', + }, + description: { + type: 'text', + }, + connector: { + properties: { + name: { + type: 'text', + }, + type: { + type: 'keyword', + }, + fields: { + properties: { + key: { + type: 'text', + }, + value: { + type: 'text', + }, + }, + }, + }, + }, + external_service: { + properties: { + pushed_at: { + type: 'date', + }, + pushed_by: { + properties: { + username: { + type: 'keyword', + }, + full_name: { + type: 'keyword', + }, + email: { + type: 'keyword', + }, + profile_uid: { + type: 'keyword', + }, + }, + }, + connector_name: { + type: 'keyword', + }, + external_id: { + type: 'keyword', + }, + external_title: { + type: 'text', + }, + external_url: { + type: 'text', + }, + }, + }, + owner: { + type: 'keyword', + }, + title: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + status: { + type: 'short', + }, + tags: { + type: 'keyword', + }, + updated_at: { + type: 'date', + }, + updated_by: { + properties: { + username: { + type: 'keyword', + }, + full_name: { + type: 'keyword', + }, + email: { + type: 'keyword', + }, + profile_uid: { + type: 'keyword', + }, + }, + }, + settings: { + properties: { + syncAlerts: { + type: 'boolean', + }, + }, + }, + severity: { + type: 'short', + }, + total_alerts: { + type: 'integer', + }, + total_comments: { + type: 'integer', + }, + }, + }, + 'cases-user-actions': { + dynamic: false, + properties: { + action: { + type: 'keyword', + }, + created_at: { + type: 'date', + }, + created_by: { + properties: { + username: { + type: 'keyword', + }, + }, + }, + payload: { + dynamic: false, + properties: { + connector: { + properties: { + type: { + type: 'keyword', + }, + }, + }, + comment: { + properties: { + type: { + type: 'keyword', + }, + externalReferenceAttachmentTypeId: { + type: 'keyword', + }, + persistableStateAttachmentTypeId: { + type: 'keyword', + }, + }, + }, + assignees: { + properties: { + uid: { + type: 'keyword', + }, + }, + }, + }, + }, + owner: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + }, + }, + 'cases-telemetry': { + dynamic: false, + properties: {}, + }, + 'canvas-element': { + dynamic: false, + properties: { + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + help: { + type: 'text', + }, + content: { + type: 'text', + }, + image: { + type: 'text', + }, + '@timestamp': { + type: 'date', + }, + '@created': { + type: 'date', + }, + }, + }, + 'canvas-workpad': { + dynamic: false, + properties: { + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + '@timestamp': { + type: 'date', + }, + '@created': { + type: 'date', + }, + }, + }, + 'canvas-workpad-template': { + dynamic: false, + properties: { + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + help: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + tags: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + template_key: { + type: 'keyword', + }, + }, + }, + slo: { + dynamic: false, + properties: { + id: { + type: 'keyword', + }, + name: { + type: 'keyword', + }, + description: { + type: 'text', + }, + indicator: { + properties: { + type: { + type: 'keyword', + }, + params: { + type: 'flattened', + }, + }, + }, + timeWindow: { + properties: { + duration: { + type: 'keyword', + }, + isRolling: { + type: 'boolean', + }, + calendar: { + properties: { + startTime: { + type: 'date', + }, + }, + }, + }, + }, + budgetingMethod: { + type: 'keyword', + }, + objective: { + properties: { + target: { + type: 'float', + }, + timesliceTarget: { + type: 'float', + }, + timesliceWindow: { + type: 'keyword', + }, + }, + }, + settings: { + properties: { + timestampField: { + type: 'keyword', + }, + syncDelay: { + type: 'keyword', + }, + frequency: { + type: 'keyword', + }, + }, + }, + revision: { + type: 'short', + }, + enabled: { + type: 'boolean', + }, + createdAt: { + type: 'date', + }, + updatedAt: { + type: 'date', + }, + }, + }, + ingest_manager_settings: { + properties: { + fleet_server_hosts: { + type: 'keyword', + }, + has_seen_add_data_notice: { + type: 'boolean', + index: false, + }, + prerelease_integrations_enabled: { + type: 'boolean', + }, + }, + }, + 'ingest-agent-policies': { + properties: { + name: { + type: 'keyword', + }, + schema_version: { + type: 'version', + }, + description: { + type: 'text', + }, + namespace: { + type: 'keyword', + }, + is_managed: { + type: 'boolean', + }, + is_default: { + type: 'boolean', + }, + is_default_fleet_server: { + type: 'boolean', + }, + status: { + type: 'keyword', + }, + unenroll_timeout: { + type: 'integer', + }, + inactivity_timeout: { + type: 'integer', + }, + updated_at: { + type: 'date', + }, + updated_by: { + type: 'keyword', + }, + revision: { + type: 'integer', + }, + monitoring_enabled: { + type: 'keyword', + index: false, + }, + is_preconfigured: { + type: 'keyword', + }, + data_output_id: { + type: 'keyword', + }, + monitoring_output_id: { + type: 'keyword', + }, + download_source_id: { + type: 'keyword', + }, + fleet_server_host_id: { + type: 'keyword', + }, + agent_features: { + properties: { + name: { + type: 'keyword', + }, + enabled: { + type: 'boolean', + }, + }, + }, + }, + }, + 'ingest-outputs': { + properties: { + output_id: { + type: 'keyword', + index: false, + }, + name: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + is_default: { + type: 'boolean', + }, + is_default_monitoring: { + type: 'boolean', + }, + hosts: { + type: 'keyword', + }, + ca_sha256: { + type: 'keyword', + index: false, + }, + ca_trusted_fingerprint: { + type: 'keyword', + index: false, + }, + config: { + type: 'flattened', + }, + config_yaml: { + type: 'text', + }, + is_preconfigured: { + type: 'boolean', + index: false, + }, + ssl: { + type: 'binary', + }, + proxy_id: { + type: 'keyword', + }, + shipper: { + dynamic: false, + properties: {}, + }, + }, + }, + 'ingest-package-policies': { + properties: { + name: { + type: 'keyword', + }, + description: { + type: 'text', + }, + namespace: { + type: 'keyword', + }, + enabled: { + type: 'boolean', + }, + is_managed: { + type: 'boolean', + }, + policy_id: { + type: 'keyword', + }, + package: { + properties: { + name: { + type: 'keyword', + }, + title: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + }, + }, + elasticsearch: { + dynamic: false, + properties: {}, + }, + vars: { + type: 'flattened', + }, + inputs: { + dynamic: false, + properties: {}, + }, + revision: { + type: 'integer', + }, + updated_at: { + type: 'date', + }, + updated_by: { + type: 'keyword', + }, + created_at: { + type: 'date', + }, + created_by: { + type: 'keyword', + }, + }, + }, + 'epm-packages': { + properties: { + name: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + internal: { + type: 'boolean', + }, + keep_policies_up_to_date: { + type: 'boolean', + index: false, + }, + es_index_patterns: { + dynamic: false, + properties: {}, + }, + verification_status: { + type: 'keyword', + }, + verification_key_id: { + type: 'keyword', + }, + installed_es: { + type: 'nested', + properties: { + id: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + }, + }, + installed_kibana: { + dynamic: false, + properties: {}, + }, + installed_kibana_space_id: { + type: 'keyword', + }, + package_assets: { + dynamic: false, + properties: {}, + }, + install_started_at: { + type: 'date', + }, + install_version: { + type: 'keyword', + }, + install_status: { + type: 'keyword', + }, + install_source: { + type: 'keyword', + }, + install_format_schema_version: { + type: 'version', + }, + experimental_data_stream_features: { + type: 'nested', + properties: { + data_stream: { + type: 'keyword', + }, + features: { + type: 'nested', + dynamic: false, + properties: { + synthetic_source: { + type: 'boolean', + }, + tsdb: { + type: 'boolean', + }, + }, + }, + }, + }, + }, + }, + 'epm-packages-assets': { + properties: { + package_name: { + type: 'keyword', + }, + package_version: { + type: 'keyword', + }, + install_source: { + type: 'keyword', + }, + asset_path: { + type: 'keyword', + }, + media_type: { + type: 'keyword', + }, + data_utf8: { + type: 'text', + index: false, + }, + data_base64: { + type: 'binary', + }, + }, + }, + 'fleet-preconfiguration-deletion-record': { + properties: { + id: { + type: 'keyword', + }, + }, + }, + 'ingest-download-sources': { + properties: { + source_id: { + type: 'keyword', + index: false, + }, + name: { + type: 'keyword', + }, + is_default: { + type: 'boolean', + }, + host: { + type: 'keyword', + }, + }, + }, + 'fleet-fleet-server-host': { + properties: { + name: { + type: 'keyword', + }, + is_default: { + type: 'boolean', + }, + host_urls: { + type: 'keyword', + index: false, + }, + is_preconfigured: { + type: 'boolean', + }, + proxy_id: { + type: 'keyword', + }, + }, + }, + 'fleet-proxy': { + properties: { + name: { + type: 'keyword', + }, + url: { + type: 'keyword', + index: false, + }, + proxy_headers: { + type: 'text', + index: false, + }, + certificate_authorities: { + type: 'keyword', + index: false, + }, + certificate: { + type: 'keyword', + index: false, + }, + certificate_key: { + type: 'keyword', + index: false, + }, + is_preconfigured: { + type: 'boolean', + }, + }, + }, + 'fleet-message-signing-keys': { + dynamic: false, + properties: {}, + }, + 'osquery-manager-usage-metric': { + properties: { + count: { + type: 'long', + }, + errors: { + type: 'long', + }, + }, + }, + 'osquery-saved-query': { + dynamic: false, + properties: { + description: { + type: 'text', + }, + id: { + type: 'keyword', + }, + query: { + type: 'text', + }, + created_at: { + type: 'date', + }, + created_by: { + type: 'text', + }, + platform: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + updated_at: { + type: 'date', + }, + updated_by: { + type: 'text', + }, + interval: { + type: 'keyword', + }, + ecs_mapping: { + dynamic: false, + properties: {}, + }, + }, + }, + 'osquery-pack': { + properties: { + description: { + type: 'text', + }, + name: { + type: 'text', + }, + created_at: { + type: 'date', + }, + created_by: { + type: 'keyword', + }, + updated_at: { + type: 'date', + }, + updated_by: { + type: 'keyword', + }, + enabled: { + type: 'boolean', + }, + shards: { + dynamic: false, + properties: {}, + }, + version: { + type: 'long', + }, + queries: { + dynamic: false, + properties: { + id: { + type: 'keyword', + }, + query: { + type: 'text', + }, + interval: { + type: 'text', + }, + platform: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + ecs_mapping: { + dynamic: false, + properties: {}, + }, + }, + }, + }, + }, + 'osquery-pack-asset': { + dynamic: false, + properties: { + description: { + type: 'text', + }, + name: { + type: 'text', + }, + version: { + type: 'long', + }, + shards: { + dynamic: false, + properties: {}, + }, + queries: { + dynamic: false, + properties: { + id: { + type: 'keyword', + }, + query: { + type: 'text', + }, + interval: { + type: 'text', + }, + platform: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + ecs_mapping: { + dynamic: false, + properties: {}, + }, + }, + }, + }, + }, + 'csp-rule-template': { + dynamic: false, + properties: { + metadata: { + type: 'object', + properties: { + name: { + type: 'keyword', + fields: { + text: { + type: 'text', + }, + }, + }, + benchmark: { + type: 'object', + properties: { + id: { + type: 'keyword', + }, + }, + }, + }, + }, + }, + }, + 'ml-job': { + properties: { + job_id: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + datafeed_id: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + type: { + type: 'keyword', + }, + }, + }, + 'ml-trained-model': { + properties: { + model_id: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + job: { + properties: { + job_id: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + create_time: { + type: 'date', + }, + }, + }, + }, + }, + 'ml-module': { + dynamic: false, + properties: { + id: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + title: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + description: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + type: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + logo: { + type: 'object', + }, + defaultIndexPattern: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + }, + }, + }, + query: { + type: 'object', + }, + jobs: { + type: 'object', + }, + datafeeds: { + type: 'object', + }, + }, + }, + 'uptime-dynamic-settings': { + dynamic: false, + properties: {}, + }, + 'synthetics-privates-locations': { + dynamic: false, + properties: {}, + }, + 'synthetics-monitor': { + dynamic: false, + properties: { + name: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + normalizer: 'lowercase', + }, + }, + }, + type: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + urls: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + hosts: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + journey_id: { + type: 'keyword', + }, + project_id: { + type: 'keyword', + fields: { + text: { + type: 'text', + }, + }, + }, + origin: { + type: 'keyword', + }, + hash: { + type: 'keyword', + }, + locations: { + properties: { + id: { + type: 'keyword', + ignore_above: 256, + fields: { + text: { + type: 'text', + }, + }, + }, + label: { + type: 'text', + }, + }, + }, + custom_heartbeat_id: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + tags: { + type: 'keyword', + fields: { + text: { + type: 'text', + }, + }, + }, + schedule: { + properties: { + number: { + type: 'integer', + }, + }, + }, + enabled: { + type: 'boolean', + }, + alert: { + properties: { + status: { + properties: { + enabled: { + type: 'boolean', + }, + }, + }, + }, + }, + }, + }, + 'uptime-synthetics-api-key': { + dynamic: false, + properties: { + apiKey: { + type: 'binary', + }, + }, + }, + 'synthetics-param': { + dynamic: false, + properties: {}, + }, + 'siem-ui-timeline-note': { + properties: { + eventId: { + type: 'keyword', + }, + note: { + type: 'text', + }, + created: { + type: 'date', + }, + createdBy: { + type: 'text', + }, + updated: { + type: 'date', + }, + updatedBy: { + type: 'text', + }, + }, + }, + 'siem-ui-timeline-pinned-event': { + properties: { + eventId: { + type: 'keyword', + }, + created: { + type: 'date', + }, + createdBy: { + type: 'text', + }, + updated: { + type: 'date', + }, + updatedBy: { + type: 'text', + }, + }, + }, + 'siem-detection-engine-rule-actions': { + properties: { + alertThrottle: { + type: 'keyword', + }, + ruleAlertId: { + type: 'keyword', + }, + ruleThrottle: { + type: 'keyword', + }, + actions: { + properties: { + actionRef: { + type: 'keyword', + }, + group: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + action_type_id: { + type: 'keyword', + }, + params: { + dynamic: false, + properties: {}, + }, + }, + }, + }, + }, + 'security-rule': { + dynamic: false, + properties: { + name: { + type: 'keyword', + }, + rule_id: { + type: 'keyword', + }, + version: { + type: 'long', + }, + }, + }, + 'siem-ui-timeline': { + properties: { + columns: { + properties: { + aggregatable: { + type: 'boolean', + }, + category: { + type: 'keyword', + }, + columnHeaderType: { + type: 'keyword', + }, + description: { + type: 'text', + }, + example: { + type: 'text', + }, + indexes: { + type: 'keyword', + }, + id: { + type: 'keyword', + }, + name: { + type: 'text', + }, + placeholder: { + type: 'text', + }, + searchable: { + type: 'boolean', + }, + type: { + type: 'keyword', + }, + }, + }, + dataProviders: { + properties: { + id: { + type: 'keyword', + }, + name: { + type: 'text', + }, + enabled: { + type: 'boolean', + }, + excluded: { + type: 'boolean', + }, + kqlQuery: { + type: 'text', + }, + type: { + type: 'text', + }, + queryMatch: { + properties: { + field: { + type: 'text', + }, + displayField: { + type: 'text', + }, + value: { + type: 'text', + }, + displayValue: { + type: 'text', + }, + operator: { + type: 'text', + }, + }, + }, + and: { + properties: { + id: { + type: 'keyword', + }, + name: { + type: 'text', + }, + enabled: { + type: 'boolean', + }, + excluded: { + type: 'boolean', + }, + kqlQuery: { + type: 'text', + }, + type: { + type: 'text', + }, + queryMatch: { + properties: { + field: { + type: 'text', + }, + displayField: { + type: 'text', + }, + value: { + type: 'text', + }, + displayValue: { + type: 'text', + }, + operator: { + type: 'text', + }, + }, + }, + }, + }, + }, + }, + description: { + type: 'text', + }, + eqlOptions: { + properties: { + eventCategoryField: { + type: 'text', + }, + tiebreakerField: { + type: 'text', + }, + timestampField: { + type: 'text', + }, + query: { + type: 'text', + }, + size: { + type: 'text', + }, + }, + }, + eventType: { + type: 'keyword', + }, + excludedRowRendererIds: { + type: 'text', + }, + favorite: { + properties: { + keySearch: { + type: 'text', + }, + fullName: { + type: 'text', + }, + userName: { + type: 'text', + }, + favoriteDate: { + type: 'date', + }, + }, + }, + filters: { + properties: { + meta: { + properties: { + alias: { + type: 'text', + }, + controlledBy: { + type: 'text', + }, + disabled: { + type: 'boolean', + }, + field: { + type: 'text', + }, + formattedValue: { + type: 'text', + }, + index: { + type: 'keyword', + }, + key: { + type: 'keyword', + }, + negate: { + type: 'boolean', + }, + params: { + type: 'text', + }, + type: { + type: 'keyword', + }, + value: { + type: 'text', + }, + }, + }, + exists: { + type: 'text', + }, + match_all: { + type: 'text', + }, + missing: { + type: 'text', + }, + query: { + type: 'text', + }, + range: { + type: 'text', + }, + script: { + type: 'text', + }, + }, + }, + indexNames: { + type: 'text', + }, + kqlMode: { + type: 'keyword', + }, + kqlQuery: { + properties: { + filterQuery: { + properties: { + kuery: { + properties: { + kind: { + type: 'keyword', + }, + expression: { + type: 'text', + }, + }, + }, + serializedQuery: { + type: 'text', + }, + }, + }, + }, + }, + title: { + type: 'text', + }, + templateTimelineId: { + type: 'text', + }, + templateTimelineVersion: { + type: 'integer', + }, + timelineType: { + type: 'keyword', + }, + dateRange: { + properties: { + start: { + type: 'date', + }, + end: { + type: 'date', + }, + }, + }, + sort: { + dynamic: false, + properties: { + columnId: { + type: 'keyword', + }, + columnType: { + type: 'keyword', + }, + sortDirection: { + type: 'keyword', + }, + }, + }, + status: { + type: 'keyword', + }, + created: { + type: 'date', + }, + createdBy: { + type: 'text', + }, + updated: { + type: 'date', + }, + updatedBy: { + type: 'text', + }, + }, + }, + 'endpoint:user-artifact': { + properties: { + identifier: { + type: 'keyword', + }, + compressionAlgorithm: { + type: 'keyword', + index: false, + }, + encryptionAlgorithm: { + type: 'keyword', + index: false, + }, + encodedSha256: { + type: 'keyword', + }, + encodedSize: { + type: 'long', + index: false, + }, + decodedSha256: { + type: 'keyword', + index: false, + }, + decodedSize: { + type: 'long', + index: false, + }, + created: { + type: 'date', + index: false, + }, + body: { + type: 'binary', + }, + }, + }, + 'endpoint:user-artifact-manifest': { + properties: { + created: { + type: 'date', + index: false, + }, + schemaVersion: { + type: 'keyword', + }, + semanticVersion: { + type: 'keyword', + index: false, + }, + artifacts: { + type: 'nested', + properties: { + policyId: { + type: 'keyword', + index: false, + }, + artifactId: { + type: 'keyword', + index: false, + }, + }, + }, + }, + }, + 'security-solution-signals-migration': { + properties: { + sourceIndex: { + type: 'keyword', + }, + destinationIndex: { + type: 'keyword', + index: false, + }, + version: { + type: 'long', + }, + error: { + type: 'text', + index: false, + }, + taskId: { + type: 'keyword', + index: false, + }, + status: { + type: 'keyword', + index: false, + }, + created: { + type: 'date', + index: false, + }, + createdBy: { + type: 'text', + index: false, + }, + updated: { + type: 'date', + index: false, + }, + updatedBy: { + type: 'text', + index: false, + }, + }, + }, + 'infrastructure-ui-source': { + dynamic: false, + properties: {}, + }, + 'metrics-explorer-view': { + dynamic: false, + properties: {}, + }, + 'inventory-view': { + dynamic: false, + properties: {}, + }, + 'infrastructure-monitoring-log-view': { + dynamic: false, + properties: { + name: { + type: 'text', + }, + }, + }, + 'upgrade-assistant-reindex-operation': { + dynamic: false, + properties: { + indexName: { + type: 'keyword', + }, + status: { + type: 'integer', + }, + }, + }, + 'upgrade-assistant-ml-upgrade-operation': { + dynamic: false, + properties: { + snapshotId: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + }, + }, + 'monitoring-telemetry': { + properties: { + reportedClusterUuids: { + type: 'keyword', + }, + }, + }, + enterprise_search_telemetry: { + dynamic: false, + properties: {}, + }, + app_search_telemetry: { + dynamic: false, + properties: {}, + }, + workplace_search_telemetry: { + dynamic: false, + properties: {}, + }, + 'apm-indices': { + dynamic: false, + properties: {}, + }, + 'apm-telemetry': { + dynamic: false, + properties: {}, + }, + 'apm-server-schema': { + properties: { + schemaJson: { + type: 'text', + index: false, + }, + }, + }, + 'apm-service-group': { + properties: { + groupName: { + type: 'keyword', + }, + kuery: { + type: 'text', + }, + description: { + type: 'text', + }, + color: { + type: 'text', + }, + }, + }, + }, + }, + '.kibana_task_manager': { + typeMappings: { + task: { + properties: { + taskType: { + type: 'keyword', + }, + scheduledAt: { + type: 'date', + }, + runAt: { + type: 'date', + }, + startedAt: { + type: 'date', + }, + retryAt: { + type: 'date', + }, + enabled: { + type: 'boolean', + }, + schedule: { + properties: { + interval: { + type: 'keyword', + }, + }, + }, + attempts: { + type: 'integer', + }, + status: { + type: 'keyword', + }, + traceparent: { + type: 'text', + }, + params: { + type: 'text', + }, + state: { + type: 'text', + }, + user: { + type: 'keyword', + }, + scope: { + type: 'keyword', + }, + ownerId: { + type: 'keyword', + }, + }, + }, + }, + script: 'ctx._id = ctx._source.type + \':\' + ctx._id; ctx._source.remove("kibana")', + }, +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts new file mode 100644 index 0000000000000..2b00d12c7bce5 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts @@ -0,0 +1,257 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { errors } from '@elastic/elasticsearch'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants'; +import { + calculateTypeStatuses, + createMultiPromiseDefer, + getIndicesInvoledInRelocation, + indexMapToIndexTypesMap, +} from './kibana_migrator_utils'; +import { INDEX_MAP_8_8_0 } from './kibana_migrator_utils.fixtures'; + +describe('createMultiPromiseDefer', () => { + it('creates defer objects with the same Promise', () => { + const defers = createMultiPromiseDefer(['.kibana', '.kibana_cases']); + expect(Object.keys(defers)).toHaveLength(2); + expect(defers['.kibana'].promise).toEqual(defers['.kibana_cases'].promise); + expect(defers['.kibana'].resolve).not.toEqual(defers['.kibana_cases'].resolve); + expect(defers['.kibana'].reject).not.toEqual(defers['.kibana_cases'].reject); + }); + + it('the common Promise resolves when all defers resolve', async () => { + const defers = createMultiPromiseDefer(['.kibana', '.kibana_cases']); + let resolved = 0; + Object.values(defers).forEach((defer) => defer.promise.then(() => ++resolved)); + defers['.kibana'].resolve(); + await new Promise((resolve) => setImmediate(resolve)); // next tick + expect(resolved).toEqual(0); + defers['.kibana_cases'].resolve(); + await new Promise((resolve) => setImmediate(resolve)); // next tick + expect(resolved).toEqual(2); + }); +}); + +describe('getIndicesInvoledInRelocation', () => { + const getIndicesInvoledInRelocationParams = () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + (client as any).child = jest.fn().mockImplementation(() => client); + + return { + client, + mainIndex: '.kibana', + legacyIndex: '.kibana_1', + indexTypesMap: {}, + defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + logger: loggerMock.create(), + }; + }; + + it('tries to get the indexTypesMap from the mainIndex', async () => { + const params = getIndicesInvoledInRelocationParams(); + try { + await getIndicesInvoledInRelocation(params); + } catch (err) { + // ignore + } + + expect(params.client.indices.getMapping).toHaveBeenCalledTimes(1); + expect(params.client.indices.getMapping).toHaveBeenCalledWith({ + index: '.kibana', + }); + }); + + it('fails if the query to get indexTypesMap fails with critical error', async () => { + const params = getIndicesInvoledInRelocationParams(); + params.client.indices.getMapping.mockImplementation(() => + elasticsearchClientMock.createErrorTransportRequestPromise( + new errors.ResponseError({ + statusCode: 500, + body: { + error: { + type: 'error_type', + reason: 'error_reason', + }, + }, + warnings: [], + headers: {}, + meta: {} as any, + }) + ) + ); + expect(getIndicesInvoledInRelocation(params)).rejects.toThrowErrorMatchingInlineSnapshot( + `"error_type"` + ); + }); + + it('assumes fresh deployment if the mainIndex and legacyIndex do not exist, returns an empty list of moving types', async () => { + const params = getIndicesInvoledInRelocationParams(); + params.client.indices.getMapping.mockImplementation(() => + elasticsearchClientMock.createErrorTransportRequestPromise( + new errors.ResponseError({ + statusCode: 404, + body: { + error: { + type: 'error_type', + reason: 'error_reason', + }, + }, + warnings: [], + headers: {}, + meta: {} as any, + }) + ) + ); + expect(getIndicesInvoledInRelocation(params)).resolves.toEqual([]); + }); + + describe('if mainIndex exists', () => { + describe('but it does not have an indexTypeMap stored', () => { + it('uses the defaultIndexTypeMap and finds out which indices are involved in a relocation', async () => { + const params = getIndicesInvoledInRelocationParams(); + params.client.indices.getMapping.mockReturnValue( + Promise.resolve({ + '.kibana_8.7.0_001': { + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: { + someType: '7997cf5a56cc02bdc9c93361bde732b0', + }, + }, + properties: { + someProperty: {}, + }, + }, + }, + }) + ); + params.defaultIndexTypesMap = { + '.indexA': ['type1', 'type2', 'type3'], + '.indexB': ['type4', 'type5', 'type6'], + }; + + params.indexTypesMap = { + '.indexA': ['type1'], // move type2 and type 3 over to new indexC + '.indexB': ['type4', 'type5', 'type6'], // stays the same + '.indexC': ['type2', 'type3'], + }; + + expect(getIndicesInvoledInRelocation(params)).resolves.toEqual(['.indexA', '.indexC']); + }); + }); + + describe('and it has an indexTypeMap stored', () => { + it('compares stored indexTypeMap against desired one, and finds out which indices are involved in a relocation', async () => { + const params = getIndicesInvoledInRelocationParams(); + params.client.indices.getMapping.mockReturnValue( + Promise.resolve({ + '.kibana_8.8.0_001': { + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: { + someType: '7997cf5a56cc02bdc9c93361bde732b0', + }, + // map stored on index + indexTypesMap: { + '.indexA': ['type1'], + '.indexB': ['type4', 'type5', 'type6'], + '.indexC': ['type2', 'type3'], + }, + }, + properties: { + someProperty: {}, + }, + }, + }, + }) + ); + + // exists on index, so this one will NOT be taken into account + params.defaultIndexTypesMap = { + '.indexA': ['type1', 'type2', 'type3'], + '.indexB': ['type4', 'type5', 'type6'], + }; + + params.indexTypesMap = { + '.indexA': ['type1'], + '.indexB': ['type4'], + '.indexC': ['type2', 'type3'], + '.indexD': ['type5', 'type6'], + }; + + expect(getIndicesInvoledInRelocation(params)).resolves.toEqual(['.indexB', '.indexD']); + }); + }); + }); +}); + +describe('indexMapToIndexTypesMap', () => { + it('converts IndexMap to IndexTypesMap', () => { + expect(indexMapToIndexTypesMap(INDEX_MAP_8_8_0)).toEqual(DEFAULT_INDEX_TYPES_MAP); + }); +}); + +describe('calculateTypeStatuses', () => { + it('takes two indexTypesMaps and checks what types have been added, removed and relocated', () => { + const currentIndexTypesMap = { + '.indexA': ['type1', 'type2', 'type3'], + '.indexB': ['type4', 'type5', 'type6'], + }; + const desiredIndexTypesMap = { + '.indexA': ['type2'], + '.indexB': ['type3', 'type5'], + '.indexC': ['type4', 'type6', 'type7'], + '.indexD': ['type8'], + }; + + expect(calculateTypeStatuses(currentIndexTypesMap, desiredIndexTypesMap)).toEqual({ + type1: { + currentIndex: '.indexA', + status: 'removed', + }, + type2: { + currentIndex: '.indexA', + status: 'untouched', + targetIndex: '.indexA', + }, + type3: { + currentIndex: '.indexA', + status: 'moved', + targetIndex: '.indexB', + }, + type4: { + currentIndex: '.indexB', + status: 'moved', + targetIndex: '.indexC', + }, + type5: { + currentIndex: '.indexB', + status: 'untouched', + targetIndex: '.indexB', + }, + type6: { + currentIndex: '.indexB', + status: 'moved', + targetIndex: '.indexC', + }, + type7: { + status: 'added', + targetIndex: '.indexC', + }, + type8: { + status: 'added', + targetIndex: '.indexD', + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts new file mode 100644 index 0000000000000..1deba9561574c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { Logger } from '@kbn/logging'; +import type { IndexMap } from './core'; +import { TypeStatus, type TypeStatusDetails } from './kibana_migrator_constants'; + +// even though this utility class is present in @kbn/kibana-utils-plugin, we can't easily import it from Core +// aka. one does not simply reuse code +export class Defer { + public resolve!: (data: T) => void; + public reject!: (error: any) => void; + public promise: Promise = new Promise((resolve, reject) => { + (this as any).resolve = resolve; + (this as any).reject = reject; + }); +} + +export const defer = () => new Defer(); + +export function createMultiPromiseDefer(indices: string[]): Record> { + const defers: Array> = new Array(indices.length).fill(true).map(defer); + const all = Promise.all(defers.map(({ promise }) => promise)); + return indices.reduce>>((acc, indexName, i) => { + const { resolve, reject } = defers[i]; + acc[indexName] = { resolve, reject, promise: all }; + return acc; + }, {}); +} + +export async function getCurrentIndexTypesMap({ + client, + mainIndex, + legacyIndex, + defaultIndexTypesMap, + logger, +}: { + client: ElasticsearchClient; + mainIndex: string; + legacyIndex: string; + defaultIndexTypesMap: IndexTypesMap; + logger: Logger; +}): Promise { + try { + // check if the main index (i.e. .kibana) exists + const mapping = await client.indices.getMapping({ + index: mainIndex, + }); + + // main index exists, try to extract the indexTypesMap from _meta + const meta = Object.values(mapping)?.[0]?.mappings._meta; + return meta?.indexTypesMap ?? defaultIndexTypesMap; + } catch (mainError) { + if (mainError.meta?.statusCode === 404) { + // the main index does NOT exist. Check if the legacy index (i.e. .kibana_1) exists + try { + await client.indices.getMapping({ + index: legacyIndex, + }); + // legacy exists, we assume we will have a default indexTypesMap after LEGACY migration + return defaultIndexTypesMap; + } catch (legacyError) { + if (legacyError.meta?.statusCode === 404) { + // legacy does NOT exist, we assume this is a fresh deployment + logger.debug( + `The ${mainIndex} and ${legacyIndex} indices do NOT exist. Assuming this is a fresh deployment` + ); + return undefined; + } else { + logger.fatal( + `Cannot query the meta information on the ${legacyIndex} saved object index` + ); + throw legacyError; + } + } + } else { + logger.fatal(`Cannot query the meta information on the ${mainIndex} saved object index`); + throw mainError; + } + } +} + +export async function getIndicesInvoledInRelocation({ + client, + mainIndex, + legacyIndex, + indexTypesMap, + defaultIndexTypesMap, + logger, +}: { + client: ElasticsearchClient; + mainIndex: string; + legacyIndex: string; + indexTypesMap: IndexTypesMap; + defaultIndexTypesMap: IndexTypesMap; + logger: Logger; +}): Promise { + const indicesWithMovingTypesSet = new Set(); + + const currentIndexTypesMap = await getCurrentIndexTypesMap({ + client, + mainIndex, + legacyIndex, + defaultIndexTypesMap, + logger, + }); + + if (!currentIndexTypesMap) { + // this is a fresh deployment, no indices must be relocated + return []; + } + + const typeIndexDistribution = calculateTypeStatuses(currentIndexTypesMap, indexTypesMap); + + const relocated = Object.entries(typeIndexDistribution).filter( + ([, { status }]) => status === TypeStatus.Moved + ); + relocated.forEach(([, { currentIndex, targetIndex }]) => { + indicesWithMovingTypesSet.add(currentIndex!); + indicesWithMovingTypesSet.add(targetIndex!); + }); + + return Array.from(indicesWithMovingTypesSet); +} + +export function indexMapToIndexTypesMap(indexMap: IndexMap): IndexTypesMap { + return Object.entries(indexMap).reduce((acc, [indexAlias, { typeMappings }]) => { + acc[indexAlias] = Object.keys(typeMappings); + return acc; + }, {}); +} + +export function calculateTypeStatuses( + currentIndexTypesMap: IndexTypesMap, + desiredIndexTypesMap: IndexTypesMap +): Record { + const statuses: Record = {}; + + Object.entries(currentIndexTypesMap).forEach(([currentIndex, types]) => { + types.forEach((type) => { + statuses[type] = { + currentIndex, + status: TypeStatus.Removed, // type is removed unless we still have it + }; + }); + }); + + Object.entries(desiredIndexTypesMap).forEach(([targetIndex, types]) => { + types.forEach((type) => { + if (!statuses[type]) { + statuses[type] = { + targetIndex, + status: TypeStatus.Added, // type didn't exist, it must be new + }; + } else { + statuses[type].targetIndex = targetIndex; + statuses[type].status = + statuses[type].currentIndex === targetIndex ? TypeStatus.Untouched : TypeStatus.Moved; + } + }); + }); + + return statuses; +} diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts index d03b4f7378da4..15a5ef1ce14d7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts @@ -6,13 +6,11 @@ * Side Public License, v 1. */ -import { cleanupMock } from './migrations_state_machine_cleanup.mocks'; import { migrationStateActionMachine } from './migrations_state_action_machine'; import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { LoggerAdapter } from '@kbn/core-logging-server-internal'; import { typeRegistryMock } from '@kbn/core-saved-objects-base-server-mocks'; -import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import * as Either from 'fp-ts/lib/Either'; import * as Option from 'fp-ts/lib/Option'; import { errors } from '@elastic/elasticsearch'; @@ -21,8 +19,6 @@ import type { AllControlStates, State } from './state'; import { createInitialState } from './initial_state'; import { ByteSizeValue } from '@kbn/config-schema'; -const esClient = elasticsearchServiceMock.createElasticsearchClient(); - describe('migrationsStateActionMachine', () => { beforeAll(() => { jest @@ -33,6 +29,7 @@ describe('migrationsStateActionMachine', () => { jest.clearAllMocks(); }); + const abort = jest.fn(); const mockLogger = loggingSystemMock.create(); const typeRegistry = typeRegistryMock.create(); const docLinks = docLinksServiceMock.createSetupContract(); @@ -40,6 +37,12 @@ describe('migrationsStateActionMachine', () => { const initialState = createInitialState({ kibanaVersion: '7.11.0', waitForMigrationCompletion: false, + mustRelocateDocuments: true, + indexTypesMap: { + '.kibana': ['typeA', 'typeB', 'typeC'], + '.kibana_task_manager': ['task'], + '.kibana_cases': ['typeD', 'typeE'], + }, targetMappings: { properties: {} }, migrationVersionPerType: {}, indexPrefix: '.my-so-index', @@ -92,7 +95,7 @@ describe('migrationsStateActionMachine', () => { logger: mockLogger.get(), model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']), next, - client: esClient, + abort, }); const logs = loggingSystemMock.collect(mockLogger); const doneLog = logs.info.splice(8, 1)[0][0]; @@ -113,7 +116,7 @@ describe('migrationsStateActionMachine', () => { logger: mockLogger.get(), model: transitionModel(['LEGACY_DELETE', 'FATAL']), next, - client: esClient, + abort, }).catch((err) => err); expect(loggingSystemMock.collect(mockLogger)).toMatchSnapshot(); }); @@ -129,7 +132,7 @@ describe('migrationsStateActionMachine', () => { logger, model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']), next, - client: esClient, + abort, }) ).resolves.toEqual(expect.anything()); @@ -155,7 +158,7 @@ describe('migrationsStateActionMachine', () => { logger: mockLogger.get(), model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']), next, - client: esClient, + abort, }) ).resolves.toEqual(expect.anything()); }); @@ -167,7 +170,7 @@ describe('migrationsStateActionMachine', () => { logger: mockLogger.get(), model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']), next, - client: esClient, + abort, }) ).resolves.toEqual(expect.objectContaining({ status: 'migrated' })); }); @@ -179,7 +182,7 @@ describe('migrationsStateActionMachine', () => { logger: mockLogger.get(), model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'LEGACY_DELETE', 'DONE']), next, - client: esClient, + abort, }) ).resolves.toEqual(expect.objectContaining({ status: 'patched' })); }); @@ -191,7 +194,7 @@ describe('migrationsStateActionMachine', () => { logger: mockLogger.get(), model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']), next, - client: esClient, + abort, }) ).rejects.toMatchInlineSnapshot( `[Error: Unable to complete saved object migrations for the [.my-so-index] index: the fatal reason]` @@ -219,7 +222,7 @@ describe('migrationsStateActionMachine', () => { }) ); }, - client: esClient, + abort, }) ).rejects.toMatchInlineSnapshot( `[Error: Unable to complete saved object migrations for the [.my-so-index] index. Please check the health of your Elasticsearch cluster and try again. Unexpected Elasticsearch ResponseError: statusCode: 200, method: POST, url: /mock error: [snapshot_in_progress_exception]: Cannot delete indices that are being snapshotted,]` @@ -249,7 +252,7 @@ describe('migrationsStateActionMachine', () => { next: () => { throw new Error('this action throws'); }, - client: esClient, + abort, }) ).rejects.toMatchInlineSnapshot( `[Error: Unable to complete saved object migrations for the [.my-so-index] index. Error: this action throws]` @@ -271,10 +274,7 @@ describe('migrationsStateActionMachine', () => { `); }); describe('cleanup', () => { - beforeEach(() => { - cleanupMock.mockClear(); - }); - it('calls cleanup function when an action throws', async () => { + it('calls abort function when an action throws', async () => { await expect( migrationStateActionMachine({ initialState: { ...initialState, reason: 'the fatal reason' } as State, @@ -283,24 +283,24 @@ describe('migrationsStateActionMachine', () => { next: () => { throw new Error('this action throws'); }, - client: esClient, + abort, }) ).rejects.toThrow(); - expect(cleanupMock).toHaveBeenCalledTimes(1); + expect(abort).toHaveBeenCalledTimes(1); }); - it('calls cleanup function when reaching the FATAL state', async () => { + it('calls abort function when reaching the FATAL state', async () => { await expect( migrationStateActionMachine({ initialState: { ...initialState, reason: 'the fatal reason' } as State, logger: mockLogger.get(), model: transitionModel(['LEGACY_REINDEX', 'LEGACY_DELETE', 'FATAL']), next, - client: esClient, + abort, }) ).rejects.toThrow(); - expect(cleanupMock).toHaveBeenCalledTimes(1); + expect(abort).toHaveBeenCalledTimes(1); }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts index 92b0054bd47c3..5bd76c80797e4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts @@ -9,15 +9,14 @@ import { errors as EsErrors } from '@elastic/elasticsearch'; import * as Option from 'fp-ts/lib/Option'; import type { Logger } from '@kbn/logging'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { getErrorMessage, getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; +import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; import { logActionResponse, logStateTransition } from './common/utils/logs'; import { type Model, type Next, stateActionMachine } from './state_action_machine'; -import { cleanup } from './migrations_state_machine_cleanup'; import type { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './state'; import { redactBulkOperationBatches } from './common/redact_state'; @@ -35,14 +34,14 @@ export async function migrationStateActionMachine({ logger, next, model, - client, + abort, }: { initialState: State; logger: Logger; next: Next; model: Model; - client: ElasticsearchClient; -}) { + abort: (state?: State) => Promise; +}): Promise { const startTime = Date.now(); // Since saved object index names usually start with a `.` and can be // configured by users to include several `.`'s we can't use a logger tag to @@ -112,24 +111,30 @@ export async function migrationStateActionMachine({ } } else if (finalState.controlState === 'FATAL') { try { - await cleanup(client, finalState); + await abort(finalState); } catch (e) { - logger.warn('Failed to cleanup after migrations:', e.message); + logger.warn('Failed to correctly abort migrations:', e.message); } - return Promise.reject( - new Error( - `Unable to complete saved object migrations for the [${initialState.indexPrefix}] index: ` + - finalState.reason - ) - ); + + const errorMessage = + `Unable to complete saved object migrations for the [${initialState.indexPrefix}] index: ` + + finalState.reason; + + if (finalState.throwDelayMillis) { + return new Promise((_, reject) => + setTimeout(() => reject(errorMessage), finalState.throwDelayMillis) + ); + } + + return Promise.reject(new Error(errorMessage)); } else { throw new Error('Invalid terminating control state'); } } catch (e) { try { - await cleanup(client, lastState); + await abort(lastState); } catch (err) { - logger.warn('Failed to cleanup after migrations:', err.message); + logger.warn('Failed to correctly abort migrations:', err.message); } if (e instanceof EsErrors.ResponseError) { // Log the failed request. This is very similar to the diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts index 3ae3b1c7f20d8..951ee86d92f0d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.test.ts @@ -7,7 +7,7 @@ */ import * as Either from 'fp-ts/lib/Either'; import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; -import { createBatches } from './create_batches'; +import { buildTempIndexMap, createBatches } from './create_batches'; describe('createBatches', () => { const documentToOperation = (document: SavedObjectsRawDoc) => [ @@ -17,56 +17,145 @@ describe('createBatches', () => { const DOCUMENT_SIZE_BYTES = 77; // 76 + \n - it('returns right one batch if all documents fit in maxBatchSizeBytes', () => { - const documents = [ - { _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, - { _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } }, - { _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } }, - ]; + describe('when indexTypesMap and kibanaVersion are not provided', () => { + it('returns right one batch if all documents fit in maxBatchSizeBytes', () => { + const documents = [ + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } }, + ]; - expect(createBatches({ documents, maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 3 })).toEqual( - Either.right([documents.map(documentToOperation)]) - ); - }); - it('creates multiple batches with each batch limited to maxBatchSizeBytes', () => { - const documents = [ - { _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, - { _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } }, - { _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } }, - { _id: '', _source: { type: 'dashboard', title: 'my saved object title 44' } }, - { _id: '', _source: { type: 'dashboard', title: 'my saved object title 55' } }, - ]; - expect(createBatches({ documents, maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 2 })).toEqual( - Either.right([ - documents.slice(0, 2).map(documentToOperation), - documents.slice(2, 4).map(documentToOperation), - documents.slice(4).map(documentToOperation), - ]) - ); - }); - it('creates a single empty batch if there are no documents', () => { - const documents = [] as SavedObjectsRawDoc[]; - expect(createBatches({ documents, maxBatchSizeBytes: 100 })).toEqual(Either.right([[]])); - }); - it('throws if any one document exceeds the maxBatchSizeBytes', () => { - const documents = [ - { _id: 'foo', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, - { - _id: 'bar', - _source: { - type: 'dashboard', - title: 'my saved object title ² with a very long title that exceeds max size bytes', + expect( + createBatches({ + documents, + maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 3, + }) + ).toEqual(Either.right([documents.map(documentToOperation)])); + }); + it('creates multiple batches with each batch limited to maxBatchSizeBytes', () => { + const documents = [ + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ®' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title 44' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title 55' } }, + ]; + expect( + createBatches({ + documents, + maxBatchSizeBytes: DOCUMENT_SIZE_BYTES * 2, + }) + ).toEqual( + Either.right([ + documents.slice(0, 2).map(documentToOperation), + documents.slice(2, 4).map(documentToOperation), + documents.slice(4).map(documentToOperation), + ]) + ); + }); + it('creates a single empty batch if there are no documents', () => { + const documents = [] as SavedObjectsRawDoc[]; + expect(createBatches({ documents, maxBatchSizeBytes: 100 })).toEqual(Either.right([[]])); + }); + it('throws if any one document exceeds the maxBatchSizeBytes', () => { + const documents = [ + { _id: 'foo', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, + { + _id: 'bar', + _source: { + type: 'dashboard', + title: 'my saved object title ² with a very long title that exceeds max size bytes', + }, }, - }, - { _id: 'baz', _source: { type: 'dashboard', title: 'my saved object title ®' } }, - ]; - expect(createBatches({ documents, maxBatchSizeBytes: 120 })).toEqual( - Either.left({ - maxBatchSizeBytes: 120, - docSizeBytes: 130, - type: 'document_exceeds_batch_size_bytes', - documentId: documents[1]._id, - }) - ); + { _id: 'baz', _source: { type: 'dashboard', title: 'my saved object title ®' } }, + ]; + expect(createBatches({ documents, maxBatchSizeBytes: 120 })).toEqual( + Either.left({ + maxBatchSizeBytes: 120, + docSizeBytes: 130, + type: 'document_exceeds_batch_size_bytes', + documentId: documents[1]._id, + }) + ); + }); + }); + + describe('when a type index map is provided', () => { + it('creates batches that contain the target index information for each type', () => { + const documents = [ + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ¹' } }, + { _id: '', _source: { type: 'dashboard', title: 'my saved object title ²' } }, + { _id: '', _source: { type: 'cases', title: 'a case' } }, + { _id: '', _source: { type: 'cases-comments', title: 'a case comment #1' } }, + { _id: '', _source: { type: 'cases-user-actions', title: 'a case user action' } }, + ]; + expect( + createBatches({ + documents, + maxBatchSizeBytes: (DOCUMENT_SIZE_BYTES + 43) * 2, // add extra length for 'index' property + typeIndexMap: buildTempIndexMap( + { + '.kibana': ['dashboard'], + '.kibana_cases': ['cases', 'cases-comments', 'cases-user-actions'], + }, + '8.8.0' + ), + }) + ).toEqual( + Either.right([ + [ + [ + { + index: { + _id: '', + _index: '.kibana_8.8.0_reindex_temp', + }, + }, + { type: 'dashboard', title: 'my saved object title ¹' }, + ], + [ + { + index: { + _id: '', + _index: '.kibana_8.8.0_reindex_temp', + }, + }, + { type: 'dashboard', title: 'my saved object title ²' }, + ], + ], + [ + [ + { + index: { + _id: '', + _index: '.kibana_cases_8.8.0_reindex_temp', + }, + }, + { type: 'cases', title: 'a case' }, + ], + [ + { + index: { + _id: '', + _index: '.kibana_cases_8.8.0_reindex_temp', + }, + }, + { type: 'cases-comments', title: 'a case comment #1' }, + ], + ], + [ + [ + { + index: { + _id: '', + _index: '.kibana_cases_8.8.0_reindex_temp', + }, + }, + { type: 'cases-user-actions', title: 'a case user action' }, + ], + ], + ]) + ); + }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts index 008d074b2cd6f..dc5b09d6c3379 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts @@ -9,7 +9,12 @@ import * as Either from 'fp-ts/lib/Either'; import type { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '@kbn/core-saved-objects-server'; import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; -import { createBulkDeleteOperationBody, createBulkIndexOperationTuple } from './helpers'; +import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; +import { + createBulkDeleteOperationBody, + createBulkIndexOperationTuple, + getTempIndexName, +} from './helpers'; import type { TransformErrorObjects } from '../core'; export type BulkIndexOperationTuple = [BulkOperationContainer, SavedObjectsRawDocSource]; @@ -21,6 +26,12 @@ export interface CreateBatchesParams { corruptDocumentIds?: string[]; transformErrors?: TransformErrorObjects[]; maxBatchSizeBytes: number; + /** This map holds a list of temporary index names for each SO type, e.g.: + * 'cases': '.kibana_cases_8.8.0_reindex_temp' + * 'task': '.kibana_task_manager_8.8.0_reindex_temp' + * ... + */ + typeIndexMap?: Record; } export interface DocumentExceedsBatchSize { @@ -30,6 +41,32 @@ export interface DocumentExceedsBatchSize { maxBatchSizeBytes: number; } +/** + * Build a relationship of temporary index names for each SO type, e.g.: + * 'cases': '.kibana_cases_8.8.0_reindex_temp' + * 'task': '.kibana_task_manager_8.8.0_reindex_temp' + * ... + * + * @param indexTypesMap information about which types are stored in each index + * @param kibanaVersion the target version of the indices + */ +export function buildTempIndexMap( + indexTypesMap: IndexTypesMap, + kibanaVersion: string +): Record { + return Object.entries(indexTypesMap || {}).reduce>( + (acc, [indexAlias, types]) => { + const tempIndex = getTempIndexName(indexAlias, kibanaVersion!); + + types.forEach((type) => { + acc[type] = tempIndex; + }); + return acc; + }, + {} + ); +} + /** * Creates batches of documents to be used by the bulk API. Each batch will * have a request body content length that's <= maxBatchSizeBytes @@ -39,6 +76,7 @@ export function createBatches({ corruptDocumentIds = [], transformErrors = [], maxBatchSizeBytes, + typeIndexMap, }: CreateBatchesParams): Either.Either { /* To build up the NDJSON request body we construct an array of objects like: * [ @@ -92,7 +130,7 @@ export function createBatches({ // create index (update) operations for all transformed documents for (const document of documents) { - const bulkIndexOperationBody = createBulkIndexOperationTuple(document); + const bulkIndexOperationBody = createBulkIndexOperationTuple(document, typeIndexMap); // take into account that this tuple's surrounding brackets `[]` won't be present in the NDJSON const docSizeBytes = Buffer.byteLength(JSON.stringify(bulkIndexOperationBody), 'utf8') - BRACKETS_BYTES; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts index 291f865f8c100..a20b517b7f3ba 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.test.ts @@ -16,6 +16,8 @@ import { buildRemoveAliasActions, versionMigrationCompleted, MigrationType, + getTempIndexName, + createBulkIndexOperationTuple, } from './helpers'; describe('addExcludedTypesToBoolQuery', () => { @@ -290,6 +292,46 @@ describe('buildRemoveAliasActions', () => { }); }); +describe('createBulkIndexOperationTuple', () => { + it('creates the proper request body to bulk index a document', () => { + const document = { _id: '', _source: { type: 'cases', title: 'a case' } }; + const typeIndexMap = { + cases: '.kibana_cases_8.8.0_reindex_temp', + }; + expect(createBulkIndexOperationTuple(document, typeIndexMap)).toMatchInlineSnapshot(` + Array [ + Object { + "index": Object { + "_id": "", + "_index": ".kibana_cases_8.8.0_reindex_temp", + }, + }, + Object { + "title": "a case", + "type": "cases", + }, + ] + `); + }); + + it('does not include the index property if it is not specified in the typeIndexMap', () => { + const document = { _id: '', _source: { type: 'cases', title: 'a case' } }; + expect(createBulkIndexOperationTuple(document)).toMatchInlineSnapshot(` + Array [ + Object { + "index": Object { + "_id": "", + }, + }, + Object { + "title": "a case", + "type": "cases", + }, + ] + `); + }); +}); + describe('getMigrationType', () => { it.each` isMappingsCompatible | isVersionMigrationCompleted | expected @@ -306,3 +348,9 @@ describe('getMigrationType', () => { } ); }); + +describe('getTempIndexName', () => { + it('composes a temporary index name for reindexing', () => { + expect(getTempIndexName('.kibana_cases', '8.8.0')).toEqual('.kibana_cases_8.8.0_reindex_temp'); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts index 9ffdf8508c8f5..93748800fae04 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts @@ -218,11 +218,15 @@ export function buildRemoveAliasActions( /** * Given a document, creates a valid body to index the document using the Bulk API. */ -export const createBulkIndexOperationTuple = (doc: SavedObjectsRawDoc): BulkIndexOperationTuple => { +export const createBulkIndexOperationTuple = ( + doc: SavedObjectsRawDoc, + typeIndexMap: Record = {} +): BulkIndexOperationTuple => { return [ { index: { _id: doc._id, + ...(typeIndexMap[doc._source.type] && { _index: typeIndexMap[doc._source.type] }), // use optimistic concurrency control to ensure that outdated // documents are only overwritten once with the latest version ...(typeof doc._seq_no !== 'undefined' && { if_seq_no: doc._seq_no }), @@ -271,3 +275,12 @@ export function getMigrationType({ return MigrationType.Invalid; } + +/** + * Generate a temporary index name, to reindex documents into it + * @param index The name of the SO index + * @param kibanaVersion The current kibana version + * @returns A temporary index name to reindex documents + */ +export const getTempIndexName = (indexPrefix: string, kibanaVersion: string): string => + `${indexPrefix}_${kibanaVersion}_reindex_temp`; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts index deba537b3f81e..969a7704e4e75 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.test.ts @@ -20,7 +20,7 @@ import type { CheckVersionIndexReadyActions, CleanupUnknownAndExcluded, CleanupUnknownAndExcludedWaitForTaskState, - CloneTempToSource, + CloneTempToTarget, CreateNewTargetState, CreateReindexTempState, FatalState, @@ -51,6 +51,8 @@ import type { UpdateTargetMappingsPropertiesState, UpdateTargetMappingsPropertiesWaitForTaskState, WaitForYellowSourceState, + ReadyToReindexSyncState, + DoneReindexingSyncState, } from '../state'; import { type TransformErrorObjects, TransformSavedObjectDocumentError } from '../core'; import type { AliasAction, RetryableEsClientError } from '../actions'; @@ -58,6 +60,7 @@ import type { ResponseType } from '../next'; import { createInitialProgress } from './progress'; import { model } from './model'; import type { BulkIndexOperationTuple, BulkOperation } from './create_batches'; +import { DEFAULT_INDEX_TYPES_MAP } from '../kibana_migrator_constants'; describe('migrations v2 model', () => { const indexMapping: IndexMapping = { @@ -115,6 +118,8 @@ describe('migrations v2 model', () => { clusterShardLimitExceeded: 'clusterShardLimitExceeded', }, waitForMigrationCompletion: false, + mustRelocateDocuments: false, + indexTypesMap: DEFAULT_INDEX_TYPES_MAP, }; const postInitState = { ...baseState, @@ -732,7 +737,7 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); - test('INIT -> CREATE_NEW_TARGET when no indices/aliases exist', () => { + test('INIT -> CREATE_NEW_TARGET when the index does not exist and the migrator is NOT involved in a relocation', () => { const res: ResponseType<'INIT'> = Either.right({}); const newState = model(initState, res); @@ -744,6 +749,29 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + test('INIT -> CREATE_REINDEX_TEMP when the index does not exist and the migrator is involved in a relocation', () => { + const res: ResponseType<'INIT'> = Either.right({}); + const newState = model( + { + ...initState, + mustRelocateDocuments: true, + }, + res + ); + + expect(newState).toMatchObject({ + controlState: 'CREATE_REINDEX_TEMP', + sourceIndex: Option.none, + targetIndex: '.kibana_7.11.0_001', + versionIndexReadyActions: Option.some([ + { add: { index: '.kibana_7.11.0_001', alias: '.kibana' } }, + { add: { index: '.kibana_7.11.0_001', alias: '.kibana_7.11.0' } }, + { remove_index: { index: '.kibana_7.11.0_reindex_temp' } }, + ]), + }); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); }); }); @@ -1146,12 +1174,29 @@ describe('migrations v2 model', () => { expect(newState.retryDelay).toEqual(0); }); - test('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES', () => { - const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); - const newState = model(waitForYellowSourceState, res); + describe('if the migrator is NOT involved in a relocation', () => { + test('WAIT_FOR_YELLOW_SOURCE -> UPDATE_SOURCE_MAPPINGS_PROPERTIES', () => { + const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); + const newState = model(waitForYellowSourceState, res); - expect(newState).toMatchObject({ - controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES', + expect(newState).toMatchObject({ + controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES', + }); + }); + }); + + describe('if the migrator is involved in a relocation', () => { + // no need to attempt to update the mappings, we are going to reindex + test('WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS', () => { + const res: ResponseType<'WAIT_FOR_YELLOW_SOURCE'> = Either.right({}); + const newState = model( + { ...waitForYellowSourceState, mustRelocateDocuments: true }, + res + ); + + expect(newState).toMatchObject({ + controlState: 'CHECK_UNKNOWN_DOCUMENTS', + }); }); }); }); @@ -1630,13 +1675,27 @@ describe('migrations v2 model', () => { sourceIndexMappings: Option.some({}) as Option.Some, tempIndexMappings: { properties: {} }, }; - it('CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT if action succeeds', () => { - const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded'); - const newState = model(state, res); - expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_OPEN_PIT'); - expect(newState.retryCount).toEqual(0); - expect(newState.retryDelay).toEqual(0); + + describe('if the migrator is NOT involved in a relocation', () => { + it('CREATE_REINDEX_TEMP -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT if action succeeds', () => { + const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded'); + const newState = model(state, res); + expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_OPEN_PIT'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); + }); + + describe('if the migrator is involved in a relocation', () => { + it('CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC if action succeeds', () => { + const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.right('create_index_succeeded'); + const newState = model({ ...state, mustRelocateDocuments: true }, res); + expect(newState.controlState).toEqual('READY_TO_REINDEX_SYNC'); + expect(newState.retryCount).toEqual(0); + expect(newState.retryDelay).toEqual(0); + }); }); + it('CREATE_REINDEX_TEMP -> CREATE_REINDEX_TEMP if action fails with index_not_green_timeout', () => { const res: ResponseType<'CREATE_REINDEX_TEMP'> = Either.left({ message: '[index_not_green_timeout] Timeout waiting for ...', @@ -1677,6 +1736,52 @@ describe('migrations v2 model', () => { }); }); + describe('READY_TO_REINDEX_SYNC', () => { + const state: ReadyToReindexSyncState = { + ...postInitState, + controlState: 'READY_TO_REINDEX_SYNC', + }; + + describe('if the migrator source index did NOT exist', () => { + test('READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC', () => { + const res: ResponseType<'READY_TO_REINDEX_SYNC'> = Either.right( + 'synchronized_successfully' as const + ); + const newState = model(state, res); + expect(newState.controlState).toEqual('DONE_REINDEXING_SYNC'); + }); + }); + + describe('if the migrator source index did exist', () => { + test('READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT', () => { + const res: ResponseType<'READY_TO_REINDEX_SYNC'> = Either.right( + 'synchronized_successfully' as const + ); + const newState = model( + { + ...state, + sourceIndex: Option.fromNullable('.kibana'), + sourceIndexMappings: Option.fromNullable({} as IndexMapping), + }, + res + ); + expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_OPEN_PIT'); + }); + }); + + test('READY_TO_REINDEX_SYNC -> FATAL if the synchronization between migrators fails', () => { + const res: ResponseType<'READY_TO_REINDEX_SYNC'> = Either.left({ + type: 'sync_failed', + error: new Error('Other migrators failed to reach the synchronization point'), + }); + const newState = model(state, res); + expect(newState.controlState).toEqual('FATAL'); + expect((newState as FatalState).reason).toMatchInlineSnapshot( + `"An error occurred whilst waiting for other migrators to get to this step."` + ); + }); + }); + describe('REINDEX_SOURCE_TO_TEMP_OPEN_PIT', () => { const state: ReindexSourceToTempOpenPit = { ...postInitState, @@ -1812,11 +1917,50 @@ describe('migrations v2 model', () => { tempIndexMappings: { properties: {} }, }; - it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK if action succeeded', () => { - const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({}); - const newState = model(state, res) as ReindexSourceToTempTransform; - expect(newState.controlState).toBe('SET_TEMP_WRITE_BLOCK'); - expect(newState.sourceIndex).toEqual(state.sourceIndex); + describe('if the migrator is NOT involved in a relocation', () => { + it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> SET_TEMP_WRITE_BLOCK if action succeeded', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({}); + const newState = model(state, res) as ReindexSourceToTempTransform; + expect(newState.controlState).toBe('SET_TEMP_WRITE_BLOCK'); + expect(newState.sourceIndex).toEqual(state.sourceIndex); + }); + }); + + describe('if the migrator is involved in a relocation', () => { + it('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC if action succeeded', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'> = Either.right({}); + const newState = model( + { ...state, mustRelocateDocuments: true }, + res + ) as ReindexSourceToTempTransform; + expect(newState.controlState).toBe('DONE_REINDEXING_SYNC'); + }); + }); + }); + + describe('DONE_REINDEXING_SYNC', () => { + const state: DoneReindexingSyncState = { + ...postInitState, + controlState: 'DONE_REINDEXING_SYNC', + }; + + test('DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK if synchronization succeeds', () => { + const res: ResponseType<'DONE_REINDEXING_SYNC'> = Either.right( + 'synchronized_successfully' as const + ); + const newState = model(state, res); + expect(newState.controlState).toEqual('SET_TEMP_WRITE_BLOCK'); + }); + test('DONE_REINDEXING_SYNC -> FATAL if the synchronization between migrators fails', () => { + const res: ResponseType<'DONE_REINDEXING_SYNC'> = Either.left({ + type: 'sync_failed', + error: new Error('Other migrators failed to reach the synchronization point'), + }); + const newState = model(state, res); + expect(newState.controlState).toEqual('FATAL'); + expect((newState as FatalState).reason).toMatchInlineSnapshot( + `"An error occurred whilst waiting for other migrators to get to this step."` + ); }); }); @@ -1977,7 +2121,7 @@ describe('migrations v2 model', () => { }); describe('CLONE_TEMP_TO_TARGET', () => { - const state: CloneTempToSource = { + const state: CloneTempToTarget = { ...postInitState, controlState: 'CLONE_TEMP_TO_TARGET', sourceIndex: Option.some('.kibana') as Option.Some, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 23ddee5043261..f895ef7406e2e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -44,7 +44,7 @@ import { buildRemoveAliasActions, MigrationType, } from './helpers'; -import { createBatches } from './create_batches'; +import { buildTempIndexMap, createBatches } from './create_batches'; import type { MigrationLog } from '../types'; import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON, @@ -121,6 +121,8 @@ export const model = (currentState: State, resW: ResponseType): // The source index .kibana is pointing to. E.g: ".kibana_8.7.0_001" const source = aliases[stateP.currentAlias]; + // The target index .kibana WILL be pointing to if we reindex. E.g: ".kibana_8.8.0_001" + const newVersionTarget = stateP.versionIndex; const postInitState = { aliases, @@ -137,7 +139,7 @@ export const model = (currentState: State, resW: ResponseType): ...stateP, ...postInitState, sourceIndex: Option.none, - targetIndex: `${stateP.indexPrefix}_${stateP.kibanaVersion}_001`, + targetIndex: newVersionTarget, controlState: 'WAIT_FOR_MIGRATION_COMPLETION', // Wait for 2s before checking again if the migration has completed retryDelay: 2000, @@ -153,7 +155,6 @@ export const model = (currentState: State, resW: ResponseType): // If the `.kibana` alias exists Option.isSome(postInitState.sourceIndex) ) { - // CHECKPOINT here we decide to go for yellow source return { ...stateP, ...postInitState, @@ -182,7 +183,6 @@ export const model = (currentState: State, resW: ResponseType): const legacyReindexTarget = `${stateP.indexPrefix}_${legacyVersion}_001`; - const target = stateP.versionIndex; return { ...stateP, ...postInitState, @@ -191,7 +191,7 @@ export const model = (currentState: State, resW: ResponseType): sourceIndexMappings: Option.some( indices[stateP.legacyIndex].mappings ) as Option.Some, - targetIndex: target, + targetIndex: newVersionTarget, legacyPreMigrationDoneActions: [ { remove_index: { index: stateP.legacyIndex } }, { @@ -209,24 +209,40 @@ export const model = (currentState: State, resW: ResponseType): must_exist: true, }, }, - { add: { index: target, alias: stateP.currentAlias } }, - { add: { index: target, alias: stateP.versionAlias } }, + { add: { index: newVersionTarget, alias: stateP.currentAlias } }, + { add: { index: newVersionTarget, alias: stateP.versionAlias } }, + { remove_index: { index: stateP.tempIndex } }, + ]), + }; + } else if ( + // if we must relocate documents to this migrator's index, but the index does NOT yet exist: + // this migrator must create a temporary index and synchronize with other migrators + // this is a similar flow to the reindex one, but this migrator will not reindexing anything + stateP.mustRelocateDocuments + ) { + return { + ...stateP, + ...postInitState, + controlState: 'CREATE_REINDEX_TEMP', + sourceIndex: Option.none as Option.None, + targetIndex: newVersionTarget, + versionIndexReadyActions: Option.some([ + { add: { index: newVersionTarget, alias: stateP.currentAlias } }, + { add: { index: newVersionTarget, alias: stateP.versionAlias } }, { remove_index: { index: stateP.tempIndex } }, ]), }; } else { - // This cluster doesn't have an existing Saved Object index, create a - // new version specific index. - const target = stateP.versionIndex; + // no need to copy anything over from other indices, we can start with a clean, empty index return { ...stateP, ...postInitState, controlState: 'CREATE_NEW_TARGET', sourceIndex: Option.none as Option.None, - targetIndex: target, + targetIndex: newVersionTarget, versionIndexReadyActions: Option.some([ - { add: { index: target, alias: stateP.currentAlias } }, - { add: { index: target, alias: stateP.versionAlias } }, + { add: { index: newVersionTarget, alias: stateP.currentAlias } }, + { add: { index: newVersionTarget, alias: stateP.versionAlias } }, ]) as Option.Some, }; } @@ -240,6 +256,7 @@ export const model = (currentState: State, resW: ResponseType): if ( // If this version's migration has already been completed we can proceed Either.isRight(aliasesRes) && + // TODO check that this behaves correctly when skipping reindexing versionMigrationCompleted(stateP.currentAlias, stateP.versionAlias, aliasesRes.right) ) { return { @@ -414,10 +431,21 @@ export const model = (currentState: State, resW: ResponseType): } else if (stateP.controlState === 'WAIT_FOR_YELLOW_SOURCE') { const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { - return { - ...stateP, - controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES', - }; + if (stateP.mustRelocateDocuments) { + // if this migrator's index must dispatch documents to other indices, + // or it must receive documents from other indices + // we must reindex and synchronize with other migrators + return { + ...stateP, + controlState: 'CHECK_UNKNOWN_DOCUMENTS', + }; + } else { + // we + return { + ...stateP, + controlState: 'UPDATE_SOURCE_MAPPINGS_PROPERTIES', + }; + } } else if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'index_not_yellow_timeout')) { @@ -711,7 +739,18 @@ export const model = (currentState: State, resW: ResponseType): } else if (stateP.controlState === 'CREATE_REINDEX_TEMP') { const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { - return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT' }; + if (stateP.mustRelocateDocuments) { + // we are reindexing, and this migrator's index is involved in document relocations + return { ...stateP, controlState: 'READY_TO_REINDEX_SYNC' }; + } else { + // we are reindexing but this migrator's index is not involved in any document relocation + return { + ...stateP, + controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT', + sourceIndex: stateP.sourceIndex as Option.Some, + sourceIndexMappings: stateP.sourceIndexMappings as Option.Some, + }; + } } else if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'index_not_green_timeout')) { @@ -738,6 +777,32 @@ export const model = (currentState: State, resW: ResponseType): // left responses to handle here. throwBadResponse(stateP, res); } + } else if (stateP.controlState === 'READY_TO_REINDEX_SYNC') { + const res = resW as ExcludeRetryableEsError>; + if (Either.isRight(res)) { + if (Option.isSome(stateP.sourceIndex) && Option.isSome(stateP.sourceIndexMappings)) { + // this migrator's source index exist, reindex its entries + return { + ...stateP, + controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT', + sourceIndex: stateP.sourceIndex as Option.Some, + sourceIndexMappings: stateP.sourceIndexMappings as Option.Some, + }; + } else { + // this migrator's source index did NOT exist + // this migrator does not need to reindex anything (others might need to) + return { ...stateP, controlState: 'DONE_REINDEXING_SYNC' }; + } + } else if (Either.isLeft(res)) { + return { + ...stateP, + controlState: 'FATAL', + reason: 'An error occurred whilst waiting for other migrators to get to this step.', + throwDelayMillis: 1000, // another migrator has failed for a reason, let it take Kibana down and log its problem + }; + } else { + return throwBadResponse(stateP, res as never); + } } else if (stateP.controlState === 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT') { const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { @@ -816,14 +881,41 @@ export const model = (currentState: State, resW: ResponseType): const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { const { sourceIndexPitId, ...state } = stateP; + + if (stateP.mustRelocateDocuments) { + return { + ...state, + controlState: 'DONE_REINDEXING_SYNC', + }; + } else { + return { + ...stateP, + controlState: 'SET_TEMP_WRITE_BLOCK', + sourceIndex: stateP.sourceIndex as Option.Some, + sourceIndexMappings: Option.none, + }; + } + } else { + throwBadResponse(stateP, res); + } + } else if (stateP.controlState === 'DONE_REINDEXING_SYNC') { + const res = resW as ExcludeRetryableEsError>; + if (Either.isRight(res)) { return { - ...state, + ...stateP, controlState: 'SET_TEMP_WRITE_BLOCK', sourceIndex: stateP.sourceIndex as Option.Some, sourceIndexMappings: Option.none, }; + } else if (Either.isLeft(res)) { + return { + ...stateP, + controlState: 'FATAL', + reason: 'An error occurred whilst waiting for other migrators to get to this step.', + throwDelayMillis: 1000, // another migrator has failed for a reason, let it take Kibana down and log its problem + }; } else { - throwBadResponse(stateP, res); + return throwBadResponse(stateP, res as never); } } else if (stateP.controlState === 'REINDEX_SOURCE_TO_TEMP_TRANSFORM') { // We follow a similar control flow as for @@ -845,7 +937,11 @@ export const model = (currentState: State, resW: ResponseType): stateP.discardCorruptObjects ) { const documents = Either.isRight(res) ? res.right.processedDocs : res.left.processedDocs; - const batches = createBatches({ documents, maxBatchSizeBytes: stateP.maxBatchSizeBytes }); + const batches = createBatches({ + documents, + maxBatchSizeBytes: stateP.maxBatchSizeBytes, + typeIndexMap: buildTempIndexMap(stateP.indexTypesMap, stateP.kibanaVersion), + }); if (Either.isRight(batches)) { let corruptDocumentIds = stateP.corruptDocumentIds; let transformErrors = stateP.transformErrors; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.test.ts index cc4ee13673940..f7cabfb6e42db 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.test.ts @@ -7,6 +7,7 @@ */ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { defer } from './kibana_migrator_utils'; import { next } from './next'; import type { State } from './state'; @@ -14,12 +15,12 @@ describe('migrations v2 next', () => { it.todo('when state.retryDelay > 0 delays execution of the next action'); it('DONE returns null', () => { const state = { controlState: 'DONE' } as State; - const action = next({} as ElasticsearchClient, (() => {}) as any)(state); + const action = next({} as ElasticsearchClient, (() => {}) as any, defer(), defer())(state); expect(action).toEqual(null); }); it('FATAL returns null', () => { const state = { controlState: 'FATAL', reason: '' } as State; - const action = next({} as ElasticsearchClient, (() => {}) as any)(state); + const action = next({} as ElasticsearchClient, (() => {}) as any, defer(), defer())(state); expect(action).toEqual(null); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts index cc26bb543ef5e..426f73436cb71 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/next.ts @@ -7,8 +7,9 @@ */ import * as Option from 'fp-ts/lib/Option'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { omit } from 'lodash'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { Defer } from './kibana_migrator_utils'; import type { AllActionStates, CalculateExcludeFiltersState, @@ -16,7 +17,7 @@ import type { CheckUnknownDocumentsState, CleanupUnknownAndExcluded, CleanupUnknownAndExcludedWaitForTaskState, - CloneTempToSource, + CloneTempToTarget, CreateNewTargetState, CreateReindexTempState, InitState, @@ -68,7 +69,12 @@ export type ResponseType = Awaited< ReturnType> >; -export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: TransformRawDocs) => { +export const nextActionMap = ( + client: ElasticsearchClient, + transformRawDocs: TransformRawDocs, + readyToReindex: Defer, + doneReindexing: Defer +) => { return { INIT: (state: InitState) => Actions.initAction({ client, indices: [state.currentAlias, state.versionAlias] }), @@ -135,6 +141,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra indexName: state.tempIndex, mappings: state.tempIndexMappings, }), + READY_TO_REINDEX_SYNC: () => Actions.synchronizeMigrators(readyToReindex), REINDEX_SOURCE_TO_TEMP_OPEN_PIT: (state: ReindexSourceToTempOpenPit) => Actions.openPit({ client, index: state.sourceIndex.value }), REINDEX_SOURCE_TO_TEMP_READ: (state: ReindexSourceToTempRead) => @@ -167,9 +174,10 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra */ refresh: false, }), + DONE_REINDEXING_SYNC: () => Actions.synchronizeMigrators(doneReindexing), SET_TEMP_WRITE_BLOCK: (state: SetTempWriteBlock) => Actions.setWriteBlock({ client, index: state.tempIndex }), - CLONE_TEMP_TO_TARGET: (state: CloneTempToSource) => + CLONE_TEMP_TO_TARGET: (state: CloneTempToTarget) => Actions.cloneIndex({ client, source: state.tempIndex, target: state.targetIndex }), REFRESH_TARGET: (state: RefreshTarget) => Actions.refreshIndex({ client, index: state.targetIndex }), @@ -192,12 +200,13 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra taskId: state.updateTargetMappingsTaskId, timeout: '60s', }), - UPDATE_TARGET_MAPPINGS_META: (state: UpdateTargetMappingsMeta) => - Actions.updateMappings({ + UPDATE_TARGET_MAPPINGS_META: (state: UpdateTargetMappingsMeta) => { + return Actions.updateMappings({ client, index: state.targetIndex, - mappings: omit(state.targetIndexMappings, 'properties'), - }), + mappings: omit(state.targetIndexMappings, ['properties']), // properties already updated on a previous step + }); + }, CHECK_VERSION_INDEX_READY_ACTIONS: () => Actions.noop, OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: (state: OutdatedDocumentsSearchOpenPit) => Actions.openPit({ client, index: state.targetIndex }), @@ -257,8 +266,13 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra }; }; -export const next = (client: ElasticsearchClient, transformRawDocs: TransformRawDocs) => { - const map = nextActionMap(client, transformRawDocs); +export const next = ( + client: ElasticsearchClient, + transformRawDocs: TransformRawDocs, + readyToReindex: Defer, + doneReindexing: Defer +) => { + const map = nextActionMap(client, transformRawDocs, readyToReindex, doneReindexing); return (state: State) => { const delay = createDelayFn(state); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts index b94a94a715056..f3ae3f2c09f75 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/run_resilient_migrator.ts @@ -15,12 +15,16 @@ import type { IndexMapping, SavedObjectsMigrationConfigType, MigrationResult, + IndexTypesMap, } from '@kbn/core-saved-objects-base-server-internal'; +import type { Defer } from './kibana_migrator_utils'; import type { TransformRawDocs } from './types'; import { next } from './next'; import { model } from './model'; import { createInitialState } from './initial_state'; import { migrationStateActionMachine } from './migrations_state_action_machine'; +import { cleanup } from './migrations_state_machine_cleanup'; +import type { State } from './state'; /** * To avoid the Elasticsearch-js client aborting our requests before we @@ -45,9 +49,13 @@ export async function runResilientMigrator({ client, kibanaVersion, waitForMigrationCompletion, + mustRelocateDocuments, + indexTypesMap, targetMappings, logger, preMigrationScript, + readyToReindex, + doneReindexing, transformRawDocs, migrationVersionPerType, indexPrefix, @@ -58,8 +66,12 @@ export async function runResilientMigrator({ client: ElasticsearchClient; kibanaVersion: string; waitForMigrationCompletion: boolean; + mustRelocateDocuments: boolean; + indexTypesMap: IndexTypesMap; targetMappings: IndexMapping; preMigrationScript?: string; + readyToReindex: Defer; + doneReindexing: Defer; logger: Logger; transformRawDocs: TransformRawDocs; migrationVersionPerType: SavedObjectsMigrationVersion; @@ -71,6 +83,8 @@ export async function runResilientMigrator({ const initialState = createInitialState({ kibanaVersion, waitForMigrationCompletion, + mustRelocateDocuments, + indexTypesMap, targetMappings, preMigrationScript, migrationVersionPerType, @@ -84,8 +98,12 @@ export async function runResilientMigrator({ return migrationStateActionMachine({ initialState, logger, - next: next(migrationClient, transformRawDocs), + next: next(migrationClient, transformRawDocs, readyToReindex, doneReindexing), model, - client: migrationClient, + abort: async (state?: State) => { + // At this point, we could reject this migrator's defers and unblock other migrators + // but we are going to throw and shutdown Kibana anyway, so there's no real point in it + await cleanup(client, state); + }, }); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts index 5dd135d4b32fe..6a483da04a399 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/state.ts @@ -13,7 +13,7 @@ import type { SavedObjectsRawDoc, SavedObjectTypeExcludeFromUpgradeFilterHook, } from '@kbn/core-saved-objects-server'; -import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal'; +import type { IndexMapping, IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; import type { ControlState } from './state_action_machine'; import type { AliasAction } from './actions'; import type { TransformErrorObjects } from './core'; @@ -152,6 +152,23 @@ export interface BaseState extends ControlState { */ readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects']; readonly waitForMigrationCompletion: boolean; + + /** + * This flag tells the migrator that SO documents must be redistributed, + * i.e. stored in different system indices, compared to where they are currently stored. + * This requires reindexing documents. + */ + readonly mustRelocateDocuments: boolean; + + /** + * This object holds a relation of all the types that are stored in each index, e.g.: + * { + * '.kibana': [ 'type_1', 'type_2', ... 'type_N' ], + * '.kibana_cases': [ 'type_N+1', 'type_N+2', ... 'type_N+M' ], + * ... + * } + */ + readonly indexTypesMap: IndexTypesMap; } export interface InitState extends BaseState { @@ -231,6 +248,8 @@ export interface FatalState extends BaseState { readonly controlState: 'FATAL'; /** The reason the migration was terminated */ readonly reason: string; + /** The delay in milliseconds before throwing the FATAL exception */ + readonly throwDelayMillis?: number; } export interface WaitForYellowSourceState extends SourceExistsState { @@ -263,7 +282,7 @@ export interface CreateNewTargetState extends PostInitState { readonly versionIndexReadyActions: Option.Some; } -export interface CreateReindexTempState extends SourceExistsState { +export interface CreateReindexTempState extends PostInitState { /** * Create a target index with mappings from the source index and registered * plugins @@ -271,6 +290,11 @@ export interface CreateReindexTempState extends SourceExistsState { readonly controlState: 'CREATE_REINDEX_TEMP'; } +export interface ReadyToReindexSyncState extends PostInitState { + /** Open PIT to the source index */ + readonly controlState: 'READY_TO_REINDEX_SYNC'; +} + export interface ReindexSourceToTempOpenPit extends SourceExistsState { /** Open PIT to the source index */ readonly controlState: 'REINDEX_SOURCE_TO_TEMP_OPEN_PIT'; @@ -304,11 +328,16 @@ export interface ReindexSourceToTempIndexBulk extends ReindexSourceToTempBatch { readonly currentBatch: number; } +export interface DoneReindexingSyncState extends PostInitState { + /** Open PIT to the source index */ + readonly controlState: 'DONE_REINDEXING_SYNC'; +} + export interface SetTempWriteBlock extends PostInitState { readonly controlState: 'SET_TEMP_WRITE_BLOCK'; } -export interface CloneTempToSource extends PostInitState { +export interface CloneTempToTarget extends PostInitState { /** * Clone the temporary reindex index into */ @@ -482,9 +511,10 @@ export type State = Readonly< | CheckVersionIndexReadyActions | CleanupUnknownAndExcluded | CleanupUnknownAndExcludedWaitForTaskState - | CloneTempToSource + | CloneTempToTarget | CreateNewTargetState | CreateReindexTempState + | DoneReindexingSyncState | DoneState | FatalState | InitState @@ -501,6 +531,7 @@ export type State = Readonly< | OutdatedDocumentsSearchRead | OutdatedDocumentsTransform | PrepareCompatibleMigration + | ReadyToReindexSyncState | RefreshSource | RefreshTarget | ReindexSourceToTempClosePit diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json index e56ee4e2c3234..a770476aaf392 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json @@ -26,7 +26,6 @@ "@kbn/core-doc-links-server-mocks", "@kbn/core-logging-server-internal", "@kbn/core-saved-objects-base-server-mocks", - "@kbn/core-elasticsearch-server-mocks", "@kbn/doc-links", "@kbn/safer-lodash-set", "@kbn/logging-mocks", diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/src/status.test.ts b/packages/core/saved-objects/core-saved-objects-server-internal/src/status.test.ts index a6c07ee6ea1ba..8f7f1281f89ea 100644 --- a/packages/core/saved-objects/core-saved-objects-server-internal/src/status.test.ts +++ b/packages/core/saved-objects/core-saved-objects-server-internal/src/status.test.ts @@ -81,7 +81,13 @@ describe('calculateStatus$', () => { it('is available after migrations have ran', async () => { await expect( calculateStatus$( - of({ status: 'completed', result: [{ status: 'skipped' }, { status: 'patched' }] }), + of({ + status: 'completed', + result: [ + { status: 'skipped' }, + { status: 'patched', destIndex: '.kibana', elapsedMs: 28 }, + ], + }), esStatus$ ) .pipe(take(2)) diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 6a02113bbf733..21cd7cd0ea033 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -65,10 +65,17 @@ export async function migrateKibanaIndex(kbnClient: KbnClient) { } /** - * Migrations mean that the Kibana index will look something like: - * .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting - * with .kibana, then filters out any that aren't actually Kibana's core - * index (e.g. we don't want to remove .kibana_task_manager or the like). + * Check if the given index is a Kibana saved object index. + * This includes most .kibana_* + * but we must make sure that indices such as '.kibana_security_session_1' are NOT deleted. + * + * IMPORTANT + * After .kibana split, different SO types can go to different indices. + * This method also takes into account legacy indices: .kibana_1 and .kibana_task_manager_1. + * Thus, if any plugin registers a type using savedObjects.registerType() and binds it to a new + * index, this code MUST be updated for the cleanKibanaIndices() to take them into account. + * @param [index] the name of the index to check + * @returns boolean 'true' if the index is a Kibana saved object index. */ function isKibanaIndex(index?: string): index is string { return Boolean( @@ -107,7 +114,7 @@ export async function cleanKibanaIndices({ while (true) { const resp = await client.deleteByQuery( { - index: `.kibana,.kibana_task_manager`, + index: `.kibana,.kibana_task_manager,.kibana_ui,.kibana_cases`, body: { query: { bool: { @@ -145,6 +152,9 @@ export async function cleanKibanaIndices({ ); stats.deletedIndex('.kibana'); + stats.deletedIndex('.kibana_ui'); + stats.deletedIndex('.kibana_cases'); + stats.deletedIndex('.kibana_task_manager'); } export async function createDefaultSpace({ index, client }: { index: string; client: Client }) { diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts index cba827a6e4d5a..a9c61dea5b106 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts @@ -118,7 +118,7 @@ describe('migration v2', () => { await root.preboot(); await root.setup(); await expect(root.start()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715248 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` + `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715289 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` ); await retryAsync( @@ -131,7 +131,7 @@ describe('migration v2', () => { expect( records.find((rec) => rec.message.startsWith( - `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715248 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` + `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715289 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` ) ) ).toBeDefined(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts index 64592be985719..5eeaf58cd889a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/actions/actions.test.ts @@ -93,7 +93,7 @@ describe('migration actions', () => { await bulkOverwriteTransformedDocuments({ client, index: 'existing_index_with_docs', - operations: docs.map(createBulkIndexOperationTuple), + operations: docs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', })(); @@ -106,7 +106,7 @@ describe('migration actions', () => { await bulkOverwriteTransformedDocuments({ client, index: 'existing_index_with_write_block', - operations: docs.map(createBulkIndexOperationTuple), + operations: docs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', })(); await setWriteBlock({ client, index: 'existing_index_with_write_block' })(); @@ -307,7 +307,7 @@ describe('migration actions', () => { const res = (await bulkOverwriteTransformedDocuments({ client, index: 'new_index_without_write_block', - operations: sourceDocs.map(createBulkIndexOperationTuple), + operations: sourceDocs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', })()) as Either.Left; @@ -887,7 +887,7 @@ describe('migration actions', () => { await bulkOverwriteTransformedDocuments({ client, index: 'reindex_target_4', - operations: sourceDocs.map(createBulkIndexOperationTuple), + operations: sourceDocs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', })(); @@ -1446,7 +1446,7 @@ describe('migration actions', () => { await bulkOverwriteTransformedDocuments({ client, index: 'existing_index_without_mappings', - operations: sourceDocs.map(createBulkIndexOperationTuple), + operations: sourceDocs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', })(); @@ -1896,7 +1896,7 @@ describe('migration actions', () => { const task = bulkOverwriteTransformedDocuments({ client, index: 'existing_index_with_docs', - operations: newDocs.map(createBulkIndexOperationTuple), + operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', }); @@ -1922,7 +1922,7 @@ describe('migration actions', () => { operations: [ ...existingDocs, { _source: { title: 'doc 8' } } as unknown as SavedObjectsRawDoc, - ].map(createBulkIndexOperationTuple), + ].map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', }); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -1942,7 +1942,7 @@ describe('migration actions', () => { bulkOverwriteTransformedDocuments({ client, index: 'existing_index_with_write_block', - operations: newDocs.map(createBulkIndexOperationTuple), + operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)), refresh: 'wait_for', })() ).resolves.toMatchInlineSnapshot(` @@ -1965,7 +1965,7 @@ describe('migration actions', () => { const task = bulkOverwriteTransformedDocuments({ client, index: 'existing_index_with_docs', - operations: newDocs.map(createBulkIndexOperationTuple), + operations: newDocs.map((doc) => createBulkIndexOperationTuple(doc)), }); await expect(task()).resolves.toMatchInlineSnapshot(` Object { From 30cffcbdae3b1a7b562164e04f52b5aabc4df3f0 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Fri, 31 Mar 2023 15:29:54 +0200 Subject: [PATCH 02/17] Revert changes in ES archiver --- .../src/lib/indices/kibana_index.ts | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 21cd7cd0ea033..882f94c5056ec 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -65,17 +65,10 @@ export async function migrateKibanaIndex(kbnClient: KbnClient) { } /** - * Check if the given index is a Kibana saved object index. - * This includes most .kibana_* - * but we must make sure that indices such as '.kibana_security_session_1' are NOT deleted. - * - * IMPORTANT - * After .kibana split, different SO types can go to different indices. - * This method also takes into account legacy indices: .kibana_1 and .kibana_task_manager_1. - * Thus, if any plugin registers a type using savedObjects.registerType() and binds it to a new - * index, this code MUST be updated for the cleanKibanaIndices() to take them into account. - * @param [index] the name of the index to check - * @returns boolean 'true' if the index is a Kibana saved object index. + * Migrations mean that the Kibana index will look something like: + * .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting + * with .kibana, then filters out any that aren't actually Kibana's core + * index (e.g. we don't want to remove .kibana_task_manager or the like). */ function isKibanaIndex(index?: string): index is string { return Boolean( @@ -114,7 +107,7 @@ export async function cleanKibanaIndices({ while (true) { const resp = await client.deleteByQuery( { - index: `.kibana,.kibana_task_manager,.kibana_ui,.kibana_cases`, + index: `.kibana`, body: { query: { bool: { @@ -152,9 +145,6 @@ export async function cleanKibanaIndices({ ); stats.deletedIndex('.kibana'); - stats.deletedIndex('.kibana_ui'); - stats.deletedIndex('.kibana_cases'); - stats.deletedIndex('.kibana_task_manager'); } export async function createDefaultSpace({ index, client }: { index: string; client: Client }) { From 75dcf23bc2fa60d1190c725b0dd8622073fd198d Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Fri, 31 Mar 2023 15:32:19 +0200 Subject: [PATCH 03/17] Revert correctly ^^ --- packages/kbn-es-archiver/src/lib/indices/kibana_index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 882f94c5056ec..6a02113bbf733 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -107,7 +107,7 @@ export async function cleanKibanaIndices({ while (true) { const resp = await client.deleteByQuery( { - index: `.kibana`, + index: `.kibana,.kibana_task_manager`, body: { query: { bool: { From c0f82498ac509bcab760358e9bd3cdbafc46f5e2 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Mon, 3 Apr 2023 16:03:54 +0200 Subject: [PATCH 04/17] Fix failing integration tests, add a new one to test the split --- .../group2/batch_size_bytes.test.ts | 4 +- .../group3/split_kibana_index.test.ts | 373 ++++++++++++++++++ .../migrations/kibana_migrator_test_kit.ts | 60 ++- 3 files changed, 434 insertions(+), 3 deletions(-) create mode 100644 src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts index a9c61dea5b106..c5d6bc3899306 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/batch_size_bytes.test.ts @@ -118,7 +118,7 @@ describe('migration v2', () => { await root.preboot(); await root.setup(); await expect(root.start()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715289 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` + `[Error: Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715286 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.]` ); await retryAsync( @@ -131,7 +131,7 @@ describe('migration v2', () => { expect( records.find((rec) => rec.message.startsWith( - `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715289 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` + `Unable to complete saved object migrations for the [.kibana] index: The document with _id "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5" is 1715286 bytes which exceeds the configured maximum batch size of 1015275 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.` ) ) ).toBeDefined(); diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts new file mode 100644 index 0000000000000..6447a70802863 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts @@ -0,0 +1,373 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server'; +import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; +import { + readLog, + startElasticsearch, + getKibanaMigratorTestKit, + getCurrentVersionTypeRegistry, + overrideTypeRegistry, + clearLog, + getAggregatedTypesCount, + currentVersion, +} from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; + +// define a type => index distribution +const RELOCATE_TYPES: Record = { + dashboard: '.kibana_so_ui', + visualization: '.kibana_so_ui', + 'canvas-workpad': '.kibana_so_ui', + search: '.kibana_so_search', +}; + +describe('split .kibana index into multiple system indices', () => { + let esServer: TestElasticsearchUtils['es']; + let runMigrations: (rerun?: boolean | undefined) => Promise; + let typeRegistry: ISavedObjectTypeRegistry; + + beforeAll(async () => { + typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); + + esServer = await startElasticsearch({ + dataArchive: Path.join(__dirname, '..', 'archives', '7.3.0_xpack_sample_saved_objects.zip'), + }); + }); + + beforeEach(async () => { + await clearLog(); + }); + + describe('when migrating from a legacy version', () => { + it('performs v1 migration and then relocates saved objects into different indices, depending on their types', async () => { + const updatedTypeRegistry = overrideTypeRegistry( + typeRegistry, + (type: SavedObjectsType) => { + return { + ...type, + indexPattern: RELOCATE_TYPES[type.name] ?? type.indexPattern, + }; + } + ); + + const { client, runMigrations: runner } = await getKibanaMigratorTestKit({ + types: updatedTypeRegistry.getAllTypes(), + kibanaIndex: '.kibana', + }); + runMigrations = runner; + + // count of types in the legacy index + expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({ + 'canvas-workpad': 3, + config: 1, + dashboard: 3, + 'index-pattern': 3, + map: 3, + 'maps-telemetry': 1, + 'sample-data-telemetry': 3, + search: 2, + telemetry: 1, + space: 1, + visualization: 39, + }); + + await runMigrations(); + + await client.indices.refresh({ index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'] }); + + expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({ + 'index-pattern': 3, + map: 3, + 'sample-data-telemetry': 3, + config: 1, + telemetry: 1, + space: 1, + }); + expect(await getAggregatedTypesCount(client, '.kibana_so_search')).toEqual({ + search: 2, + }); + expect(await getAggregatedTypesCount(client, '.kibana_so_ui')).toEqual({ + visualization: 39, + 'canvas-workpad': 3, + dashboard: 3, + }); + + const indicesInfo = await client.indices.get({ index: '.kibana*' }); + expect(indicesInfo).toEqual( + expect.objectContaining({ + '.kibana_8.8.0_001': { + aliases: { '.kibana': expect.any(Object), '.kibana_8.8.0': expect.any(Object) }, + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: expect.any(Object), + indexTypesMap: expect.any(Object), + }, + properties: expect.any(Object), + }, + settings: { index: expect.any(Object) }, + }, + '.kibana_so_search_8.8.0_001': { + aliases: { + '.kibana_so_search': expect.any(Object), + '.kibana_so_search_8.8.0': expect.any(Object), + }, + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: expect.any(Object), + indexTypesMap: expect.any(Object), + }, + properties: expect.any(Object), + }, + settings: { index: expect.any(Object) }, + }, + '.kibana_so_ui_8.8.0_001': { + aliases: { + '.kibana_so_ui': expect.any(Object), + '.kibana_so_ui_8.8.0': expect.any(Object), + }, + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: expect.any(Object), + indexTypesMap: expect.any(Object), + }, + properties: expect.any(Object), + }, + settings: { index: expect.any(Object) }, + }, + }) + ); + + expect(indicesInfo[`.kibana_${currentVersion}_001`].mappings?._meta?.indexTypesMap) + .toMatchInlineSnapshot(` + Object { + ".kibana": Array [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "tag", + "graph-workspace", + "canvas-element", + "canvas-workpad-template", + "lens", + "lens-ui-telemetry", + "map", + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group", + ], + ".kibana_so_search": Array [ + "search", + ], + ".kibana_so_ui": Array [ + "visualization", + "canvas-workpad", + "dashboard", + ], + ".kibana_task_manager": Array [ + "task", + ], + } + `); + + const logs = await readLog(); + + // .kibana_task_manager index exists and has no aliases => LEGACY_* migration path + expect(logs).toMatch('[.kibana_task_manager] INIT -> LEGACY_SET_WRITE_BLOCK.'); + // .kibana_task_manager migrator is NOT involved in relocation, must not sync + expect(logs).not.toMatch('[.kibana_task_manager] READY_TO_REINDEX_SYNC'); + + // newer indices migrators did not exist, so they all have to reindex (create temp index + sync) + ['.kibana_so_ui', '.kibana_so_search'].forEach((newIndex) => { + expect(logs).toMatch(`[${newIndex}] INIT -> CREATE_REINDEX_TEMP.`); + expect(logs).toMatch(`[${newIndex}] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.`); + // no docs to reindex, as source index did NOT exist + expect(logs).toMatch(`[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`); + }); + + // the .kibana migrator is involed in a relocation, it must also reindex + expect(logs).toMatch('[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.'); + expect(logs).toMatch('[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS.'); + expect(logs).toMatch('[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.'); + expect(logs).toMatch('[.kibana] SET_SOURCE_WRITE_BLOCK -> CALCULATE_EXCLUDE_FILTERS.'); + expect(logs).toMatch('[.kibana] CALCULATE_EXCLUDE_FILTERS -> CREATE_REINDEX_TEMP.'); + expect(logs).toMatch('[.kibana] CREATE_REINDEX_TEMP -> READY_TO_REINDEX_SYNC.'); + expect(logs).toMatch('[.kibana] READY_TO_REINDEX_SYNC -> REINDEX_SOURCE_TO_TEMP_OPEN_PIT.'); + expect(logs).toMatch( + '[.kibana] REINDEX_SOURCE_TO_TEMP_OPEN_PIT -> REINDEX_SOURCE_TO_TEMP_READ.' + ); + expect(logs).toMatch('[.kibana] Starting to process 59 documents.'); + expect(logs).toMatch( + '[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_TRANSFORM.' + ); + expect(logs).toMatch( + '[.kibana] REINDEX_SOURCE_TO_TEMP_TRANSFORM -> REINDEX_SOURCE_TO_TEMP_INDEX_BULK.' + ); + expect(logs).toMatch('[.kibana_task_manager] LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_DELETE.'); + expect(logs).toMatch( + '[.kibana] REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ.' + ); + expect(logs).toMatch('[.kibana] Processed 59 documents out of 59.'); + expect(logs).toMatch( + '[.kibana] REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT.' + ); + expect(logs).toMatch('[.kibana] REINDEX_SOURCE_TO_TEMP_CLOSE_PIT -> DONE_REINDEXING_SYNC.'); + + // after .kibana migrator is done relocating documents + // the 3 migrators share the final part of the flow + [ + ['.kibana', 8], + ['.kibana_so_ui', 45], + ['.kibana_so_search', 2], + ].forEach(([index, docCount]) => { + expect(logs).toMatch(`[${index}] DONE_REINDEXING_SYNC -> SET_TEMP_WRITE_BLOCK.`); + expect(logs).toMatch(`[${index}] SET_TEMP_WRITE_BLOCK -> CLONE_TEMP_TO_TARGET.`); + + expect(logs).toMatch(`[${index}] CLONE_TEMP_TO_TARGET -> REFRESH_TARGET.`); + expect(logs).toMatch(`[${index}] REFRESH_TARGET -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT.`); + expect(logs).toMatch( + `[${index}] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ.` + ); + expect(logs).toMatch(`[${index}] Starting to process ${docCount} documents.`); + expect(logs).toMatch( + `[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_TRANSFORM.` + ); + expect(logs).toMatch( + `[${index}] OUTDATED_DOCUMENTS_TRANSFORM -> TRANSFORMED_DOCUMENTS_BULK_INDEX.` + ); + expect(logs).toMatch( + `[${index}] OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT.` + ); + expect(logs).toMatch( + `[${index}] OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_REFRESH.` + ); + expect(logs).toMatch(`[${index}] OUTDATED_DOCUMENTS_REFRESH -> CHECK_TARGET_MAPPINGS.`); + expect(logs).toMatch( + `[${index}] CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES.` + ); + + expect(logs).toMatch( + `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES -> UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK.` + ); + expect(logs).toMatch( + `[${index}] UPDATE_TARGET_MAPPINGS_PROPERTIES_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_META.` + ); + expect(logs).toMatch( + `[${index}] UPDATE_TARGET_MAPPINGS_META -> CHECK_VERSION_INDEX_READY_ACTIONS.` + ); + expect(logs).toMatch( + `[${index}] CHECK_VERSION_INDEX_READY_ACTIONS -> MARK_VERSION_INDEX_READY.` + ); + + expect(logs).toMatch(`[${index}] MARK_VERSION_INDEX_READY -> DONE.`); + expect(logs).toMatch(`[${index}] Migration completed`); + }); + }); + }); + + afterEach(async () => { + // we run the migrator again to ensure that the next time state is loaded everything still works as expected + await clearLog(); + await runMigrations(true); + + const logs = await readLog(); + expect(logs).not.toMatch('REINDEX'); + expect(logs).not.toMatch('CREATE'); + expect(logs).not.toMatch('UPDATE_TARGET_MAPPINGS'); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts index ebb8cb5c7a697..f2093f448e1e1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts @@ -36,7 +36,7 @@ import { type LoggingConfigType, LoggingSystem } from '@kbn/core-logging-server- import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server'; import { esTestConfig, kibanaServerTestUser } from '@kbn/test'; import type { LoggerFactory } from '@kbn/logging'; -import { createTestServers } from '@kbn/core-test-helpers-kbn-server'; +import { createRootWithCorePlugins, createTestServers } from '@kbn/core-test-helpers-kbn-server'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { registerServiceConfig } from '@kbn/core-root-server-internal'; import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server'; @@ -276,6 +276,42 @@ const getMigrator = async ( }); }; +export const getAggregatedTypesCount = async (client: ElasticsearchClient, index: string) => { + await client.indices.refresh(); + const response = await client.search({ + index, + _source: false, + aggs: { + typesAggregation: { + terms: { + // assign type __UNKNOWN__ to those documents that don't define one + missing: '__UNKNOWN__', + field: 'type', + size: 100, + }, + aggs: { + docs: { + top_hits: { + size: 10, + _source: { + excludes: ['*'], + }, + }, + }, + }, + }, + }, + }); + + return (response.aggregations!.typesAggregation.buckets as unknown as any).reduce( + (acc: any, current: any) => { + acc[current.key] = current.doc_count; + return acc; + }, + {} + ); +}; + const registerTypes = ( typeRegistry: SavedObjectTypeRegistry, types?: Array> @@ -384,6 +420,28 @@ export const getIncompatibleMappingsMigrator = async ({ }); }; +export const getCurrentVersionTypeRegistry = async ({ + oss, +}: { + oss: boolean; +}): Promise => { + const root = createRootWithCorePlugins({}, { oss }); + await root.preboot(); + const coreSetup = await root.setup(); + const typeRegistry = coreSetup.savedObjects.getTypeRegistry(); + root.shutdown(); // do not await for it, or we might block the tests + return typeRegistry; +}; + +export const overrideTypeRegistry = ( + typeRegistry: ISavedObjectTypeRegistry, + transform: (type: SavedObjectsType) => SavedObjectsType +): ISavedObjectTypeRegistry => { + const updatedTypeRegistry = new SavedObjectTypeRegistry(); + typeRegistry.getAllTypes().forEach((type) => updatedTypeRegistry.registerType(transform(type))); + return updatedTypeRegistry; +}; + export const readLog = async (logFilePath: string = defaultLogFilePath): Promise => { return await fs.readFile(logFilePath, 'utf-8'); }; From dfc804525335685fb4784cb6eecc4431e846372b Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 4 Apr 2023 10:45:00 +0200 Subject: [PATCH 05/17] Define MAIN_SAVED_OBJECT_INDEX in @kbn/core-saved-objects-server --- .../repository.encryption_extension.test.ts | 11 ++--- .../src/lib/repository.test.ts | 19 ++++---- .../test_helpers/repository.test.common.ts | 43 +++++++++---------- .../src/utils/get_index_for_type.test.ts | 7 ++- .../src/kibana_migrator.ts | 17 +++----- .../src/kibana_migrator_constants.ts | 3 -- .../src/kibana_migrator_utils.fixtures.ts | 2 +- .../src/kibana_migrator_utils.test.ts | 12 +++--- .../src/kibana_migrator_utils.ts | 34 +++------------ .../deprecations/unknown_object_types.test.ts | 4 +- .../src/saved_objects_service.ts | 13 +++--- .../src/saved_objects_service.mock.ts | 3 +- .../core-saved-objects-server/index.ts | 1 + .../src/saved_objects_index_pattern.ts | 9 ++++ .../src/core_usage_data_service.ts | 31 +++---------- 15 files changed, 87 insertions(+), 122 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.encryption_extension.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.encryption_extension.test.ts index a297d6b5a2b7e..56b9f433a9593 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.encryption_extension.test.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.encryption_extension.test.ts @@ -22,8 +22,9 @@ import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-m import { kibanaMigratorMock } from '../mocks'; import { SavedObjectsSerializer } from '@kbn/core-saved-objects-base-server-internal'; import { - ISavedObjectsEncryptionExtension, - SavedObjectsRawDocSource, + MAIN_SAVED_OBJECT_INDEX, + type ISavedObjectsEncryptionExtension, + type SavedObjectsRawDocSource, } from '@kbn/core-saved-objects-server'; import { bulkCreateSuccess, @@ -41,8 +42,8 @@ import { mockVersion, mockVersionProps, MULTI_NAMESPACE_ENCRYPTED_TYPE, - TypeIdTuple, updateSuccess, + type TypeIdTuple, } from '../test_helpers/repository.test.common'; import { savedObjectsExtensionsMock } from '../mocks/saved_objects_extensions.mock'; @@ -633,7 +634,7 @@ describe('SavedObjectsRepository Encryption Extension', () => { total: 2, hits: [ { - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, _id: `${space ? `${space}:` : ''}${encryptedSO.type}:${encryptedSO.id}`, _score: 1, ...mockVersionProps, @@ -643,7 +644,7 @@ describe('SavedObjectsRepository Encryption Extension', () => { }, }, { - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, _id: `${space ? `${space}:` : ''}index-pattern:logstash-*`, _score: 2, ...mockVersionProps, diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts index acc586cb8378d..86a2dfd9bc5a1 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts @@ -47,13 +47,14 @@ import type { SavedObjectsBulkDeleteObject, SavedObjectsBulkDeleteOptions, } from '@kbn/core-saved-objects-api-server'; -import type { - SavedObjectsRawDoc, - SavedObjectsRawDocSource, - SavedObjectUnsanitizedDoc, - SavedObject, - SavedObjectReference, - BulkResolveError, +import { + type SavedObjectsRawDoc, + type SavedObjectsRawDocSource, + type SavedObjectUnsanitizedDoc, + type SavedObject, + type SavedObjectReference, + type BulkResolveError, + MAIN_SAVED_OBJECT_INDEX, } from '@kbn/core-saved-objects-server'; import { ALL_NAMESPACES_STRING } from '@kbn/core-saved-objects-utils-server'; import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; @@ -4115,7 +4116,7 @@ describe('SavedObjectsRepository', () => { body: { _id: params.id, ...mockVersionProps, - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, get: { found: true, _source: { @@ -4419,7 +4420,7 @@ describe('SavedObjectsRepository', () => { body: { _id: params.id, ...mockVersionProps, - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, get: { found: true, _source: { diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts index 914e01a16da13..5d4735cedc09a 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts @@ -9,19 +9,20 @@ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { schema } from '@kbn/config-schema'; import { loggerMock } from '@kbn/logging-mocks'; -import { Payload } from 'elastic-apm-node'; -import type { - AuthorizationTypeEntry, - AuthorizeAndRedactMultiNamespaceReferencesParams, - CheckAuthorizationResult, - ISavedObjectsSecurityExtension, - SavedObjectsMappingProperties, - SavedObjectsRawDocSource, - SavedObjectsType, - SavedObjectsTypeMappingDefinition, - SavedObject, - SavedObjectReference, - AuthorizeFindParams, +import type { Payload } from 'elastic-apm-node'; +import { + type AuthorizationTypeEntry, + type AuthorizeAndRedactMultiNamespaceReferencesParams, + type CheckAuthorizationResult, + type ISavedObjectsSecurityExtension, + type SavedObjectsMappingProperties, + type SavedObjectsRawDocSource, + type SavedObjectsType, + type SavedObjectsTypeMappingDefinition, + type SavedObject, + type SavedObjectReference, + type AuthorizeFindParams, + MAIN_SAVED_OBJECT_INDEX, } from '@kbn/core-saved-objects-server'; import type { SavedObjectsBaseOptions, @@ -47,9 +48,9 @@ import { } from '@kbn/core-elasticsearch-client-server-mocks'; import { DocumentMigrator } from '@kbn/core-saved-objects-migration-server-internal'; import { - AuthorizeAndRedactInternalBulkResolveParams, - GetFindRedactTypeMapParams, - AuthorizationTypeMap, + type AuthorizeAndRedactInternalBulkResolveParams, + type GetFindRedactTypeMapParams, + type AuthorizationTypeMap, SavedObjectsErrorHelpers, } from '@kbn/core-saved-objects-server'; import { mockGetSearchDsl } from '../lib/repository.test.mock'; @@ -592,8 +593,6 @@ export const getMockBulkCreateResponse = ( items: objects.map( ({ type, id, originId, attributes, references, migrationVersion, typeMigrationVersion }) => ({ create: { - // status: 1, - // _index: '.kibana', _id: `${namespace ? `${namespace}:` : ''}${type}:${id}`, _source: { [type]: attributes, @@ -714,7 +713,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => { total: 4, hits: [ { - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, _id: `${namespace ? `${namespace}:` : ''}index-pattern:logstash-*`, _score: 1, ...mockVersionProps, @@ -731,7 +730,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => { }, }, { - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, _id: `${namespace ? `${namespace}:` : ''}config:6.0.0-alpha1`, _score: 2, ...mockVersionProps, @@ -746,7 +745,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => { }, }, { - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, _id: `${namespace ? `${namespace}:` : ''}index-pattern:stocks-*`, _score: 3, ...mockVersionProps, @@ -762,7 +761,7 @@ export const generateIndexPatternSearchResults = (namespace?: string) => { }, }, { - _index: '.kibana', + _index: MAIN_SAVED_OBJECT_INDEX, _id: `${NAMESPACE_AGNOSTIC_TYPE}:something`, _score: 4, ...mockVersionProps, diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/get_index_for_type.test.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/get_index_for_type.test.ts index 551c9dc1187eb..d5bc19a11d17e 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/get_index_for_type.test.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/utils/get_index_for_type.test.ts @@ -6,7 +6,10 @@ * Side Public License, v 1. */ -import { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server'; +import { + type ISavedObjectTypeRegistry, + MAIN_SAVED_OBJECT_INDEX, +} from '@kbn/core-saved-objects-server'; import { getIndexForType } from './get_index_for_type'; const createTypeRegistry = () => { @@ -17,7 +20,7 @@ const createTypeRegistry = () => { describe('getIndexForType', () => { const kibanaVersion = '8.0.0'; - const defaultIndex = '.kibana'; + const defaultIndex = MAIN_SAVED_OBJECT_INDEX; let typeRegistry: ReturnType; beforeEach(() => { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index 38566a9d3f2da..77c47c0f7e8fc 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -16,10 +16,11 @@ import Semver from 'semver'; import type { Logger } from '@kbn/logging'; import type { DocLinksServiceStart } from '@kbn/core-doc-links-server'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { - SavedObjectUnsanitizedDoc, - SavedObjectsRawDoc, - ISavedObjectTypeRegistry, +import { + MAIN_SAVED_OBJECT_INDEX, + type SavedObjectUnsanitizedDoc, + type SavedObjectsRawDoc, + type ISavedObjectTypeRegistry, } from '@kbn/core-saved-objects-server'; import { SavedObjectsSerializer, @@ -39,12 +40,7 @@ import { runResilientMigrator } from './run_resilient_migrator'; import { migrateRawDocsSafely } from './core/migrate_raw_docs'; import { runZeroDowntimeMigration } from './zdt'; import { createMultiPromiseDefer, indexMapToIndexTypesMap } from './kibana_migrator_utils'; -import { - ALLOWED_CONVERT_VERSION, - DEFAULT_INDEX_TYPES_MAP, - LEGACY_SAVED_OBJECT_INDEX, - MAIN_SAVED_OBJECT_INDEX, -} from './kibana_migrator_constants'; +import { ALLOWED_CONVERT_VERSION, DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants'; export interface KibanaMigratorOptions { client: ElasticsearchClient; @@ -195,7 +191,6 @@ export class KibanaMigrator implements IKibanaMigrator { // and check if some SO types have been moved to different indices const indicesWithMovingTypes = await getIndicesInvoledInRelocation({ mainIndex: MAIN_SAVED_OBJECT_INDEX, - legacyIndex: LEGACY_SAVED_OBJECT_INDEX, client: this.client, indexTypesMap, logger: this.log, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts index be199122879ab..72915178e39f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_constants.ts @@ -21,9 +21,6 @@ export interface TypeStatusDetails { status: TypeStatus; } -export const MAIN_SAVED_OBJECT_INDEX = '.kibana'; -export const LEGACY_SAVED_OBJECT_INDEX = '.kibana_1'; - // ensure plugins don't try to convert SO namespaceTypes after 8.0.0 // see https://github.com/elastic/kibana/issues/147344 export const ALLOWED_CONVERT_VERSION = '8.0.0'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts index c748e57705cea..802167d733fb5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.fixtures.ts @@ -8,7 +8,7 @@ import { IndexMap } from './core'; -export const INDEX_MAP_8_8_0: IndexMap = { +export const INDEX_MAP_BEFORE_SPLIT: IndexMap = { '.kibana': { typeMappings: { 'core-usage-stats': { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts index 2b00d12c7bce5..fe24cf36f788f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts @@ -8,6 +8,7 @@ import { errors } from '@elastic/elasticsearch'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants'; import { @@ -16,7 +17,7 @@ import { getIndicesInvoledInRelocation, indexMapToIndexTypesMap, } from './kibana_migrator_utils'; -import { INDEX_MAP_8_8_0 } from './kibana_migrator_utils.fixtures'; +import { INDEX_MAP_BEFORE_SPLIT } from './kibana_migrator_utils.fixtures'; describe('createMultiPromiseDefer', () => { it('creates defer objects with the same Promise', () => { @@ -47,8 +48,7 @@ describe('getIndicesInvoledInRelocation', () => { return { client, - mainIndex: '.kibana', - legacyIndex: '.kibana_1', + mainIndex: MAIN_SAVED_OBJECT_INDEX, indexTypesMap: {}, defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, logger: loggerMock.create(), @@ -65,7 +65,7 @@ describe('getIndicesInvoledInRelocation', () => { expect(params.client.indices.getMapping).toHaveBeenCalledTimes(1); expect(params.client.indices.getMapping).toHaveBeenCalledWith({ - index: '.kibana', + index: MAIN_SAVED_OBJECT_INDEX, }); }); @@ -92,7 +92,7 @@ describe('getIndicesInvoledInRelocation', () => { ); }); - it('assumes fresh deployment if the mainIndex and legacyIndex do not exist, returns an empty list of moving types', async () => { + it('assumes fresh deployment if the mainIndex does not exist, returns an empty list of moving types', async () => { const params = getIndicesInvoledInRelocationParams(); params.client.indices.getMapping.mockImplementation(() => elasticsearchClientMock.createErrorTransportRequestPromise( @@ -197,7 +197,7 @@ describe('getIndicesInvoledInRelocation', () => { describe('indexMapToIndexTypesMap', () => { it('converts IndexMap to IndexTypesMap', () => { - expect(indexMapToIndexTypesMap(INDEX_MAP_8_8_0)).toEqual(DEFAULT_INDEX_TYPES_MAP); + expect(indexMapToIndexTypesMap(INDEX_MAP_BEFORE_SPLIT)).toEqual(DEFAULT_INDEX_TYPES_MAP); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts index 1deba9561574c..029b6b9da55e7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts @@ -38,13 +38,11 @@ export function createMultiPromiseDefer(indices: string[]): Record { @@ -57,32 +55,13 @@ export async function getCurrentIndexTypesMap({ // main index exists, try to extract the indexTypesMap from _meta const meta = Object.values(mapping)?.[0]?.mappings._meta; return meta?.indexTypesMap ?? defaultIndexTypesMap; - } catch (mainError) { - if (mainError.meta?.statusCode === 404) { - // the main index does NOT exist. Check if the legacy index (i.e. .kibana_1) exists - try { - await client.indices.getMapping({ - index: legacyIndex, - }); - // legacy exists, we assume we will have a default indexTypesMap after LEGACY migration - return defaultIndexTypesMap; - } catch (legacyError) { - if (legacyError.meta?.statusCode === 404) { - // legacy does NOT exist, we assume this is a fresh deployment - logger.debug( - `The ${mainIndex} and ${legacyIndex} indices do NOT exist. Assuming this is a fresh deployment` - ); - return undefined; - } else { - logger.fatal( - `Cannot query the meta information on the ${legacyIndex} saved object index` - ); - throw legacyError; - } - } + } catch (error) { + if (error.meta?.statusCode === 404) { + logger.debug(`The ${mainIndex} index do NOT exist. Assuming this is a fresh deployment`); + return undefined; } else { logger.fatal(`Cannot query the meta information on the ${mainIndex} saved object index`); - throw mainError; + throw error; } } } @@ -90,14 +69,12 @@ export async function getCurrentIndexTypesMap({ export async function getIndicesInvoledInRelocation({ client, mainIndex, - legacyIndex, indexTypesMap, defaultIndexTypesMap, logger, }: { client: ElasticsearchClient; mainIndex: string; - legacyIndex: string; indexTypesMap: IndexTypesMap; defaultIndexTypesMap: IndexTypesMap; logger: Logger; @@ -107,7 +84,6 @@ export async function getIndicesInvoledInRelocation({ const currentIndexTypesMap = await getCurrentIndexTypesMap({ client, mainIndex, - legacyIndex, defaultIndexTypesMap, logger, }); diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/src/deprecations/unknown_object_types.test.ts b/packages/core/saved-objects/core-saved-objects-server-internal/src/deprecations/unknown_object_types.test.ts index 2199a7fb32b5e..defcbd591e2a4 100644 --- a/packages/core/saved-objects/core-saved-objects-server-internal/src/deprecations/unknown_object_types.test.ts +++ b/packages/core/saved-objects/core-saved-objects-server-internal/src/deprecations/unknown_object_types.test.ts @@ -12,7 +12,7 @@ import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import { deleteUnknownTypeObjects, getUnknownTypesDeprecations } from './unknown_object_types'; import { typeRegistryMock } from '@kbn/core-saved-objects-base-server-mocks'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; -import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { type SavedObjectsType, MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; const createAggregateTypesSearchResponse = ( typesIds: Record = {} @@ -50,7 +50,7 @@ describe('unknown saved object types deprecation', () => { let typeRegistry: ReturnType; let esClient: ReturnType; - const kibanaIndex = '.kibana'; + const kibanaIndex = MAIN_SAVED_OBJECT_INDEX; beforeEach(() => { typeRegistry = typeRegistryMock.create(); diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts b/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts index dbb04577d3f7e..1c04c109df83a 100644 --- a/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts +++ b/packages/core/saved-objects/core-saved-objects-server-internal/src/saved_objects_service.ts @@ -52,13 +52,12 @@ import { import type { InternalCoreUsageDataSetup } from '@kbn/core-usage-data-base-server-internal'; import type { DeprecationRegistryProvider } from '@kbn/core-deprecations-server'; import type { NodeInfo } from '@kbn/core-node-server'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { registerRoutes } from './routes'; import { calculateStatus$ } from './status'; import { registerCoreObjectTypes } from './object_types'; import { getSavedObjectsDeprecationsProvider } from './deprecations'; -const kibanaIndex = '.kibana'; - /** * @internal */ @@ -125,7 +124,7 @@ export class SavedObjectsService this.config = new SavedObjectConfig(savedObjectsConfig, savedObjectsMigrationConfig); deprecations.getRegistry('savedObjects').registerDeprecations( getSavedObjectsDeprecationsProvider({ - kibanaIndex, + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, savedObjectsConfig: this.config, kibanaVersion: this.kibanaVersion, typeRegistry: this.typeRegistry, @@ -140,7 +139,7 @@ export class SavedObjectsService logger: this.logger, config: this.config, migratorPromise: firstValueFrom(this.migrator$), - kibanaIndex, + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, kibanaVersion: this.kibanaVersion, }); @@ -198,7 +197,7 @@ export class SavedObjectsService this.typeRegistry.registerType(type); }, getTypeRegistry: () => this.typeRegistry, - getKibanaIndex: () => kibanaIndex, + getKibanaIndex: () => MAIN_SAVED_OBJECT_INDEX, }; } @@ -280,7 +279,7 @@ export class SavedObjectsService return SavedObjectsRepository.createRepository( migrator, this.typeRegistry, - kibanaIndex, + MAIN_SAVED_OBJECT_INDEX, esClient, this.logger.get('repository'), includedHiddenTypes, @@ -357,7 +356,7 @@ export class SavedObjectsService logger: this.logger, kibanaVersion: this.kibanaVersion, soMigrationsConfig, - kibanaIndex, + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, client, docLinks, waitForMigrationCompletion, diff --git a/packages/core/saved-objects/core-saved-objects-server-mocks/src/saved_objects_service.mock.ts b/packages/core/saved-objects/core-saved-objects-server-mocks/src/saved_objects_service.mock.ts index eccdb9ca16c54..5bccf3b55055c 100644 --- a/packages/core/saved-objects/core-saved-objects-server-mocks/src/saved_objects_service.mock.ts +++ b/packages/core/saved-objects/core-saved-objects-server-mocks/src/saved_objects_service.mock.ts @@ -29,6 +29,7 @@ import { savedObjectsImporterMock, } from '@kbn/core-saved-objects-import-export-server-mocks'; import { migrationMocks } from '@kbn/core-saved-objects-migration-server-mocks'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; type SavedObjectsServiceContract = PublicMethodsOf; @@ -70,7 +71,7 @@ const createSetupContractMock = () => { getKibanaIndex: jest.fn(), }; - setupContract.getKibanaIndex.mockReturnValue('.kibana'); + setupContract.getKibanaIndex.mockReturnValue(MAIN_SAVED_OBJECT_INDEX); return setupContract; }; diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index d3d2ffe71aa34..ed4981770f281 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -52,6 +52,7 @@ export type { SavedObjectsExportablePredicate, } from './src/saved_objects_management'; export type { SavedObjectStatusMeta } from './src/saved_objects_status'; +export { MAIN_SAVED_OBJECT_INDEX } from './src/saved_objects_index_pattern'; export type { SavedObjectsType, SavedObjectTypeExcludeFromUpgradeFilterHook, diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts new file mode 100644 index 0000000000000..53d05d7c1d45f --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const MAIN_SAVED_OBJECT_INDEX = '.kibana'; diff --git a/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_data_service.ts b/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_data_service.ts index bd9fe1ac47859..63a8da844e090 100644 --- a/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_data_service.ts +++ b/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_data_service.ts @@ -40,7 +40,10 @@ import { type InternalCoreUsageDataSetup, } from '@kbn/core-usage-data-base-server-internal'; import type { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-internal'; -import type { SavedObjectsServiceStart } from '@kbn/core-saved-objects-server'; +import { + MAIN_SAVED_OBJECT_INDEX, + type SavedObjectsServiceStart, +} from '@kbn/core-saved-objects-server'; import { isConfigured } from './is_configured'; import { coreUsageStatsType } from './saved_objects'; @@ -61,26 +64,6 @@ export interface StartDeps { exposedConfigsToUsage: ExposedConfigsToUsage; } -const kibanaIndex = '.kibana'; - -/** - * Because users can configure their Saved Object to any arbitrary index name, - * we need to map customized index names back to a "standard" index name. - * - * e.g. If a user configures `kibana.index: .my_saved_objects` we want to the - * collected data to be grouped under `.kibana` not ".my_saved_objects". - * - * This is rather brittle, but the option to configure index names might go - * away completely anyway (see #60053). - * - * @param index The index name configured for this SO type - * @param kibanaConfigIndex The default kibana index as configured by the user - * with `kibana.index` - */ -const kibanaOrTaskManagerIndex = (index: string, kibanaConfigIndex: string) => { - return index === kibanaConfigIndex ? '.kibana' : '.kibana_task_manager'; -}; - interface UsageDataAggs extends AggregationsMultiBucketAggregateBase { buckets: { disabled: AggregationsSingleBucketAggregateBase; @@ -133,7 +116,7 @@ export class CoreUsageDataService .getTypeRegistry() .getAllTypes() .reduce((acc, type) => { - const index = type.indexPattern ?? kibanaIndex; + const index = type.indexPattern ?? MAIN_SAVED_OBJECT_INDEX; return acc.add(index); }, new Set()) .values() @@ -151,7 +134,7 @@ export class CoreUsageDataService const stats = body[0]; return { - alias: kibanaOrTaskManagerIndex(index, kibanaIndex), + alias: index, docsCount: stats['docs.count'] ? parseInt(stats['docs.count'], 10) : 0, docsDeleted: stats['docs.deleted'] ? parseInt(stats['docs.deleted'], 10) : 0, storeSizeBytes: stats['store.size'] ? parseInt(stats['store.size'], 10) : 0, @@ -192,7 +175,7 @@ export class CoreUsageDataService unknown, { aliases: UsageDataAggs } >({ - index: kibanaIndex, + index: MAIN_SAVED_OBJECT_INDEX, // depends on the .kibana split (assuming 'legacy-url-alias' is stored in '.kibana') body: { track_total_hits: true, query: { match: { type: LEGACY_URL_ALIAS_TYPE } }, From 14e9f8ed18f92546017163899f97807f1e8dca93 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Wed, 5 Apr 2023 13:40:38 +0200 Subject: [PATCH 06/17] Relocate 'cases' saved objects to '.kibana_cases' index --- .../core-saved-objects-server/index.ts | 7 +- .../src/saved_objects_index_pattern.ts | 14 ++++ .../src/saved_objects_type.ts | 3 +- .../src/actions/empty_kibana_index.ts | 3 +- packages/kbn-es-archiver/src/actions/load.ts | 10 +-- .../lib/docs/generate_doc_records_stream.ts | 5 +- .../src/lib/indices/create_index_stream.ts | 8 ++- .../src/lib/indices/delete_index_stream.ts | 3 +- .../indices/generate_index_records_stream.ts | 6 +- .../src/lib/indices/kibana_index.ts | 45 ++++++++----- packages/kbn-es-archiver/tsconfig.json | 1 + .../kibana_server/extend_es_archiver.ts | 4 +- .../tsconfig.json | 1 + .../group1/7.7.2_xpack_100k.test.ts | 7 +- .../group3/split_kibana_index.test.ts | 9 ++- .../service/lib/repository_with_proxy.test.ts | 27 +++++--- .../lib/repository_with_proxy_utils.ts | 67 ++++++++++++++++--- .../plugins/cases/common/constants/index.ts | 2 + .../cases/server/saved_object_types/cases.ts | 3 +- .../server/saved_object_types/comments.ts | 3 +- .../server/saved_object_types/configure.ts | 3 +- .../saved_object_types/connector_mappings.ts | 3 +- .../server/saved_object_types/telemetry.ts | 3 +- .../server/saved_object_types/user_actions.ts | 3 +- .../apis/security/index_fields.ts | 3 +- .../apis/security_solution/utils.ts | 3 +- .../common/lib/api/index.ts | 19 +++--- .../tests/common/cases/find_cases.ts | 3 +- x-pack/test/common/lib/test_data_loader.ts | 3 +- .../common/lib/space_test_utils.ts | 3 +- .../common/suites/delete.ts | 3 +- .../suites/disable_legacy_url_aliases.ts | 8 ++- .../common/suites/update_objects_spaces.ts | 5 +- 33 files changed, 211 insertions(+), 79 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index ed4981770f281..6ffb78b547389 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -52,7 +52,12 @@ export type { SavedObjectsExportablePredicate, } from './src/saved_objects_management'; export type { SavedObjectStatusMeta } from './src/saved_objects_status'; -export { MAIN_SAVED_OBJECT_INDEX } from './src/saved_objects_index_pattern'; +export { + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, + SavedObjectsIndexPatterns, + type SavedObjectsIndexPattern, +} from './src/saved_objects_index_pattern'; export type { SavedObjectsType, SavedObjectTypeExcludeFromUpgradeFilterHook, diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts index 53d05d7c1d45f..7a79400f07b25 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts @@ -6,4 +6,18 @@ * Side Public License, v 1. */ +/** + * Collect and centralize the names of the different saved object indices. + * Note that all of them start with the '.kibana' prefix. + * There are multiple places in the code that these indices have the form .kibana*. + * However, beware that there are some system indices that have the same prefix + * but are NOT used to store saved objects, e.g.: .kibana_security_session_1 + */ export const MAIN_SAVED_OBJECT_INDEX = '.kibana'; +export const TASK_MANAGER_SAVED_OBJECT_INDEX = `${MAIN_SAVED_OBJECT_INDEX}_task_manager`; +export const SavedObjectsIndexPatterns = [ + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, + `${MAIN_SAVED_OBJECT_INDEX}_cases`, +]; +export type SavedObjectsIndexPattern = '.kibana' | '.kibana_task_manager' | '.kibana_cases'; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts index b9fa8e6365114..fe0698c944677 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts @@ -18,6 +18,7 @@ import type { SavedObjectsModelVersionMap, SavedObjectsModelVersionMapProvider, } from './model_version'; +import type { SavedObjectsIndexPattern } from './saved_objects_index_pattern'; /** * @public @@ -52,7 +53,7 @@ export interface SavedObjectsType { /** * If defined, the type instances will be stored in the given index instead of the default one. */ - indexPattern?: string; + indexPattern?: SavedObjectsIndexPattern; /** * If defined, will be used to convert the type to an alias. */ diff --git a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts index c9b67e4745d45..cfc809636fd36 100644 --- a/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts +++ b/packages/kbn-es-archiver/src/actions/empty_kibana_index.ts @@ -10,6 +10,7 @@ import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { KbnClient } from '@kbn/test'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib'; export async function emptyKibanaIndexAction({ @@ -25,6 +26,6 @@ export async function emptyKibanaIndexAction({ await cleanKibanaIndices({ client, stats, log }); await migrateKibanaIndex(kbnClient); - stats.createdIndex('.kibana'); + SavedObjectsIndexPatterns.forEach((indexPattern) => stats.createdIndex(indexPattern)); return stats.toJSON(); } diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts index b6a4c46d42743..efda58193b118 100644 --- a/packages/kbn-es-archiver/src/actions/load.ts +++ b/packages/kbn-es-archiver/src/actions/load.ts @@ -14,6 +14,7 @@ import { REPO_ROOT } from '@kbn/repo-info'; import type { KbnClient } from '@kbn/test'; import type { Client } from '@elastic/elasticsearch'; import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { ES_CLIENT_HEADERS } from '../client_headers'; import { @@ -104,14 +105,15 @@ export async function loadAction({ } ); - // If we affected the Kibana index, we need to ensure it's migrated... - if (Object.keys(result).some((k) => k.startsWith('.kibana'))) { + // If we affected saved objects indices, we need to ensure they are migrated... + if (Object.keys(result).some((k) => k.startsWith(MAIN_SAVED_OBJECT_INDEX))) { await migrateKibanaIndex(kbnClient); log.debug('[%s] Migrated Kibana index after loading Kibana data', name); if (kibanaPluginIds.includes('spaces')) { - await createDefaultSpace({ client, index: '.kibana' }); - log.debug('[%s] Ensured that default space exists in .kibana', name); + // WARNING affected by #104081. Assumes 'spaces' saved objects are stored in MAIN_SAVED_OBJECT_INDEX + await createDefaultSpace({ client, index: MAIN_SAVED_OBJECT_INDEX }); + log.debug(`[%s] Ensured that default space exists in ${MAIN_SAVED_OBJECT_INDEX}`, name); } } diff --git a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts index 6e3310a7347e7..f0c6db9c89fcb 100644 --- a/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/docs/generate_doc_records_stream.ts @@ -8,6 +8,7 @@ import { Transform } from 'stream'; import type { Client } from '@elastic/elasticsearch'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { Progress } from '../progress'; import { ES_CLIENT_HEADERS } from '../../client_headers'; @@ -78,7 +79,9 @@ export function createGenerateDocRecordsStream({ // if keepIndexNames is false, rewrite the .kibana_* index to .kibana_1 so that // when it is loaded it can skip migration, if possible index: - hit._index.startsWith('.kibana') && !keepIndexNames ? '.kibana_1' : hit._index, + hit._index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !keepIndexNames + ? `${MAIN_SAVED_OBJECT_INDEX}_1` + : hit._index, data_stream: dataStream, id: hit._id, source: hit._source, diff --git a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts index 38f4bed755262..a13c1b7e856e9 100644 --- a/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts @@ -14,6 +14,10 @@ import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + MAIN_SAVED_OBJECT_INDEX, + TASK_MANAGER_SAVED_OBJECT_INDEX, +} from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { deleteKibanaIndices } from './kibana_index'; import { deleteIndex } from './delete_index'; @@ -96,8 +100,8 @@ export function createCreateIndexStream({ async function handleIndex(record: DocRecord) { const { index, settings, mappings, aliases } = record.value; - const isKibanaTaskManager = index.startsWith('.kibana_task_manager'); - const isKibana = index.startsWith('.kibana') && !isKibanaTaskManager; + const isKibanaTaskManager = index.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX); + const isKibana = index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !isKibanaTaskManager; if (docsOnly) { return; diff --git a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts index c7633465ccc4c..1c10ab0fc273c 100644 --- a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts @@ -10,6 +10,7 @@ import { Transform } from 'stream'; import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; import { cleanKibanaIndices } from './kibana_index'; @@ -28,7 +29,7 @@ export function createDeleteIndexStream(client: Client, stats: Stats, log: Tooli if (record.type === 'index') { const { index } = record.value; - if (index.startsWith('.kibana')) { + if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) { await cleanKibanaIndices({ client, stats, log }); } else { await deleteIndex({ client, stats, log, index }); diff --git a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts index de32e93e27398..2f2dd60982a94 100644 --- a/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts +++ b/packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts @@ -9,6 +9,7 @@ import type { Client } from '@elastic/elasticsearch'; import { Transform } from 'stream'; import { ToolingLog } from '@kbn/tooling-log'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { ES_CLIENT_HEADERS } from '../../client_headers'; import { getIndexTemplate } from '..'; @@ -100,7 +101,10 @@ export function createGenerateIndexRecordsStream({ value: { // if keepIndexNames is false, rewrite the .kibana_* index to .kibana_1 so that // when it is loaded it can skip migration, if possible - index: index.startsWith('.kibana') && !keepIndexNames ? '.kibana_1' : index, + index: + index.startsWith(MAIN_SAVED_OBJECT_INDEX) && !keepIndexNames + ? `${MAIN_SAVED_OBJECT_INDEX}_1` + : index, settings, mappings, aliases, diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index 6a02113bbf733..75aa5f5fce3d5 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -11,12 +11,17 @@ import { inspect } from 'util'; import type { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { KbnClient } from '@kbn/test'; +import { + MAIN_SAVED_OBJECT_INDEX, + SavedObjectsIndexPatterns, + TASK_MANAGER_SAVED_OBJECT_INDEX, +} from '@kbn/core-saved-objects-server'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; import { ES_CLIENT_HEADERS } from '../../client_headers'; /** - * Deletes all indices that start with `.kibana`, or if onlyTaskManager==true, all indices that start with `.kibana_task_manager` + * Deletes all saved object indices, or if onlyTaskManager==true, it deletes task_manager indices */ export async function deleteKibanaIndices({ client, @@ -29,8 +34,10 @@ export async function deleteKibanaIndices({ onlyTaskManager?: boolean; log: ToolingLog; }) { - const indexPattern = onlyTaskManager ? '.kibana_task_manager*' : '.kibana*'; - const indexNames = await fetchKibanaIndices(client, indexPattern); + // WARNING note that we are deleting ALL .kibana* indices here, NOT only the saved object ones + const indexNames = (await fetchKibanaIndices(client)).filter( + (indexName) => !onlyTaskManager || indexName.includes(TASK_MANAGER_SAVED_OBJECT_INDEX) + ); if (!indexNames.length) { return; } @@ -65,22 +72,28 @@ export async function migrateKibanaIndex(kbnClient: KbnClient) { } /** - * Migrations mean that the Kibana index will look something like: - * .kibana, .kibana_1, .kibana_323, etc. This finds all indices starting - * with .kibana, then filters out any that aren't actually Kibana's core - * index (e.g. we don't want to remove .kibana_task_manager or the like). + * Check if the given index is a Kibana saved object index. + * This includes most .kibana_* + * but we must make sure that indices such as '.kibana_security_session_1' are NOT deleted. + * + * IMPORTANT + * Note that we can have more than 2 system indices (different SO types can go to different indices) + * ATM we have '.kibana', '.kibana_task_manager', '.kibana_cases' + * This method also takes into account legacy indices: .kibana_1, .kibana_task_manager_1. + * @param [index] the name of the index to check + * @returns boolean 'true' if the index is a Kibana saved object index. */ + +const LEGACY_INDICES_REGEXP = new RegExp(`^(${SavedObjectsIndexPatterns.join('|')})(:?_\\d*)?$`); +const INDICES_REGEXP = new RegExp(`^(${SavedObjectsIndexPatterns.join('|')})_(pre)?\\d+.\\d+.\\d+`); + function isKibanaIndex(index?: string): index is string { - return Boolean( - index && - (/^\.kibana(:?_\d*)?$/.test(index) || - /^\.kibana(_task_manager)?_(pre)?\d+\.\d+\.\d+/.test(index)) - ); + return Boolean(index && (LEGACY_INDICES_REGEXP.test(index) || INDICES_REGEXP.test(index))); } -async function fetchKibanaIndices(client: Client, indexPattern: string) { +async function fetchKibanaIndices(client: Client) { const resp = await client.cat.indices( - { index: indexPattern, format: 'json' }, + { index: `${MAIN_SAVED_OBJECT_INDEX}*`, format: 'json' }, { headers: ES_CLIENT_HEADERS, } @@ -107,7 +120,7 @@ export async function cleanKibanaIndices({ while (true) { const resp = await client.deleteByQuery( { - index: `.kibana,.kibana_task_manager`, + index: SavedObjectsIndexPatterns, body: { query: { bool: { @@ -144,7 +157,7 @@ export async function cleanKibanaIndices({ `.kibana rather than deleting the whole index` ); - stats.deletedIndex('.kibana'); + SavedObjectsIndexPatterns.forEach((indexPattern) => stats.deletedIndex(indexPattern)); } export async function createDefaultSpace({ index, client }: { index: string; client: Client }) { diff --git a/packages/kbn-es-archiver/tsconfig.json b/packages/kbn-es-archiver/tsconfig.json index 0301480548fc7..15fccdf68be4f 100644 --- a/packages/kbn-es-archiver/tsconfig.json +++ b/packages/kbn-es-archiver/tsconfig.json @@ -11,6 +11,7 @@ "**/*.ts" ], "kbn_references": [ + "@kbn/core-saved-objects-server", "@kbn/dev-utils", "@kbn/test", "@kbn/tooling-log", diff --git a/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts b/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts index 98c28960bf523..4c2613d273c4a 100644 --- a/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts +++ b/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import type { ProvidedType } from '@kbn/test'; import type { EsArchiverProvider } from '../es_archiver'; @@ -13,7 +14,6 @@ import type { RetryService } from '../retry'; import type { KibanaServerProvider } from './kibana_server'; const ES_ARCHIVER_LOAD_METHODS = ['load', 'loadIfNeeded', 'unload', 'emptyKibanaIndex'] as const; -const KIBANA_INDEX = '.kibana'; interface Options { esArchiver: ProvidedType; @@ -38,7 +38,7 @@ export function extendEsArchiver({ esArchiver, kibanaServer, retry, defaults }: const statsKeys = Object.keys(stats); const kibanaKeys = statsKeys.filter( // this also matches stats keys like '.kibana_1' and '.kibana_2,.kibana_1' - (key) => key.includes(KIBANA_INDEX) && stats[key].created + (key) => key.includes(MAIN_SAVED_OBJECT_INDEX) && stats[key].created ); // if the kibana index was created by the esArchiver then update the uiSettings diff --git a/packages/kbn-ftr-common-functional-services/tsconfig.json b/packages/kbn-ftr-common-functional-services/tsconfig.json index 639991bb2ce77..3641c807e4d6d 100644 --- a/packages/kbn-ftr-common-functional-services/tsconfig.json +++ b/packages/kbn-ftr-common-functional-services/tsconfig.json @@ -11,6 +11,7 @@ "**/*.ts", ], "kbn_references": [ + "@kbn/core-saved-objects-server", "@kbn/tooling-log", "@kbn/es-archiver", "@kbn/test" diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts index b8010eacbbae0..6c77c786522d7 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts @@ -19,6 +19,7 @@ import { createRootWithCorePlugins, type TestElasticsearchUtils, } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log'); @@ -105,7 +106,9 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { await new Promise((resolve) => setTimeout(resolve, 10000)); }; - const migratedIndex = `.kibana_${kibanaVersion}_001`; + const migratedIndices = SavedObjectsIndexPatterns.map( + (indexPattern) => `.${indexPattern}_${kibanaVersion}_001` + ); beforeAll(async () => { await removeLogFile(); @@ -121,7 +124,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { it('copies all the document of the previous index to the new one', async () => { const migratedIndexResponse = await esClient.count({ - index: migratedIndex, + index: migratedIndices, }); const oldIndexResponse = await esClient.count({ index: '.kibana_1', diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts index 6447a70802863..8d7a37e599e40 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts @@ -8,7 +8,11 @@ import Path from 'path'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server'; +import type { + ISavedObjectTypeRegistry, + SavedObjectsIndexPattern, + SavedObjectsType, +} from '@kbn/core-saved-objects-server'; import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; import { readLog, @@ -54,7 +58,8 @@ describe('split .kibana index into multiple system indices', () => { (type: SavedObjectsType) => { return { ...type, - indexPattern: RELOCATE_TYPES[type.name] ?? type.indexPattern, + indexPattern: (RELOCATE_TYPES[type.name] ?? + type.indexPattern) as SavedObjectsIndexPattern, }; } ); diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts index e38231a6e0ea4..38377cff6fed7 100644 --- a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts +++ b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts @@ -9,7 +9,7 @@ import Hapi from '@hapi/hapi'; import h2o2 from '@hapi/h2o2'; import { URL } from 'url'; -import type { SavedObject } from '@kbn/core-saved-objects-server'; +import { SavedObject, SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import type { ISavedObjectsRepository } from '@kbn/core-saved-objects-api-server'; import type { InternalCoreSetup, InternalCoreStart } from '@kbn/core-lifecycle-server-internal'; import { Root } from '@kbn/core-root-server-internal'; @@ -18,6 +18,7 @@ import { createTestServers, type TestElasticsearchUtils, } from '@kbn/core-test-helpers-kbn-server'; +import { kibanaPackageJson as pkg } from '@kbn/repo-info'; import { declareGetRoute, declareDeleteRoute, @@ -30,6 +31,7 @@ import { declarePostUpdateByQueryRoute, declarePassthroughRoute, setProxyInterrupt, + allCombinationsPermutations, } from './repository_with_proxy_utils'; let esServer: TestElasticsearchUtils; @@ -98,17 +100,24 @@ describe('404s from proxies', () => { await hapiServer.register(h2o2); // register specific routes to modify the response and a catch-all to relay the request/response as-is - declareGetRoute(hapiServer, esHostname, esPort); - declareDeleteRoute(hapiServer, esHostname, esPort); - declarePostUpdateRoute(hapiServer, esHostname, esPort); + allCombinationsPermutations( + SavedObjectsIndexPatterns.map((indexPattern) => `${indexPattern}_${pkg.version}`) + ) + .map((indices) => indices.join(',')) + .forEach((kbnIndexPath) => { + declareGetRoute(hapiServer, esHostname, esPort, kbnIndexPath); + declareDeleteRoute(hapiServer, esHostname, esPort, kbnIndexPath); + declarePostUpdateRoute(hapiServer, esHostname, esPort, kbnIndexPath); + + declareGetSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath); + declarePostSearchRoute(hapiServer, esHostname, esPort, kbnIndexPath); + declarePostPitRoute(hapiServer, esHostname, esPort, kbnIndexPath); + declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort, kbnIndexPath); + }); - declareGetSearchRoute(hapiServer, esHostname, esPort); - declarePostSearchRoute(hapiServer, esHostname, esPort); + // register index-agnostic routes declarePostBulkRoute(hapiServer, esHostname, esPort); declarePostMgetRoute(hapiServer, esHostname, esPort); - declarePostPitRoute(hapiServer, esHostname, esPort); - declarePostUpdateByQueryRoute(hapiServer, esHostname, esPort); - declarePassthroughRoute(hapiServer, esHostname, esPort); await hapiServer.start(); diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts index 499d0d01d9de1..35b6b37b9c413 100644 --- a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts +++ b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts @@ -7,7 +7,6 @@ */ import Hapi from '@hapi/hapi'; import { IncomingMessage } from 'http'; -import { kibanaPackageJson as pkg } from '@kbn/repo-info'; // proxy setup const defaultProxyOptions = (hostname: string, port: string) => ({ @@ -52,10 +51,13 @@ const proxyOnResponseHandler = async (res: IncomingMessage, h: Hapi.ResponseTool .code(404); }; -const kbnIndex = `.kibana_${pkg.version}`; - // GET /.kibana_8.0.0/_doc/{type*} route (repository.get calls) -export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port: string) => +export const declareGetRoute = ( + hapiServer: Hapi.Server, + hostname: string, + port: string, + kbnIndex: string +) => hapiServer.route({ method: 'GET', path: `/${kbnIndex}/_doc/{type*}`, @@ -70,7 +72,12 @@ export const declareGetRoute = (hapiServer: Hapi.Server, hostname: string, port: }, }); // DELETE /.kibana_8.0.0/_doc/{type*} route (repository.delete calls) -export const declareDeleteRoute = (hapiServer: Hapi.Server, hostname: string, port: string) => +export const declareDeleteRoute = ( + hapiServer: Hapi.Server, + hostname: string, + port: string, + kbnIndex: string +) => hapiServer.route({ method: 'DELETE', path: `/${kbnIndex}/_doc/{_id*}`, @@ -133,7 +140,12 @@ export const declarePostMgetRoute = (hapiServer: Hapi.Server, hostname: string, }, }); // GET _search route -export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) => +export const declareGetSearchRoute = ( + hapiServer: Hapi.Server, + hostname: string, + port: string, + kbnIndex: string +) => hapiServer.route({ method: 'GET', path: `/${kbnIndex}/_search`, @@ -149,7 +161,12 @@ export const declareGetSearchRoute = (hapiServer: Hapi.Server, hostname: string, }, }); // POST _search route (`find` calls) -export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string, port: string) => +export const declarePostSearchRoute = ( + hapiServer: Hapi.Server, + hostname: string, + port: string, + kbnIndex: string +) => hapiServer.route({ method: 'POST', path: `/${kbnIndex}/_search`, @@ -168,7 +185,12 @@ export const declarePostSearchRoute = (hapiServer: Hapi.Server, hostname: string }, }); // POST _update -export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string, port: string) => +export const declarePostUpdateRoute = ( + hapiServer: Hapi.Server, + hostname: string, + port: string, + kbnIndex: string +) => hapiServer.route({ method: 'POST', path: `/${kbnIndex}/_update/{_id*}`, @@ -187,7 +209,12 @@ export const declarePostUpdateRoute = (hapiServer: Hapi.Server, hostname: string }, }); // POST _pit -export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, port: string) => +export const declarePostPitRoute = ( + hapiServer: Hapi.Server, + hostname: string, + port: string, + kbnIndex: string +) => hapiServer.route({ method: 'POST', path: `/${kbnIndex}/_pit`, @@ -209,7 +236,8 @@ export const declarePostPitRoute = (hapiServer: Hapi.Server, hostname: string, p export const declarePostUpdateByQueryRoute = ( hapiServer: Hapi.Server, hostname: string, - port: string + port: string, + kbnIndex: string ) => hapiServer.route({ method: 'POST', @@ -244,3 +272,22 @@ export const declarePassthroughRoute = (hapiServer: Hapi.Server, hostname: strin }, }, }); + +export function allCombinationsPermutations(collection: T[]): T[][] { + const recur = (subcollection: T[], size: number): T[][] => { + if (size <= 0) { + return [[]]; + } + const permutations: T[][] = []; + subcollection.forEach((value, index, array) => { + array = array.slice(); + array.splice(index, 1); + recur(array, size - 1).forEach((permutation) => { + permutation.unshift(value); + permutations.push(permutation); + }); + }); + return permutations; + }; + return collection.map((_, n) => recur(collection, n + 1)).flat(); +} diff --git a/x-pack/plugins/cases/common/constants/index.ts b/x-pack/plugins/cases/common/constants/index.ts index b0bf34eef6c29..0ef2623554ecd 100644 --- a/x-pack/plugins/cases/common/constants/index.ts +++ b/x-pack/plugins/cases/common/constants/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { SavedObjectsIndexPattern } from '@kbn/core-saved-objects-server'; import type { CasesFeaturesAllRequired } from '../ui/types'; export * from './owners'; @@ -18,6 +19,7 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz' as const; * Saved objects */ +export const CASES_INDEX: SavedObjectsIndexPattern = '.kibana_cases'; export const CASE_SAVED_OBJECT = 'cases' as const; export const CASE_CONNECTOR_MAPPINGS_SAVED_OBJECT = 'cases-connector-mappings' as const; export const CASE_USER_ACTION_SAVED_OBJECT = 'cases-user-actions' as const; diff --git a/x-pack/plugins/cases/server/saved_object_types/cases.ts b/x-pack/plugins/cases/server/saved_object_types/cases.ts index 1d70808f14db2..f9a7fbc7a773a 100644 --- a/x-pack/plugins/cases/server/saved_object_types/cases.ts +++ b/x-pack/plugins/cases/server/saved_object_types/cases.ts @@ -12,7 +12,7 @@ import type { SavedObjectsExportTransformContext, SavedObjectsType, } from '@kbn/core/server'; -import { CASE_SAVED_OBJECT } from '../../common/constants'; +import { CASES_INDEX, CASE_SAVED_OBJECT } from '../../common/constants'; import type { ESCaseAttributes } from '../services/cases/types'; import { handleExport } from './import_export/export'; import { caseMigrations } from './migrations'; @@ -22,6 +22,7 @@ export const createCaseSavedObjectType = ( logger: Logger ): SavedObjectsType => ({ name: CASE_SAVED_OBJECT, + indexPattern: CASES_INDEX, hidden: true, namespaceType: 'multiple-isolated', convertToMultiNamespaceTypeVersion: '8.0.0', diff --git a/x-pack/plugins/cases/server/saved_object_types/comments.ts b/x-pack/plugins/cases/server/saved_object_types/comments.ts index 071f26b1e2bad..ad61d8bfc2e7f 100644 --- a/x-pack/plugins/cases/server/saved_object_types/comments.ts +++ b/x-pack/plugins/cases/server/saved_object_types/comments.ts @@ -6,7 +6,7 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; -import { CASE_COMMENT_SAVED_OBJECT } from '../../common/constants'; +import { CASES_INDEX, CASE_COMMENT_SAVED_OBJECT } from '../../common/constants'; import type { CreateCommentsMigrationsDeps } from './migrations'; import { createCommentsMigrations } from './migrations'; @@ -21,6 +21,7 @@ export const createCaseCommentSavedObjectType = ({ migrationDeps: CreateCommentsMigrationsDeps; }): SavedObjectsType => ({ name: CASE_COMMENT_SAVED_OBJECT, + indexPattern: CASES_INDEX, hidden: true, namespaceType: 'multiple-isolated', convertToMultiNamespaceTypeVersion: '8.0.0', diff --git a/x-pack/plugins/cases/server/saved_object_types/configure.ts b/x-pack/plugins/cases/server/saved_object_types/configure.ts index cd8b98daa3c94..e5834b3da74ea 100644 --- a/x-pack/plugins/cases/server/saved_object_types/configure.ts +++ b/x-pack/plugins/cases/server/saved_object_types/configure.ts @@ -6,7 +6,7 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; -import { CASE_CONFIGURE_SAVED_OBJECT } from '../../common/constants'; +import { CASES_INDEX, CASE_CONFIGURE_SAVED_OBJECT } from '../../common/constants'; import { configureMigrations } from './migrations'; /** @@ -16,6 +16,7 @@ import { configureMigrations } from './migrations'; export const caseConfigureSavedObjectType: SavedObjectsType = { name: CASE_CONFIGURE_SAVED_OBJECT, + indexPattern: CASES_INDEX, hidden: true, namespaceType: 'multiple-isolated', convertToMultiNamespaceTypeVersion: '8.0.0', diff --git a/x-pack/plugins/cases/server/saved_object_types/connector_mappings.ts b/x-pack/plugins/cases/server/saved_object_types/connector_mappings.ts index 81f9e1ca6f2f1..4393a6457520a 100644 --- a/x-pack/plugins/cases/server/saved_object_types/connector_mappings.ts +++ b/x-pack/plugins/cases/server/saved_object_types/connector_mappings.ts @@ -6,7 +6,7 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; -import { CASE_CONNECTOR_MAPPINGS_SAVED_OBJECT } from '../../common/constants'; +import { CASES_INDEX, CASE_CONNECTOR_MAPPINGS_SAVED_OBJECT } from '../../common/constants'; import { connectorMappingsMigrations } from './migrations'; /** @@ -16,6 +16,7 @@ import { connectorMappingsMigrations } from './migrations'; export const caseConnectorMappingsSavedObjectType: SavedObjectsType = { name: CASE_CONNECTOR_MAPPINGS_SAVED_OBJECT, + indexPattern: CASES_INDEX, hidden: true, namespaceType: 'multiple-isolated', convertToMultiNamespaceTypeVersion: '8.0.0', diff --git a/x-pack/plugins/cases/server/saved_object_types/telemetry.ts b/x-pack/plugins/cases/server/saved_object_types/telemetry.ts index 515d1e63c7858..216c151bc6737 100644 --- a/x-pack/plugins/cases/server/saved_object_types/telemetry.ts +++ b/x-pack/plugins/cases/server/saved_object_types/telemetry.ts @@ -6,10 +6,11 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; -import { CASE_TELEMETRY_SAVED_OBJECT } from '../../common/constants'; +import { CASES_INDEX, CASE_TELEMETRY_SAVED_OBJECT } from '../../common/constants'; export const casesTelemetrySavedObjectType: SavedObjectsType = { name: CASE_TELEMETRY_SAVED_OBJECT, + indexPattern: CASES_INDEX, hidden: false, namespaceType: 'agnostic', mappings: { diff --git a/x-pack/plugins/cases/server/saved_object_types/user_actions.ts b/x-pack/plugins/cases/server/saved_object_types/user_actions.ts index 60180595999aa..f3f14661d1fda 100644 --- a/x-pack/plugins/cases/server/saved_object_types/user_actions.ts +++ b/x-pack/plugins/cases/server/saved_object_types/user_actions.ts @@ -6,7 +6,7 @@ */ import type { SavedObjectsType } from '@kbn/core/server'; -import { CASE_USER_ACTION_SAVED_OBJECT } from '../../common/constants'; +import { CASES_INDEX, CASE_USER_ACTION_SAVED_OBJECT } from '../../common/constants'; import type { UserActionsMigrationsDeps } from './migrations/user_actions'; import { createUserActionsMigrations } from './migrations/user_actions'; @@ -19,6 +19,7 @@ export const createCaseUserActionSavedObjectType = ( migrationDeps: UserActionsMigrationsDeps ): SavedObjectsType => ({ name: CASE_USER_ACTION_SAVED_OBJECT, + indexPattern: CASES_INDEX, hidden: true, namespaceType: 'multiple-isolated', convertToMultiNamespaceTypeVersion: '8.0.0', diff --git a/x-pack/test/api_integration/apis/security/index_fields.ts b/x-pack/test/api_integration/apis/security/index_fields.ts index 621a183182348..f7b3768f643d8 100644 --- a/x-pack/test/api_integration/apis/security/index_fields.ts +++ b/x-pack/test/api_integration/apis/security/index_fields.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -24,7 +25,7 @@ export default function ({ getService }: FtrProviderContext) { describe('GET /internal/security/fields/{query}', () => { it('should return a list of available index mapping fields', async () => { await supertest - .get('/internal/security/fields/.kibana') + .get(`/internal/security/fields/${SavedObjectsIndexPatterns.join(',')}`) .set('kbn-xsrf', 'xxx') .send() .expect(200) diff --git a/x-pack/test/api_integration/apis/security_solution/utils.ts b/x-pack/test/api_integration/apis/security_solution/utils.ts index f5e65c6da3e7c..587bff5368364 100644 --- a/x-pack/test/api_integration/apis/security_solution/utils.ts +++ b/x-pack/test/api_integration/apis/security_solution/utils.ts @@ -9,6 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { TransportResult } from '@elastic/elasticsearch'; import type { Client } from '@elastic/elasticsearch'; import { JsonObject } from '@kbn/utility-types'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; export async function getSavedObjectFromES( es: Client, @@ -17,7 +18,7 @@ export async function getSavedObjectFromES( ): Promise, unknown>> { return await es.search( { - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { query: { bool: { diff --git a/x-pack/test/cases_api_integration/common/lib/api/index.ts b/x-pack/test/cases_api_integration/common/lib/api/index.ts index 3bf0c470b9ba2..00d02b1979bc3 100644 --- a/x-pack/test/cases_api_integration/common/lib/api/index.ts +++ b/x-pack/test/cases_api_integration/common/lib/api/index.ts @@ -46,6 +46,7 @@ import type { SavedObjectsRawDocSource } from '@kbn/core/server'; import { User } from '../authentication/types'; import { superUser } from '../authentication/users'; import { getSpaceUrlPrefix, setupAuth } from './helpers'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; export * from './attachments'; export * from './case'; @@ -190,7 +191,7 @@ export const deleteAllCaseItems = async (es: Client) => { export const deleteCasesUserActions = async (es: Client): Promise => { await es.deleteByQuery({ - index: '.kibana', + index: SavedObjectsIndexPatterns, q: 'type:cases-user-actions', wait_for_completion: true, refresh: true, @@ -201,7 +202,7 @@ export const deleteCasesUserActions = async (es: Client): Promise => { export const deleteCasesByESQuery = async (es: Client): Promise => { await es.deleteByQuery({ - index: '.kibana', + index: SavedObjectsIndexPatterns, q: 'type:cases', wait_for_completion: true, refresh: true, @@ -212,7 +213,7 @@ export const deleteCasesByESQuery = async (es: Client): Promise => { export const deleteComments = async (es: Client): Promise => { await es.deleteByQuery({ - index: '.kibana', + index: SavedObjectsIndexPatterns, q: 'type:cases-comments', wait_for_completion: true, refresh: true, @@ -223,7 +224,7 @@ export const deleteComments = async (es: Client): Promise => { export const deleteConfiguration = async (es: Client): Promise => { await es.deleteByQuery({ - index: '.kibana', + index: SavedObjectsIndexPatterns, q: 'type:cases-configure', wait_for_completion: true, refresh: true, @@ -234,7 +235,7 @@ export const deleteConfiguration = async (es: Client): Promise => { export const deleteMappings = async (es: Client): Promise => { await es.deleteByQuery({ - index: '.kibana', + index: SavedObjectsIndexPatterns, q: 'type:cases-connector-mappings', wait_for_completion: true, refresh: true, @@ -289,7 +290,7 @@ export const getConnectorMappingsFromES = async ({ es }: { es: Client }) => { unknown > = await es.search( { - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { query: { term: { @@ -319,7 +320,7 @@ export const getConfigureSavedObjectsFromES = async ({ es }: { es: Client }) => unknown > = await es.search( { - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { query: { term: { @@ -342,7 +343,7 @@ export const getCaseSavedObjectsFromES = async ({ es }: { es: Client }) => { unknown > = await es.search( { - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { query: { term: { @@ -724,7 +725,7 @@ export const getSOFromKibanaIndex = async ({ }) => { const esResponse = await es.get( { - index: '.kibana', + index: SavedObjectsIndexPatterns, id: `${soType}:${soId}`, }, { meta: true } diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts index 712ec4722d500..ace37645945a0 100644 --- a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts @@ -15,6 +15,7 @@ import { CaseStatuses, CommentType, } from '@kbn/cases-plugin/common/api'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { @@ -523,7 +524,7 @@ export default ({ getService }: FtrProviderContext): void => { */ const getAllCasesSortedByCreatedAtAsc = async () => { const cases = await es.search({ - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { size: 10000, sort: [{ 'cases.created_at': { unmapped_type: 'date', order: 'asc' } }], diff --git a/x-pack/test/common/lib/test_data_loader.ts b/x-pack/test/common/lib/test_data_loader.ts index b379d4b61e3ba..889797f6e92ef 100644 --- a/x-pack/test/common/lib/test_data_loader.ts +++ b/x-pack/test/common/lib/test_data_loader.ts @@ -6,6 +6,7 @@ */ import { LegacyUrlAlias } from '@kbn/core-saved-objects-base-server-internal'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import Fs from 'fs/promises'; import { FtrProviderContext } from '../ftr_provider_context'; @@ -176,7 +177,7 @@ export function getTestDataLoader({ getService }: Pick { await es.deleteByQuery({ - index: '.kibana', + index: SavedObjectsIndexPatterns, wait_for_completion: true, body: { // @ts-expect-error diff --git a/x-pack/test/spaces_api_integration/common/lib/space_test_utils.ts b/x-pack/test/spaces_api_integration/common/lib/space_test_utils.ts index b7e698244ee29..f3ca946e89529 100644 --- a/x-pack/test/spaces_api_integration/common/lib/space_test_utils.ts +++ b/x-pack/test/spaces_api_integration/common/lib/space_test_utils.ts @@ -6,6 +6,7 @@ */ import type { Client } from '@elastic/elasticsearch'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; export function getUrlPrefix(spaceId?: string) { @@ -39,7 +40,7 @@ export function getTestScenariosForSpace(spaceId: string) { export function getAggregatedSpaceData(es: Client, objectTypes: string[]) { return es.search({ - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { size: 0, runtime_mappings: { diff --git a/x-pack/test/spaces_api_integration/common/suites/delete.ts b/x-pack/test/spaces_api_integration/common/suites/delete.ts index a1c73125ede28..02ee0b0c5fd45 100644 --- a/x-pack/test/spaces_api_integration/common/suites/delete.ts +++ b/x-pack/test/spaces_api_integration/common/suites/delete.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import { SuperTest } from 'supertest'; import type { Client } from '@elastic/elasticsearch'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { getAggregatedSpaceData, getTestScenariosForSpace } from '../lib/space_test_utils'; import { MULTI_NAMESPACE_SAVED_OBJECT_TEST_CASES as CASES } from '../lib/saved_object_test_cases'; import { DescribeFn, TestDefinitionAuthentication } from '../lib/types'; @@ -105,7 +106,7 @@ export function deleteTestSuiteFactory(es: Client, esArchiver: any, supertest: S // Since Space 2 was deleted, any multi-namespace objects that existed in that space // are updated to remove it, and of those, any that don't exist in any space are deleted. const multiNamespaceResponse = await es.search>({ - index: '.kibana', + index: SavedObjectsIndexPatterns, size: 100, body: { query: { terms: { type: ['sharedtype'] } } }, }); diff --git a/x-pack/test/spaces_api_integration/common/suites/disable_legacy_url_aliases.ts b/x-pack/test/spaces_api_integration/common/suites/disable_legacy_url_aliases.ts index 4719d0e5164a6..002ed6c3e6515 100644 --- a/x-pack/test/spaces_api_integration/common/suites/disable_legacy_url_aliases.ts +++ b/x-pack/test/spaces_api_integration/common/suites/disable_legacy_url_aliases.ts @@ -6,12 +6,13 @@ */ import expect from '@kbn/expect'; -import { SuperTest } from 'supertest'; +import type { SuperTest } from 'supertest'; import type { Client } from '@elastic/elasticsearch'; import type { LegacyUrlAlias } from '@kbn/core-saved-objects-base-server-internal'; +import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { SPACES } from '../lib/spaces'; import { getUrlPrefix } from '../../../saved_object_api_integration/common/lib/saved_object_test_utils'; -import { +import type { ExpectResponseBody, TestDefinition, TestSuite, @@ -62,7 +63,8 @@ export function disableLegacyUrlAliasesTestSuiteFactory( } const esResponse = await es.get( { - index: '.kibana', + // affected by the .kibana split, assumes LEGACY_URL_ALIAS_TYPE is stored in .kibana + index: MAIN_SAVED_OBJECT_INDEX, id: `${LEGACY_URL_ALIAS_TYPE}:${targetSpace}:${targetType}:${sourceId}`, }, { ignore: [404] } diff --git a/x-pack/test/spaces_api_integration/common/suites/update_objects_spaces.ts b/x-pack/test/spaces_api_integration/common/suites/update_objects_spaces.ts index 5528ceaa27602..3895842844e40 100644 --- a/x-pack/test/spaces_api_integration/common/suites/update_objects_spaces.ts +++ b/x-pack/test/spaces_api_integration/common/suites/update_objects_spaces.ts @@ -14,6 +14,7 @@ import { SavedObjectsErrorHelpers, SavedObjectsUpdateObjectsSpacesResponse, } from '@kbn/core/server'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { SPACES } from '../lib/spaces'; import { expectResponses, @@ -98,11 +99,11 @@ export function updateObjectsSpacesTestSuiteFactory( if (expectAliasDifference !== undefined) { // if we deleted an object that had an alias pointing to it, the alias should have been deleted as well if (!hasRefreshed) { - await es.indices.refresh({ index: '.kibana' }); // alias deletion uses refresh: false, so we need to manually refresh the index before searching + await es.indices.refresh({ index: SavedObjectsIndexPatterns }); // alias deletion uses refresh: false, so we need to manually refresh the index before searching hasRefreshed = true; } const searchResponse = await es.search({ - index: '.kibana', + index: SavedObjectsIndexPatterns, body: { size: 0, query: { terms: { type: ['legacy-url-alias'] } }, From 03bdb74a502a11b5561ffd766b83d1f4f9a06956 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 5 Apr 2023 11:50:13 +0000 Subject: [PATCH 07/17] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- x-pack/test/cases_api_integration/common/lib/api/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/cases_api_integration/common/lib/api/index.ts b/x-pack/test/cases_api_integration/common/lib/api/index.ts index 00d02b1979bc3..f41130b598752 100644 --- a/x-pack/test/cases_api_integration/common/lib/api/index.ts +++ b/x-pack/test/cases_api_integration/common/lib/api/index.ts @@ -43,10 +43,10 @@ import { ActionResult } from '@kbn/actions-plugin/server/types'; import { ESCasesConfigureAttributes } from '@kbn/cases-plugin/server/services/configure/types'; import { ESCaseAttributes } from '@kbn/cases-plugin/server/services/cases/types'; import type { SavedObjectsRawDocSource } from '@kbn/core/server'; +import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { User } from '../authentication/types'; import { superUser } from '../authentication/users'; import { getSpaceUrlPrefix, setupAuth } from './helpers'; -import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; export * from './attachments'; export * from './case'; From 05dea55d8548589de166ba661953d930fcfe996d Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Thu, 6 Apr 2023 10:12:39 +0200 Subject: [PATCH 08/17] Fix integration tests --- .../group1/7.7.2_xpack_100k.test.ts | 2 +- .../group3/split_kibana_index.test.ts | 23 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts index 6c77c786522d7..134436b6c9c82 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts @@ -107,7 +107,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { }; const migratedIndices = SavedObjectsIndexPatterns.map( - (indexPattern) => `.${indexPattern}_${kibanaVersion}_001` + (indexPattern) => `${indexPattern}_${kibanaVersion}_001` ); beforeAll(async () => { diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts index 8d7a37e599e40..bc4bc99195747 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts @@ -8,11 +8,7 @@ import Path from 'path'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import type { - ISavedObjectTypeRegistry, - SavedObjectsIndexPattern, - SavedObjectsType, -} from '@kbn/core-saved-objects-server'; +import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server'; import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; import { readLog, @@ -58,8 +54,7 @@ describe('split .kibana index into multiple system indices', () => { (type: SavedObjectsType) => { return { ...type, - indexPattern: (RELOCATE_TYPES[type.name] ?? - type.indexPattern) as SavedObjectsIndexPattern, + indexPattern: RELOCATE_TYPES[type.name] ?? type.indexPattern, }; } ); @@ -197,12 +192,6 @@ describe('split .kibana index into multiple system indices', () => { "lens", "lens-ui-telemetry", "map", - "cases-comments", - "cases-configure", - "cases-connector-mappings", - "cases", - "cases-user-actions", - "cases-telemetry", "slo", "ingest_manager_settings", "ingest-agent-policies", @@ -251,6 +240,14 @@ describe('split .kibana index into multiple system indices', () => { "apm-server-schema", "apm-service-group", ], + ".kibana_cases": Array [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry", + ], ".kibana_so_search": Array [ "search", ], From 3e3d4b9d043993b944f045c486b8c69161438d71 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Thu, 6 Apr 2023 10:13:03 +0200 Subject: [PATCH 09/17] Make `indexPattern: string` again --- packages/core/saved-objects/core-saved-objects-server/index.ts | 1 - .../src/saved_objects_index_pattern.ts | 1 - .../core-saved-objects-server/src/saved_objects_type.ts | 3 +-- x-pack/plugins/cases/common/constants/index.ts | 3 +-- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-server/index.ts b/packages/core/saved-objects/core-saved-objects-server/index.ts index 6ffb78b547389..e24edb73f4d59 100644 --- a/packages/core/saved-objects/core-saved-objects-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server/index.ts @@ -56,7 +56,6 @@ export { MAIN_SAVED_OBJECT_INDEX, TASK_MANAGER_SAVED_OBJECT_INDEX, SavedObjectsIndexPatterns, - type SavedObjectsIndexPattern, } from './src/saved_objects_index_pattern'; export type { SavedObjectsType, diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts index 7a79400f07b25..5eeeb541f0391 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_index_pattern.ts @@ -20,4 +20,3 @@ export const SavedObjectsIndexPatterns = [ TASK_MANAGER_SAVED_OBJECT_INDEX, `${MAIN_SAVED_OBJECT_INDEX}_cases`, ]; -export type SavedObjectsIndexPattern = '.kibana' | '.kibana_task_manager' | '.kibana_cases'; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts index fe0698c944677..b9fa8e6365114 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/saved_objects_type.ts @@ -18,7 +18,6 @@ import type { SavedObjectsModelVersionMap, SavedObjectsModelVersionMapProvider, } from './model_version'; -import type { SavedObjectsIndexPattern } from './saved_objects_index_pattern'; /** * @public @@ -53,7 +52,7 @@ export interface SavedObjectsType { /** * If defined, the type instances will be stored in the given index instead of the default one. */ - indexPattern?: SavedObjectsIndexPattern; + indexPattern?: string; /** * If defined, will be used to convert the type to an alias. */ diff --git a/x-pack/plugins/cases/common/constants/index.ts b/x-pack/plugins/cases/common/constants/index.ts index 0ef2623554ecd..95070a664ad0c 100644 --- a/x-pack/plugins/cases/common/constants/index.ts +++ b/x-pack/plugins/cases/common/constants/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { SavedObjectsIndexPattern } from '@kbn/core-saved-objects-server'; import type { CasesFeaturesAllRequired } from '../ui/types'; export * from './owners'; @@ -19,7 +18,7 @@ export const DEFAULT_DATE_FORMAT_TZ = 'dateFormat:tz' as const; * Saved objects */ -export const CASES_INDEX: SavedObjectsIndexPattern = '.kibana_cases'; +export const CASES_INDEX = '.kibana_cases'; export const CASE_SAVED_OBJECT = 'cases' as const; export const CASE_CONNECTOR_MAPPINGS_SAVED_OBJECT = 'cases-connector-mappings' as const; export const CASE_USER_ACTION_SAVED_OBJECT = 'cases-user-actions' as const; From be3541ee31cba07f51c587b15d0758a1117b45de Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Thu, 6 Apr 2023 10:29:10 +0200 Subject: [PATCH 10/17] Fix typo: involed => involved --- .../src/kibana_migrator.ts | 4 +-- .../src/kibana_migrator_utils.test.ts | 26 +++++++++---------- .../src/kibana_migrator_utils.ts | 2 +- .../group3/split_kibana_index.test.ts | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index 77c47c0f7e8fc..579133d721306 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -32,7 +32,7 @@ import { type MigrationResult, type IndexTypesMap, } from '@kbn/core-saved-objects-base-server-internal'; -import { getIndicesInvoledInRelocation } from './kibana_migrator_utils'; +import { getIndicesInvolvedInRelocation } from './kibana_migrator_utils'; import { buildActiveMappings, buildTypesMappings } from './core'; import { DocumentMigrator } from './document_migrator'; import { createIndexMap } from './core/build_index_map'; @@ -189,7 +189,7 @@ export class KibanaMigrator implements IKibanaMigrator { // compare indexTypesMap with the one present (or not) in the .kibana index meta // and check if some SO types have been moved to different indices - const indicesWithMovingTypes = await getIndicesInvoledInRelocation({ + const indicesWithMovingTypes = await getIndicesInvolvedInRelocation({ mainIndex: MAIN_SAVED_OBJECT_INDEX, client: this.client, indexTypesMap, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts index fe24cf36f788f..8927b0e82f8aa 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts @@ -14,7 +14,7 @@ import { DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants'; import { calculateTypeStatuses, createMultiPromiseDefer, - getIndicesInvoledInRelocation, + getIndicesInvolvedInRelocation, indexMapToIndexTypesMap, } from './kibana_migrator_utils'; import { INDEX_MAP_BEFORE_SPLIT } from './kibana_migrator_utils.fixtures'; @@ -41,8 +41,8 @@ describe('createMultiPromiseDefer', () => { }); }); -describe('getIndicesInvoledInRelocation', () => { - const getIndicesInvoledInRelocationParams = () => { +describe('getIndicesInvolvedInRelocation', () => { + const getIndicesInvolvedInRelocationParams = () => { const client = elasticsearchClientMock.createElasticsearchClient(); (client as any).child = jest.fn().mockImplementation(() => client); @@ -56,9 +56,9 @@ describe('getIndicesInvoledInRelocation', () => { }; it('tries to get the indexTypesMap from the mainIndex', async () => { - const params = getIndicesInvoledInRelocationParams(); + const params = getIndicesInvolvedInRelocationParams(); try { - await getIndicesInvoledInRelocation(params); + await getIndicesInvolvedInRelocation(params); } catch (err) { // ignore } @@ -70,7 +70,7 @@ describe('getIndicesInvoledInRelocation', () => { }); it('fails if the query to get indexTypesMap fails with critical error', async () => { - const params = getIndicesInvoledInRelocationParams(); + const params = getIndicesInvolvedInRelocationParams(); params.client.indices.getMapping.mockImplementation(() => elasticsearchClientMock.createErrorTransportRequestPromise( new errors.ResponseError({ @@ -87,13 +87,13 @@ describe('getIndicesInvoledInRelocation', () => { }) ) ); - expect(getIndicesInvoledInRelocation(params)).rejects.toThrowErrorMatchingInlineSnapshot( + expect(getIndicesInvolvedInRelocation(params)).rejects.toThrowErrorMatchingInlineSnapshot( `"error_type"` ); }); it('assumes fresh deployment if the mainIndex does not exist, returns an empty list of moving types', async () => { - const params = getIndicesInvoledInRelocationParams(); + const params = getIndicesInvolvedInRelocationParams(); params.client.indices.getMapping.mockImplementation(() => elasticsearchClientMock.createErrorTransportRequestPromise( new errors.ResponseError({ @@ -110,13 +110,13 @@ describe('getIndicesInvoledInRelocation', () => { }) ) ); - expect(getIndicesInvoledInRelocation(params)).resolves.toEqual([]); + expect(getIndicesInvolvedInRelocation(params)).resolves.toEqual([]); }); describe('if mainIndex exists', () => { describe('but it does not have an indexTypeMap stored', () => { it('uses the defaultIndexTypeMap and finds out which indices are involved in a relocation', async () => { - const params = getIndicesInvoledInRelocationParams(); + const params = getIndicesInvolvedInRelocationParams(); params.client.indices.getMapping.mockReturnValue( Promise.resolve({ '.kibana_8.7.0_001': { @@ -145,13 +145,13 @@ describe('getIndicesInvoledInRelocation', () => { '.indexC': ['type2', 'type3'], }; - expect(getIndicesInvoledInRelocation(params)).resolves.toEqual(['.indexA', '.indexC']); + expect(getIndicesInvolvedInRelocation(params)).resolves.toEqual(['.indexA', '.indexC']); }); }); describe('and it has an indexTypeMap stored', () => { it('compares stored indexTypeMap against desired one, and finds out which indices are involved in a relocation', async () => { - const params = getIndicesInvoledInRelocationParams(); + const params = getIndicesInvolvedInRelocationParams(); params.client.indices.getMapping.mockReturnValue( Promise.resolve({ '.kibana_8.8.0_001': { @@ -189,7 +189,7 @@ describe('getIndicesInvoledInRelocation', () => { '.indexD': ['type5', 'type6'], }; - expect(getIndicesInvoledInRelocation(params)).resolves.toEqual(['.indexB', '.indexD']); + expect(getIndicesInvolvedInRelocation(params)).resolves.toEqual(['.indexB', '.indexD']); }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts index 029b6b9da55e7..55551bc5225de 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts @@ -66,7 +66,7 @@ export async function getCurrentIndexTypesMap({ } } -export async function getIndicesInvoledInRelocation({ +export async function getIndicesInvolvedInRelocation({ client, mainIndex, indexTypesMap, diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts index 6447a70802863..932749e7313a1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts @@ -275,7 +275,7 @@ describe('split .kibana index into multiple system indices', () => { expect(logs).toMatch(`[${newIndex}] READY_TO_REINDEX_SYNC -> DONE_REINDEXING_SYNC.`); }); - // the .kibana migrator is involed in a relocation, it must also reindex + // the .kibana migrator is involved in a relocation, it must also reindex expect(logs).toMatch('[.kibana] INIT -> WAIT_FOR_YELLOW_SOURCE.'); expect(logs).toMatch('[.kibana] WAIT_FOR_YELLOW_SOURCE -> CHECK_UNKNOWN_DOCUMENTS.'); expect(logs).toMatch('[.kibana] CHECK_UNKNOWN_DOCUMENTS -> SET_SOURCE_WRITE_BLOCK.'); From 087742d77c753fe90b28e608ff4c3bd5e395bfa5 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 11 Apr 2023 10:38:46 +0200 Subject: [PATCH 11/17] Address some PR remarks --- .../src/kibana_migrator.test.ts | 6 +++--- .../src/kibana_migrator_utils.ts | 15 +++++++-------- .../src/migrations_state_action_machine.ts | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index f394d4866b54e..c86001c2d202d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -134,7 +134,8 @@ describe('KibanaMigrator', () => { ); }); - it('only runs migrations once if called multiple times', async () => { + // eslint-disable-next-line jest/no-focused-tests + fit('only runs migrations once if called multiple times', async () => { const options = mockOptions(); options.client.indices.get.mockResponse({}, { statusCode: 200 }); options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { @@ -234,8 +235,7 @@ describe('KibanaMigrator', () => { describe('for V2 migrations', () => { describe('where some SO types must be relocated', () => { - // eslint-disable-next-line jest/no-focused-tests - fit('runs successfully', async () => { + it('runs successfully', async () => { const options = mockV2MigrationOptions(); options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { statusCode: 200, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts index 55551bc5225de..feda988edc458 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts @@ -26,7 +26,7 @@ export class Defer { export const defer = () => new Defer(); export function createMultiPromiseDefer(indices: string[]): Record> { - const defers: Array> = new Array(indices.length).fill(true).map(defer); + const defers: Array> = indices.map(defer); const all = Promise.all(defers.map(({ promise }) => promise)); return indices.reduce>>((acc, indexName, i) => { const { resolve, reject } = defers[i]; @@ -95,13 +95,12 @@ export async function getIndicesInvolvedInRelocation({ const typeIndexDistribution = calculateTypeStatuses(currentIndexTypesMap, indexTypesMap); - const relocated = Object.entries(typeIndexDistribution).filter( - ([, { status }]) => status === TypeStatus.Moved - ); - relocated.forEach(([, { currentIndex, targetIndex }]) => { - indicesWithMovingTypesSet.add(currentIndex!); - indicesWithMovingTypesSet.add(targetIndex!); - }); + Object.values(typeIndexDistribution) + .filter(({ status }) => status === TypeStatus.Moved) + .forEach(({ currentIndex, targetIndex }) => { + indicesWithMovingTypesSet.add(currentIndex!); + indicesWithMovingTypesSet.add(targetIndex!); + }); return Array.from(indicesWithMovingTypesSet); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts index 5bd76c80797e4..f96c52df52ab7 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts @@ -113,7 +113,7 @@ export async function migrationStateActionMachine({ try { await abort(finalState); } catch (e) { - logger.warn('Failed to correctly abort migrations:', e.message); + logger.warn('Failed to cleanup after migrations:', e.message); } const errorMessage = @@ -134,7 +134,7 @@ export async function migrationStateActionMachine({ try { await abort(lastState); } catch (err) { - logger.warn('Failed to correctly abort migrations:', err.message); + logger.warn('Failed to cleanup after migrations:', err.message); } if (e instanceof EsErrors.ResponseError) { // Log the failed request. This is very similar to the From 6483764f188abe3592e7ba3df7fee69f11e8eb5b Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 11 Apr 2023 11:15:02 +0200 Subject: [PATCH 12/17] Fix incorrect call to esClient.get(); use .kibana_cases index --- .../common/lib/api/index.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/test/cases_api_integration/common/lib/api/index.ts b/x-pack/test/cases_api_integration/common/lib/api/index.ts index f41130b598752..9bd93f65f77b5 100644 --- a/x-pack/test/cases_api_integration/common/lib/api/index.ts +++ b/x-pack/test/cases_api_integration/common/lib/api/index.ts @@ -13,6 +13,7 @@ import { GetResponse } from '@elastic/elasticsearch/lib/api/types'; import type SuperTest from 'supertest'; import { + CASES_INDEX, CASES_INTERNAL_URL, CASES_URL, CASE_CONFIGURE_URL, @@ -43,7 +44,6 @@ import { ActionResult } from '@kbn/actions-plugin/server/types'; import { ESCasesConfigureAttributes } from '@kbn/cases-plugin/server/services/configure/types'; import { ESCaseAttributes } from '@kbn/cases-plugin/server/services/cases/types'; import type { SavedObjectsRawDocSource } from '@kbn/core/server'; -import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; import { User } from '../authentication/types'; import { superUser } from '../authentication/users'; import { getSpaceUrlPrefix, setupAuth } from './helpers'; @@ -191,7 +191,7 @@ export const deleteAllCaseItems = async (es: Client) => { export const deleteCasesUserActions = async (es: Client): Promise => { await es.deleteByQuery({ - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, q: 'type:cases-user-actions', wait_for_completion: true, refresh: true, @@ -202,7 +202,7 @@ export const deleteCasesUserActions = async (es: Client): Promise => { export const deleteCasesByESQuery = async (es: Client): Promise => { await es.deleteByQuery({ - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, q: 'type:cases', wait_for_completion: true, refresh: true, @@ -213,7 +213,7 @@ export const deleteCasesByESQuery = async (es: Client): Promise => { export const deleteComments = async (es: Client): Promise => { await es.deleteByQuery({ - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, q: 'type:cases-comments', wait_for_completion: true, refresh: true, @@ -224,7 +224,7 @@ export const deleteComments = async (es: Client): Promise => { export const deleteConfiguration = async (es: Client): Promise => { await es.deleteByQuery({ - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, q: 'type:cases-configure', wait_for_completion: true, refresh: true, @@ -235,7 +235,7 @@ export const deleteConfiguration = async (es: Client): Promise => { export const deleteMappings = async (es: Client): Promise => { await es.deleteByQuery({ - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, q: 'type:cases-connector-mappings', wait_for_completion: true, refresh: true, @@ -290,7 +290,7 @@ export const getConnectorMappingsFromES = async ({ es }: { es: Client }) => { unknown > = await es.search( { - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, body: { query: { term: { @@ -320,7 +320,7 @@ export const getConfigureSavedObjectsFromES = async ({ es }: { es: Client }) => unknown > = await es.search( { - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, body: { query: { term: { @@ -343,7 +343,7 @@ export const getCaseSavedObjectsFromES = async ({ es }: { es: Client }) => { unknown > = await es.search( { - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, body: { query: { term: { @@ -725,7 +725,7 @@ export const getSOFromKibanaIndex = async ({ }) => { const esResponse = await es.get( { - index: SavedObjectsIndexPatterns, + index: CASES_INDEX, id: `${soType}:${soId}`, }, { meta: true } From 2046c2778a1b04a015d6ed588dbf49e776c18e83 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Tue, 11 Apr 2023 13:39:34 +0200 Subject: [PATCH 13/17] Update mappings with .kibana_ to fake that reindex split is complete --- .../delete_unknown_types/mappings.json | 105 +++++++++++++++++ .../deprecations_service/mappings.json | 105 +++++++++++++++++ .../export_transform/mappings.json | 105 +++++++++++++++++ .../hidden_saved_objects/mappings.json | 105 +++++++++++++++++ .../nested_export_transform/mappings.json | 105 +++++++++++++++++ .../visible_in_management/mappings.json | 105 +++++++++++++++++ .../saved_objects/spaces/mappings.json | 107 +++++++++++++++++ .../saved_objects/spaces/mappings.json | 109 +++++++++++++++++- 8 files changed, 845 insertions(+), 1 deletion(-) diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/mappings.json index fb2337c15216c..7ed55c8e2bd36 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/mappings.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/delete_unknown_types/mappings.json @@ -37,6 +37,111 @@ "url": "c7f66a0df8b1b52f17c28c4adb111105", "usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4", "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + }, + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] } }, "dynamic": "strict", diff --git a/test/functional/fixtures/es_archiver/deprecations_service/mappings.json b/test/functional/fixtures/es_archiver/deprecations_service/mappings.json index 47bd39ebf8bcf..83ca46d08e118 100644 --- a/test/functional/fixtures/es_archiver/deprecations_service/mappings.json +++ b/test/functional/fixtures/es_archiver/deprecations_service/mappings.json @@ -36,6 +36,111 @@ "url": "c7f66a0df8b1b52f17c28c4adb111105", "usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4", "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + }, + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] } }, "dynamic": "strict", diff --git a/test/functional/fixtures/es_archiver/saved_objects_management/export_transform/mappings.json b/test/functional/fixtures/es_archiver/saved_objects_management/export_transform/mappings.json index 84628ef0366fa..64a6b167cb9e3 100644 --- a/test/functional/fixtures/es_archiver/saved_objects_management/export_transform/mappings.json +++ b/test/functional/fixtures/es_archiver/saved_objects_management/export_transform/mappings.json @@ -36,6 +36,111 @@ "url": "c7f66a0df8b1b52f17c28c4adb111105", "usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4", "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + }, + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] } }, "dynamic": "strict", diff --git a/test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects/mappings.json b/test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects/mappings.json index 43711680a1f12..f79ae3ae71d16 100644 --- a/test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects/mappings.json +++ b/test/functional/fixtures/es_archiver/saved_objects_management/hidden_saved_objects/mappings.json @@ -36,6 +36,111 @@ "url": "c7f66a0df8b1b52f17c28c4adb111105", "usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4", "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + }, + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] } }, "dynamic": "strict", diff --git a/test/functional/fixtures/es_archiver/saved_objects_management/nested_export_transform/mappings.json b/test/functional/fixtures/es_archiver/saved_objects_management/nested_export_transform/mappings.json index 84628ef0366fa..64a6b167cb9e3 100644 --- a/test/functional/fixtures/es_archiver/saved_objects_management/nested_export_transform/mappings.json +++ b/test/functional/fixtures/es_archiver/saved_objects_management/nested_export_transform/mappings.json @@ -36,6 +36,111 @@ "url": "c7f66a0df8b1b52f17c28c4adb111105", "usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4", "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + }, + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] } }, "dynamic": "strict", diff --git a/test/functional/fixtures/es_archiver/saved_objects_management/visible_in_management/mappings.json b/test/functional/fixtures/es_archiver/saved_objects_management/visible_in_management/mappings.json index 6ffcd71f29e50..3b078d3f905fd 100644 --- a/test/functional/fixtures/es_archiver/saved_objects_management/visible_in_management/mappings.json +++ b/test/functional/fixtures/es_archiver/saved_objects_management/visible_in_management/mappings.json @@ -36,6 +36,111 @@ "url": "c7f66a0df8b1b52f17c28c4adb111105", "usage-counters": "8cc260bdceffec4ffc3ad165c97dc1b4", "visualization": "f819cf6636b75c9e76ba733a0c6ef355" + }, + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] } }, "dynamic": "strict", diff --git a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json index 34ddbc9ac7846..d33a852644a0b 100644 --- a/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json +++ b/x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json @@ -8,6 +8,113 @@ "index": ".kibana_$KIBANA_PACKAGE_VERSION_001", "mappings": { "dynamic": "false", + "_meta": { + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] + } + }, "properties": { "space": { "dynamic": false, diff --git a/x-pack/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json b/x-pack/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json index 3f837ae5fa208..afc7a29ac6738 100644 --- a/x-pack/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json +++ b/x-pack/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/mappings.json @@ -7,7 +7,114 @@ }, "index": ".kibana_$KIBANA_PACKAGE_VERSION_001", "mappings": { - "dynamic": "false" + "dynamic": "false", + "_meta": { + "indexTypesMap": { + ".kibana_task_manager": [ + "task" + ], + ".kibana": [ + "core-usage-stats", + "legacy-url-alias", + "config", + "config-global", + "usage-counters", + "guided-onboarding-guide-state", + "guided-onboarding-plugin-state", + "ui-metric", + "application_usage_totals", + "application_usage_daily", + "event_loop_delays_daily", + "url", + "index-pattern", + "sample-data-telemetry", + "space", + "spaces-usage-stats", + "exception-list-agnostic", + "exception-list", + "telemetry", + "file", + "fileShare", + "action", + "action_task_params", + "connector_token", + "query", + "kql-telemetry", + "search-session", + "search-telemetry", + "file-upload-usage-collection-telemetry", + "alert", + "api_key_pending_invalidation", + "rules-settings", + "search", + "tag", + "graph-workspace", + "visualization", + "canvas-element", + "canvas-workpad", + "canvas-workpad-template", + "dashboard", + "lens", + "lens-ui-telemetry", + "map", + "slo", + "ingest_manager_settings", + "ingest-agent-policies", + "ingest-outputs", + "ingest-package-policies", + "epm-packages", + "epm-packages-assets", + "fleet-preconfiguration-deletion-record", + "ingest-download-sources", + "fleet-fleet-server-host", + "fleet-proxy", + "fleet-message-signing-keys", + "osquery-manager-usage-metric", + "osquery-saved-query", + "osquery-pack", + "osquery-pack-asset", + "csp-rule-template", + "ml-job", + "ml-trained-model", + "ml-module", + "uptime-dynamic-settings", + "synthetics-privates-locations", + "synthetics-monitor", + "uptime-synthetics-api-key", + "synthetics-param", + "siem-ui-timeline-note", + "siem-ui-timeline-pinned-event", + "siem-detection-engine-rule-actions", + "security-rule", + "siem-ui-timeline", + "endpoint:user-artifact", + "endpoint:user-artifact-manifest", + "security-solution-signals-migration", + "infrastructure-ui-source", + "metrics-explorer-view", + "inventory-view", + "infrastructure-monitoring-log-view", + "upgrade-assistant-reindex-operation", + "upgrade-assistant-ml-upgrade-operation", + "monitoring-telemetry", + "enterprise_search_telemetry", + "app_search_telemetry", + "workplace_search_telemetry", + "apm-indices", + "apm-telemetry", + "apm-server-schema", + "apm-service-group" + ], + ".kibana_cases": [ + "cases-comments", + "cases-configure", + "cases-connector-mappings", + "cases", + "cases-user-actions", + "cases-telemetry" + ] + } + } }, "settings": { "index": { From 4efdb10e570427154d2bb59c9d87f6c71605d3d4 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 11 Apr 2023 17:51:06 +0200 Subject: [PATCH 14/17] Address some of the PR remarks --- .../src/kibana_migrator.test.ts | 133 ++++++++++++------ .../src/kibana_migrator.ts | 2 +- .../group3/split_kibana_index.test.ts | 23 +-- .../migrations/kibana_migrator_test_kit.ts | 2 +- 4 files changed, 103 insertions(+), 57 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index c86001c2d202d..cd34f5848ccbb 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -20,17 +20,13 @@ import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import { lastValueFrom } from 'rxjs'; import { runResilientMigrator } from './run_resilient_migrator'; -jest.mock('./run_resilient_migrator', () => ({ - runResilientMigrator: jest.fn((options) => { - return { - status: 'migrated', - // TODO check the actual value of the MigrationResult for these fields - destIndex: `${options.indexPrefix}_8.2.3_001`, - sourceIndex: `${options.indexPrefix}_8.2.2_001`, - elapsedMs: 28, - }; - }), -})); +jest.mock('./run_resilient_migrator', () => { + const actual = jest.requireActual('./run_resilient_migrator'); + + return { + runResilientMigrator: jest.fn(actual.runResilientMigrator), + }; +}); jest.mock('./document_migrator', () => { return { @@ -89,7 +85,7 @@ describe('KibanaMigrator', () => { }, { name: 'bmap', - indexPattern: 'other-index', + indexPattern: '.other-index', mappings: { properties: { field: { type: 'text' } }, }, @@ -134,8 +130,22 @@ describe('KibanaMigrator', () => { ); }); - // eslint-disable-next-line jest/no-focused-tests - fit('only runs migrations once if called multiple times', async () => { + it('only runs migrations once if called multiple times', async () => { + const successfulRun: typeof runResilientMigrator = ({ indexPrefix }) => + Promise.resolve({ + sourceIndex: indexPrefix, + destIndex: indexPrefix, + elapsedMs: 28, + status: 'migrated', + }); + const mockRunResilientMigrator = runResilientMigrator as jest.MockedFunction< + typeof runResilientMigrator + >; + + mockRunResilientMigrator.mockImplementationOnce(successfulRun); + mockRunResilientMigrator.mockImplementationOnce(successfulRun); + mockRunResilientMigrator.mockImplementationOnce(successfulRun); + mockRunResilientMigrator.mockImplementationOnce(successfulRun); const options = mockOptions(); options.client.indices.get.mockResponse({}, { statusCode: 200 }); options.client.indices.getMapping.mockResponse(mappingsResponseWithoutIndexTypesMap, { @@ -157,7 +167,35 @@ describe('KibanaMigrator', () => { await migrator.runMigrations(); // indices.get is called twice during a single migration - expect(options.client.indices.get).toHaveBeenCalledTimes(2); + expect(runResilientMigrator).toHaveBeenCalledTimes(4); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + indexPrefix: '.my-index', + mustRelocateDocuments: true, + }) + ); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + indexPrefix: '.other-index', + mustRelocateDocuments: true, + }) + ); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + indexPrefix: '.my-task-index', + mustRelocateDocuments: false, + }) + ); + expect(runResilientMigrator).toHaveBeenNthCalledWith( + 4, + expect.objectContaining({ + indexPrefix: '.my-complementary-index', + mustRelocateDocuments: true, + }) + ); }); it('emits results on getMigratorResult$()', async () => { @@ -179,12 +217,12 @@ describe('KibanaMigrator', () => { status: 'migrated', }); expect(result![1]).toMatchObject({ - destIndex: 'other-index_8.2.3_001', + destIndex: '.other-index_8.2.3_001', elapsedMs: expect.any(Number), status: 'patched', }); }); - it('rejects when the migration state machine terminates in a FATAL state', () => { + it('rejects when the migration state machine terminates in a FATAL state', async () => { const options = mockV2MigrationOptions(); options.client.indices.get.mockResponse( { @@ -244,34 +282,37 @@ describe('KibanaMigrator', () => { const migrator = new KibanaMigrator(options); migrator.prepareMigrations(); const results = await migrator.runMigrations(); - expect(results).toMatchInlineSnapshot(` - Array [ - Object { - "destIndex": ".my-index_8.2.3_001", - "elapsedMs": 28, - "sourceIndex": ".my-index_8.2.2_001", - "status": "migrated", - }, - Object { - "destIndex": ".other-index_8.2.3_001", - "elapsedMs": 28, - "sourceIndex": ".other-index_8.2.2_001", - "status": "migrated", - }, - Object { - "destIndex": ".my-task-index_8.2.3_001", - "elapsedMs": 28, - "sourceIndex": ".my-task-index_8.2.2_001", - "status": "migrated", - }, - Object { - "destIndex": ".my-complementary-index_8.2.3_001", - "elapsedMs": 28, - "sourceIndex": ".my-complementary-index_8.2.2_001", - "status": "migrated", - }, - ] - `); + + expect(results.length).toEqual(4); + expect(results[0]).toEqual( + expect.objectContaining({ + sourceIndex: '.my-index_pre8.2.3_001', + destIndex: '.my-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'migrated', + }) + ); + expect(results[1]).toEqual( + expect.objectContaining({ + destIndex: '.other-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'patched', + }) + ); + expect(results[2]).toEqual( + expect.objectContaining({ + destIndex: '.my-task-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'patched', + }) + ); + expect(results[3]).toEqual( + expect.objectContaining({ + destIndex: '.my-complementary-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'patched', + }) + ); expect(runResilientMigrator).toHaveBeenCalledTimes(4); expect(runResilientMigrator).toHaveBeenNthCalledWith( @@ -472,7 +513,7 @@ const mockOptions = () => { name: 'testtype2', hidden: false, namespaceType: 'single', - // We are moving 'testtype2' from '.my-index' to '.my-other-index' + // We are moving 'testtype2' from '.my-index' to '.other-index' indexPattern: '.other-index', mappings: { properties: { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts index 579133d721306..d9b9f19b785e4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.ts @@ -142,7 +142,7 @@ export class KibanaMigrator implements IKibanaMigrator { private async runMigrationsInternal(): Promise { const migrationAlgorithm = this.soMigrationsConfig.algorithm; if (migrationAlgorithm === 'zdt') { - return this.runMigrationZdt(); + return await this.runMigrationZdt(); } else { return await this.runMigrationV2(); } diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts index 932749e7313a1..a9a5dfc2e7e85 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group3/split_kibana_index.test.ts @@ -9,7 +9,6 @@ import Path from 'path'; import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import type { ISavedObjectTypeRegistry, SavedObjectsType } from '@kbn/core-saved-objects-server'; -import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal'; import { readLog, startElasticsearch, @@ -19,6 +18,7 @@ import { clearLog, getAggregatedTypesCount, currentVersion, + type KibanaMigratorTestKit, } from '../kibana_migrator_test_kit'; import { delay } from '../test_utils'; @@ -32,8 +32,8 @@ const RELOCATE_TYPES: Record = { describe('split .kibana index into multiple system indices', () => { let esServer: TestElasticsearchUtils['es']; - let runMigrations: (rerun?: boolean | undefined) => Promise; let typeRegistry: ISavedObjectTypeRegistry; + let migratorTestKitFactory: () => Promise; beforeAll(async () => { typeRegistry = await getCurrentVersionTypeRegistry({ oss: false }); @@ -59,11 +59,13 @@ describe('split .kibana index into multiple system indices', () => { } ); - const { client, runMigrations: runner } = await getKibanaMigratorTestKit({ - types: updatedTypeRegistry.getAllTypes(), - kibanaIndex: '.kibana', - }); - runMigrations = runner; + migratorTestKitFactory = () => + getKibanaMigratorTestKit({ + types: updatedTypeRegistry.getAllTypes(), + kibanaIndex: '.kibana', + }); + + const { runMigrations, client } = await migratorTestKitFactory(); // count of types in the legacy index expect(await getAggregatedTypesCount(client, '.kibana_1')).toEqual({ @@ -82,7 +84,9 @@ describe('split .kibana index into multiple system indices', () => { await runMigrations(); - await client.indices.refresh({ index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'] }); + await client.indices.refresh({ + index: ['.kibana', '.kibana_so_search', '.kibana_so_ui'], + }); expect(await getAggregatedTypesCount(client, '.kibana')).toEqual({ 'index-pattern': 3, @@ -357,8 +361,9 @@ describe('split .kibana index into multiple system indices', () => { afterEach(async () => { // we run the migrator again to ensure that the next time state is loaded everything still works as expected + const { runMigrations } = await migratorTestKitFactory(); await clearLog(); - await runMigrations(true); + await runMigrations(); const logs = await readLog(); expect(logs).not.toMatch('REINDEX'); diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts index 2f57c3b9dc950..22e3fe218a495 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts @@ -72,7 +72,7 @@ export interface KibanaMigratorTestKitParams { export interface KibanaMigratorTestKit { client: ElasticsearchClient; migrator: IKibanaMigrator; - runMigrations: (rerun?: boolean) => Promise; + runMigrations: () => Promise; typeRegistry: ISavedObjectTypeRegistry; savedObjectsRepository: ISavedObjectsRepository; } From 97251760ac6b951ef00fdda85ea91e12ce6b0426 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 11 Apr 2023 18:49:46 +0200 Subject: [PATCH 15/17] Take into account targetMappings._meta when merging --- .../src/model/helpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts index 93748800fae04..c8d8884c980cd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/helpers.ts @@ -65,6 +65,7 @@ export function mergeMigrationMappingPropertyHashes( return { ...targetMappings, _meta: { + ...targetMappings._meta, migrationMappingPropertyHashes: { ...indexMappings._meta?.migrationMappingPropertyHashes, ...targetMappings._meta?.migrationMappingPropertyHashes, From 288c4b42d9af17b79f1eebe8feffb39865b56287 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 11 Apr 2023 22:11:07 +0200 Subject: [PATCH 16/17] Fix outdated integration test --- .../migrations/group1/7.7.2_xpack_100k.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts index 134436b6c9c82..61df54542b07d 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts @@ -106,10 +106,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { await new Promise((resolve) => setTimeout(resolve, 10000)); }; - const migratedIndices = SavedObjectsIndexPatterns.map( - (indexPattern) => `${indexPattern}_${kibanaVersion}_001` - ); - beforeAll(async () => { await removeLogFile(); await startServers({ @@ -124,7 +120,7 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { it('copies all the document of the previous index to the new one', async () => { const migratedIndexResponse = await esClient.count({ - index: migratedIndices, + index: SavedObjectsIndexPatterns, }); const oldIndexResponse = await esClient.count({ index: '.kibana_1', From 342c4005647b7ea7abe1f6e05835442f097760f4 Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Wed, 12 Apr 2023 09:17:36 +0200 Subject: [PATCH 17/17] remove unused var --- .../saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts index 61df54542b07d..5f3451d262c17 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group1/7.7.2_xpack_100k.test.ts @@ -8,9 +8,6 @@ import path from 'path'; import { unlink } from 'fs/promises'; -import { REPO_ROOT } from '@kbn/repo-info'; -import { Env } from '@kbn/config'; -import { getEnvOptions } from '@kbn/config-mocks'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { InternalCoreStart } from '@kbn/core-lifecycle-server-internal'; import { Root } from '@kbn/core-root-server-internal'; @@ -21,7 +18,6 @@ import { } from '@kbn/core-test-helpers-kbn-server'; import { SavedObjectsIndexPatterns } from '@kbn/core-saved-objects-server'; -const kibanaVersion = Env.createDefault(REPO_ROOT, getEnvOptions()).packageInfo.version; const logFilePath = path.join(__dirname, '7.7.2_xpack_100k.log'); async function removeLogFile() {