diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index ddf52ccad768..09d0c43d6b6c 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -497,9 +497,6 @@ override this parameter to use their own Tile Map Service. For example: | `migrations.maxBatchSizeBytes:` | Defines the maximum payload size for indexing batches of upgraded saved objects to avoid migrations failing due to a 413 Request Entity Too Large response from Elasticsearch. This value should be lower than or equal to your Elasticsearch cluster's `http.max_content_length` configuration option. *Default: `100mb`* -| `migrations.enableV2:` - | experimental[]. Enables the new Saved Objects migration algorithm. For information about the migration algorithm, refer to <>. When `migrations v2` is stable, the setting will be removed in an upcoming release without any further notice. Setting the value to `false` causes {kib} to use the legacy migration algorithm, which shipped in 7.11 and earlier versions. *Default: `true`* - | `migrations.retryAttempts:` | The number of times migrations retry temporary failures, such as a network timeout, 503 status code, or `snapshot_in_progress_exception`. When upgrade migrations frequently fail after exhausting all retry attempts with a message such as `Unable to complete the [...] step after 15 attempts, terminating.`, increase the setting value. *Default: `15`* diff --git a/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts b/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts index d7ea73456e23..1f9ca741691d 100644 --- a/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts +++ b/src/core/server/saved_objects/deprecations/unknown_object_types.test.ts @@ -13,7 +13,6 @@ import { deleteUnknownTypeObjects, getUnknownTypesDeprecations } from './unknown import { typeRegistryMock } from '../saved_objects_type_registry.mock'; import { elasticsearchClientMock } from '../../elasticsearch/client/mocks'; import type { KibanaConfigType } from '../../kibana_config'; -import type { SavedObjectConfig } from '../saved_objects_config'; import { SavedObjectsType } from 'kibana/server'; const createSearchResponse = (count: number): estypes.SearchResponse => { @@ -32,7 +31,6 @@ describe('unknown saved object types deprecation', () => { let typeRegistry: ReturnType; let esClient: ReturnType; let kibanaConfig: KibanaConfigType; - let savedObjectsConfig: SavedObjectConfig; beforeEach(() => { typeRegistry = typeRegistryMock.create(); @@ -48,12 +46,6 @@ describe('unknown saved object types deprecation', () => { index: '.kibana', enabled: true, }; - - savedObjectsConfig = { - migration: { - enableV2: true, - }, - } as SavedObjectConfig; }); afterEach(() => { @@ -69,7 +61,6 @@ describe('unknown saved object types deprecation', () => { it('calls `esClient.asInternalUser.search` with the correct parameters', async () => { await getUnknownTypesDeprecations({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, @@ -96,7 +87,6 @@ describe('unknown saved object types deprecation', () => { ); const deprecations = await getUnknownTypesDeprecations({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, @@ -112,7 +102,6 @@ describe('unknown saved object types deprecation', () => { ); const deprecations = await getUnknownTypesDeprecations({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, @@ -141,7 +130,6 @@ describe('unknown saved object types deprecation', () => { describe('deleteUnknownTypeObjects', () => { it('calls `esClient.asInternalUser.search` with the correct parameters', async () => { await deleteUnknownTypeObjects({ - savedObjectsConfig, esClient, typeRegistry, kibanaConfig, diff --git a/src/core/server/saved_objects/deprecations/unknown_object_types.ts b/src/core/server/saved_objects/deprecations/unknown_object_types.ts index c966e621ca60..8cd650bac8a2 100644 --- a/src/core/server/saved_objects/deprecations/unknown_object_types.ts +++ b/src/core/server/saved_objects/deprecations/unknown_object_types.ts @@ -13,14 +13,12 @@ import { IScopedClusterClient } from '../../elasticsearch'; import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry'; import { SavedObjectsRawDocSource } from '../serialization'; import type { KibanaConfigType } from '../../kibana_config'; -import type { SavedObjectConfig } from '../saved_objects_config'; import { getIndexForType } from '../service/lib'; interface UnknownTypesDeprecationOptions { typeRegistry: ISavedObjectTypeRegistry; esClient: IScopedClusterClient; kibanaConfig: KibanaConfigType; - savedObjectsConfig: SavedObjectConfig; kibanaVersion: string; } @@ -32,11 +30,9 @@ const getTargetIndices = ({ typeRegistry, kibanaVersion, kibanaConfig, - savedObjectsConfig, }: { types: string[]; typeRegistry: ISavedObjectTypeRegistry; - savedObjectsConfig: SavedObjectConfig; kibanaConfig: KibanaConfigType; kibanaVersion: string; }) => { @@ -46,7 +42,6 @@ const getTargetIndices = ({ getIndexForType({ type, typeRegistry, - migV2Enabled: savedObjectsConfig.migration.enableV2, kibanaVersion, defaultIndex: kibanaConfig.index, }) @@ -69,7 +64,6 @@ const getUnknownSavedObjects = async ({ typeRegistry, esClient, kibanaConfig, - savedObjectsConfig, kibanaVersion, }: UnknownTypesDeprecationOptions) => { const knownTypes = getKnownTypes(typeRegistry); @@ -78,7 +72,6 @@ const getUnknownSavedObjects = async ({ typeRegistry, kibanaConfig, kibanaVersion, - savedObjectsConfig, }); const query = getUnknownTypesQuery(knownTypes); @@ -141,7 +134,6 @@ interface DeleteUnknownTypesOptions { typeRegistry: ISavedObjectTypeRegistry; esClient: IScopedClusterClient; kibanaConfig: KibanaConfigType; - savedObjectsConfig: SavedObjectConfig; kibanaVersion: string; } @@ -149,7 +141,6 @@ export const deleteUnknownTypeObjects = async ({ esClient, typeRegistry, kibanaConfig, - savedObjectsConfig, kibanaVersion, }: DeleteUnknownTypesOptions) => { const knownTypes = getKnownTypes(typeRegistry); @@ -158,7 +149,6 @@ export const deleteUnknownTypeObjects = async ({ typeRegistry, kibanaConfig, kibanaVersion, - savedObjectsConfig, }); const query = getUnknownTypesQuery(knownTypes); diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts index 9471bbc1b87a..660300ea867f 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts @@ -42,8 +42,6 @@ const createMigrator = ( scrollDuration: '15m', pollInterval: 1500, skip: false, - // TODO migrationsV2: remove/deprecate once we remove migrations v1 - enableV2: false, retryAttempts: 10, }, runMigrations: jest.fn(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index 6e10349f4b57..c397559b5257 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -7,7 +7,7 @@ */ import { take } from 'rxjs/operators'; -import { estypes, errors as esErrors } from '@elastic/elasticsearch'; +import { estypes } from '@elastic/elasticsearch'; import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks'; import { KibanaMigratorOptions, KibanaMigrator } from './kibana_migrator'; @@ -125,13 +125,6 @@ describe('KibanaMigrator', () => { it('only runs migrations once if called multiple times', async () => { const options = mockOptions(); - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); options.client.indices.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) ); @@ -144,159 +137,79 @@ describe('KibanaMigrator', () => { migrator.prepareMigrations(); await migrator.runMigrations(); await migrator.runMigrations(); + await migrator.runMigrations(); - expect(options.client.cat.templates).toHaveBeenCalledTimes(1); + // indices.get is called twice during a single migration + expect(options.client.indices.get).toHaveBeenCalledTimes(2); }); - describe('when enableV2 = false', () => { - it('when enableV2 = false creates an IndexMigrator which retries NoLivingConnectionsError errors from ES client', async () => { - const options = mockOptions(); - - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - options.client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - options.client.indices.create = jest - .fn() - .mockReturnValueOnce( - elasticsearchClientMock.createErrorTransportRequestPromise( - new esErrors.NoLivingConnectionsError('reason', {} as any) - ) - ) - .mockImplementationOnce(() => - elasticsearchClientMock.createSuccessTransportRequestPromise('success') - ); - - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - - migrator.prepareMigrations(); - await migrator.runMigrations(); + it('emits results on getMigratorResult$()', async () => { + const options = mockV2MigrationOptions(); + const migrator = new KibanaMigrator(options); + const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); + migrator.prepareMigrations(); + await migrator.runMigrations(); - expect(options.client.indices.create).toHaveBeenCalledTimes(3); - const { status } = await migratorStatus; - return expect(status).toEqual('completed'); + const { status, result } = await migratorStatus; + expect(status).toEqual('completed'); + expect(result![0]).toMatchObject({ + destIndex: '.my-index_8.2.3_001', + sourceIndex: '.my-index_pre8.2.3_001', + elapsedMs: expect.any(Number), + status: 'migrated', }); - - it('emits results on getMigratorResult$()', async () => { - const options = mockOptions(); - - options.client.cat.templates.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - // @ts-expect-error - { templates: [] } as CatTemplatesResponse, - { statusCode: 404 } - ) - ); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - options.client.indices.getAlias.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) - ); - - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - migrator.prepareMigrations(); - await migrator.runMigrations(); - const { status, result } = await migratorStatus; - expect(status).toEqual('completed'); - expect(result![0]).toMatchObject({ - destIndex: '.my-index_1', - elapsedMs: expect.any(Number), - sourceIndex: '.my-index', - status: 'migrated', - }); - expect(result![1]).toMatchObject({ - destIndex: 'other-index_1', - elapsedMs: expect.any(Number), - sourceIndex: 'other-index', - status: 'migrated', - }); + expect(result![1]).toMatchObject({ + destIndex: 'other-index_8.2.3_001', + elapsedMs: expect.any(Number), + status: 'patched', }); }); - describe('when enableV2 = true', () => { - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('emits results on getMigratorResult$()', async () => { - const options = mockV2MigrationOptions(); - const migrator = new KibanaMigrator(options); - const migratorStatus = migrator.getStatus$().pipe(take(3)).toPromise(); - migrator.prepareMigrations(); - await migrator.runMigrations(); - - const { status, result } = await migratorStatus; - expect(status).toEqual('completed'); - expect(result![0]).toMatchObject({ - destIndex: '.my-index_8.2.3_001', - sourceIndex: '.my-index_pre8.2.3_001', - elapsedMs: expect.any(Number), - status: 'migrated', - }); - expect(result![1]).toMatchObject({ - destIndex: 'other-index_8.2.3_001', - elapsedMs: expect.any(Number), - status: 'patched', - }); - }); - it('rejects when the migration state machine terminates in a FATAL state', () => { - const options = mockV2MigrationOptions(); - options.client.indices.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise( - { - '.my-index_8.2.4_001': { - aliases: { - '.my-index': {}, - '.my-index_8.2.4': {}, - }, - mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } }, - settings: {}, + it('rejects when the migration state machine terminates in a FATAL state', () => { + const options = mockV2MigrationOptions(); + options.client.indices.get.mockReturnValue( + elasticsearchClientMock.createSuccessTransportRequestPromise( + { + '.my-index_8.2.4_001': { + aliases: { + '.my-index': {}, + '.my-index_8.2.4': {}, }, + mappings: { properties: {}, _meta: { migrationMappingPropertyHashes: {} } }, + settings: {}, }, - { statusCode: 200 } - ) - ); + }, + { statusCode: 200 } + ) + ); - const migrator = new KibanaMigrator(options); - migrator.prepareMigrations(); - return expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot( - `[Error: Unable to complete saved object migrations for the [.my-index] index: The .my-index alias is pointing to a newer version of Kibana: v8.2.4]` - ); - }); - it('rejects when an unexpected exception occurs in an action', async () => { - const options = mockV2MigrationOptions(); - options.client.tasks.get.mockReturnValue( - elasticsearchClientMock.createSuccessTransportRequestPromise({ - completed: true, - error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, - failures: [], - task: { description: 'task description' } as any, - }) - ); + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + return expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot( + `[Error: Unable to complete saved object migrations for the [.my-index] index: The .my-index alias is pointing to a newer version of Kibana: v8.2.4]` + ); + }); - const migrator = new KibanaMigrator(options); - migrator.prepareMigrations(); - await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` - [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] - `); - expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` - [Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] - `); - }); + it('rejects when an unexpected exception occurs in an action', async () => { + const options = mockV2MigrationOptions(); + options.client.tasks.get.mockReturnValue( + elasticsearchClientMock.createSuccessTransportRequestPromise({ + completed: true, + error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, + failures: [], + task: { description: 'task description' } as any, + }) + ); + + const migrator = new KibanaMigrator(options); + migrator.prepareMigrations(); + await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` + [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] + `); + expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` + [Error: Reindex failed with the following error: + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] + `); }); }); }); @@ -306,7 +219,7 @@ type MockedOptions = KibanaMigratorOptions & { }; const mockV2MigrationOptions = () => { - const options = mockOptions({ enableV2: true }); + const options = mockOptions(); options.client.indices.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise( @@ -362,7 +275,7 @@ const mockV2MigrationOptions = () => { return options; }; -const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) => { +const mockOptions = () => { const options: MockedOptions = { logger: loggingSystemMock.create().get(), kibanaVersion: '8.2.3', @@ -401,7 +314,6 @@ const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) pollInterval: 20000, scrollDuration: '10m', skip: false, - enableV2, retryAttempts: 20, }, client: elasticsearchClientMock.createElasticsearchClient(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index 572b2934e49b..d3755f8c7e66 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -22,13 +22,7 @@ import { SavedObjectsSerializer, SavedObjectsRawDoc, } from '../../serialization'; -import { - buildActiveMappings, - createMigrationEsClient, - IndexMigrator, - MigrationResult, - MigrationStatus, -} from '../core'; +import { buildActiveMappings, MigrationResult, MigrationStatus } from '../core'; import { DocumentMigrator, VersionedTransformer } from '../core/document_migrator'; import { createIndexMap } from '../core/build_index_map'; import { SavedObjectsMigrationConfigType } from '../../saved_objects_config'; @@ -71,7 +65,6 @@ export class KibanaMigrator { status: 'waiting_to_start', }); private readonly activeMappings: IndexMapping; - private migrationsRetryDelay?: number; // TODO migrationsV2: make private once we remove migrations v1 public readonly kibanaVersion: string; // TODO migrationsV2: make private once we remove migrations v1 @@ -105,7 +98,6 @@ export class KibanaMigrator { // Building the active mappings (and associated md5sums) is an expensive // operation so we cache the result this.activeMappings = buildActiveMappings(this.mappingProperties); - this.migrationsRetryDelay = migrationsRetryDelay; } /** @@ -173,49 +165,28 @@ export class KibanaMigrator { }); const migrators = Object.keys(indexMap).map((index) => { - // TODO migrationsV2: remove old migrations algorithm - if (this.soMigrationsConfig.enableV2) { - return { - migrate: (): Promise => { - return runResilientMigrator({ - client: this.client, - kibanaVersion: this.kibanaVersion, - targetMappings: buildActiveMappings(indexMap[index].typeMappings), - logger: this.log, - preMigrationScript: indexMap[index].script, - transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => - migrateRawDocsSafely({ - serializer: this.serializer, - knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)), - migrateDoc: this.documentMigrator.migrateAndConvert, - rawDocs, - }), - migrationVersionPerType: this.documentMigrator.migrationVersion, - indexPrefix: index, - migrationsConfig: this.soMigrationsConfig, - typeRegistry: this.typeRegistry, - }); - }, - }; - } else { - return new IndexMigrator({ - batchSize: this.soMigrationsConfig.batchSize, - client: createMigrationEsClient(this.client, this.log, this.migrationsRetryDelay), - documentMigrator: this.documentMigrator, - index, - kibanaVersion: this.kibanaVersion, - log: this.log, - mappingProperties: indexMap[index].typeMappings, - setStatus: (status) => this.status$.next(status), - pollInterval: this.soMigrationsConfig.pollInterval, - scrollDuration: this.soMigrationsConfig.scrollDuration, - serializer: this.serializer, - // Only necessary for the migrator of the kibana index. - obsoleteIndexTemplatePattern: - index === kibanaIndexName ? 'kibana_index_template*' : undefined, - convertToAliasScript: indexMap[index].script, - }); - } + return { + migrate: (): Promise => { + return runResilientMigrator({ + client: this.client, + kibanaVersion: this.kibanaVersion, + targetMappings: buildActiveMappings(indexMap[index].typeMappings), + logger: this.log, + preMigrationScript: indexMap[index].script, + transformRawDocs: (rawDocs: SavedObjectsRawDoc[]) => + migrateRawDocsSafely({ + serializer: this.serializer, + knownTypes: new Set(this.typeRegistry.getAllTypes().map((t) => t.name)), + migrateDoc: this.documentMigrator.migrateAndConvert, + rawDocs, + }), + migrationVersionPerType: this.documentMigrator.migrationVersion, + indexPrefix: index, + migrationsConfig: this.soMigrationsConfig, + typeRegistry: this.typeRegistry, + }); + }, + }; }); return Promise.all(migrators.map((migrator) => migrator.migrate())); diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts index d072ccc5c976..3dcfb34c47a1 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7.7.2_xpack_100k.test.ts @@ -49,7 +49,6 @@ describe('migration from 7.7.2-xpack with 100k objects', () => { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts index d70e03470315..a4ce95a9e058 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_failed_action_tasks.test.ts @@ -113,7 +113,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 250, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts index fb40bda81cba..c8e17a64a3fa 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_transform_failures.test.ts @@ -155,7 +155,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts index f15e0c5d684d..31c802bb561f 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/7_13_0_unknown_types.test.ts @@ -218,7 +218,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts index ead46627e324..9daae7eddafe 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts @@ -118,7 +118,6 @@ function createRoot(options: { maxBatchSizeBytes?: number }) { { migrations: { skip: false, - enableV2: true, batchSize: 1000, maxBatchSizeBytes: options.maxBatchSizeBytes, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts index 4f456b0bc741..d61426d92d39 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts @@ -88,7 +88,6 @@ function createRoot(options: { maxBatchSizeBytes?: number }) { { migrations: { skip: false, - enableV2: true, batchSize: 1000, maxBatchSizeBytes: options.maxBatchSizeBytes, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts index d76bbc786cff..c84f72b18426 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/cleanup.test.ts @@ -28,7 +28,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts index 9738650b0db8..0f59d1946149 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/collects_corrupt_docs.test.ts @@ -150,7 +150,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts index d19ae5131574..7e4b52dea185 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/corrupt_outdated_docs.test.ts @@ -154,7 +154,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, batchSize: 5, }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts index 8e01e11eaccf..12d04d89dee7 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/migration_from_older_v1.test.ts @@ -77,7 +77,6 @@ describe('migrating from 7.3.0-xpack which used v1 migrations', () => { { migrations: { skip: false, - enableV2: true, // There are 53 docs in fixtures. Batch size configured to enforce 3 migration steps. batchSize: 20, }, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts index 755bb5f946e4..6956e53ebc7f 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_es_nodes.test.ts @@ -67,7 +67,6 @@ function createRoot({ logFileName, hosts }: RootConfig) { }, migrations: { skip: false, - enableV2: true, batchSize: 100, // fixture contains 5000 docs }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts index 11c5b33c0fd3..ef92c823182d 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/multiple_kibana_nodes.test.ts @@ -67,7 +67,6 @@ async function createRoot({ logFileName }: CreateRootConfig) { }, migrations: { skip: false, - enableV2: true, batchSize: 100, // fixture contains 5000 docs }, logging: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts index 19d77637902f..4718b5d1285f 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/outdated_docs.test.ts @@ -96,7 +96,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts index 78eec5fe94ef..84c25fcdef8b 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/rewriting_id.test.ts @@ -62,7 +62,6 @@ function createRoot() { { migrations: { skip: false, - enableV2: true, }, logging: { appenders: { diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index 21468d755232..338eecf15117 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -45,7 +45,6 @@ describe('migrationsStateActionMachine', () => { pollInterval: 0, scrollDuration: '0s', skip: false, - enableV2: true, retryAttempts: 5, }, typeRegistry, diff --git a/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts b/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts index a9e1a41f01d9..2b6d64bef4f1 100644 --- a/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts +++ b/src/core/server/saved_objects/routes/deprecations/delete_unknown_types.ts @@ -9,18 +9,16 @@ import { IRouter } from '../../../http'; import { catchAndReturnBoomErrors } from '../utils'; import { deleteUnknownTypeObjects } from '../../deprecations'; -import { SavedObjectConfig } from '../../saved_objects_config'; import { KibanaConfigType } from '../../../kibana_config'; interface RouteDependencies { - config: SavedObjectConfig; kibanaConfig: KibanaConfigType; kibanaVersion: string; } export const registerDeleteUnknownTypesRoute = ( router: IRouter, - { config, kibanaConfig, kibanaVersion }: RouteDependencies + { kibanaConfig, kibanaVersion }: RouteDependencies ) => { router.post( { @@ -31,7 +29,6 @@ export const registerDeleteUnknownTypesRoute = ( await deleteUnknownTypeObjects({ esClient: context.core.elasticsearch.client, typeRegistry: context.core.savedObjects.typeRegistry, - savedObjectsConfig: config, kibanaConfig, kibanaVersion, }); diff --git a/src/core/server/saved_objects/routes/index.ts b/src/core/server/saved_objects/routes/index.ts index e2698c619b10..7aa6711a37c3 100644 --- a/src/core/server/saved_objects/routes/index.ts +++ b/src/core/server/saved_objects/routes/index.ts @@ -76,5 +76,5 @@ export function registerRoutes({ const internalRouter = http.createRouter('/internal/saved_objects/'); registerMigrateRoute(internalRouter, migratorPromise); - registerDeleteUnknownTypesRoute(internalRouter, { config, kibanaConfig, kibanaVersion }); + registerDeleteUnknownTypesRoute(internalRouter, { kibanaConfig, kibanaVersion }); } diff --git a/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts b/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts index fef2b2d5870e..0c7fbdda89fb 100644 --- a/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/delete_unknown_types.test.ts @@ -13,7 +13,6 @@ import { elasticsearchServiceMock } from '../../../../../core/server/elasticsear import { typeRegistryMock } from '../../saved_objects_type_registry.mock'; import { setupServer } from '../test_utils'; import { KibanaConfigType } from '../../../kibana_config'; -import { SavedObjectConfig } from '../../saved_objects_config'; import { SavedObjectsType } from 'kibana/server'; type SetupServerReturn = UnwrapPromise>; @@ -24,13 +23,6 @@ describe('POST /internal/saved_objects/deprecations/_delete_unknown_types', () = enabled: true, index: '.kibana', }; - const config: SavedObjectConfig = { - maxImportExportSize: 10000, - maxImportPayloadBytes: 24000000, - migration: { - enableV2: true, - } as SavedObjectConfig['migration'], - }; let server: SetupServerReturn['server']; let httpSetup: SetupServerReturn['httpSetup']; @@ -54,7 +46,6 @@ describe('POST /internal/saved_objects/deprecations/_delete_unknown_types', () = registerDeleteUnknownTypesRoute(router, { kibanaVersion, kibanaConfig, - config, }); await server.start(); diff --git a/src/core/server/saved_objects/saved_objects_config.test.ts b/src/core/server/saved_objects/saved_objects_config.test.ts index 720b28403edf..06b9e9661b74 100644 --- a/src/core/server/saved_objects/saved_objects_config.test.ts +++ b/src/core/server/saved_objects/saved_objects_config.test.ts @@ -22,7 +22,7 @@ describe('migrations config', function () { const { messages } = applyMigrationsDeprecations({ enableV2: true }); expect(messages).toMatchInlineSnapshot(` Array [ - "\\"migrations.enableV2\\" is deprecated and will be removed in an upcoming release without any further notice.", + "You no longer need to configure \\"migrations.enableV2\\".", ] `); }); @@ -31,7 +31,7 @@ describe('migrations config', function () { const { messages } = applyMigrationsDeprecations({ enableV2: false }); expect(messages).toMatchInlineSnapshot(` Array [ - "\\"migrations.enableV2\\" is deprecated and will be removed in an upcoming release without any further notice.", + "You no longer need to configure \\"migrations.enableV2\\".", ] `); }); diff --git a/src/core/server/saved_objects/saved_objects_config.ts b/src/core/server/saved_objects/saved_objects_config.ts index c9b4b4499fa8..02fbd974da4a 100644 --- a/src/core/server/saved_objects/saved_objects_config.ts +++ b/src/core/server/saved_objects/saved_objects_config.ts @@ -7,8 +7,8 @@ */ import { schema, TypeOf } from '@kbn/config-schema'; +import { ConfigDeprecationProvider } from '../config'; import type { ServiceConfigDescriptor } from '../internal_types'; -import type { ConfigDeprecationProvider } from '../config'; const migrationSchema = schema.object({ batchSize: schema.number({ defaultValue: 1_000 }), @@ -16,29 +16,12 @@ const migrationSchema = schema.object({ scrollDuration: schema.string({ defaultValue: '15m' }), pollInterval: schema.number({ defaultValue: 1_500 }), skip: schema.boolean({ defaultValue: false }), - enableV2: schema.boolean({ defaultValue: true }), retryAttempts: schema.number({ defaultValue: 15 }), }); export type SavedObjectsMigrationConfigType = TypeOf; -const migrationDeprecations: ConfigDeprecationProvider = () => [ - (settings, fromPath, addDeprecation) => { - const migrationsConfig = settings[fromPath]; - if (migrationsConfig?.enableV2 !== undefined) { - addDeprecation({ - configPath: `${fromPath}.enableV2`, - message: - '"migrations.enableV2" is deprecated and will be removed in an upcoming release without any further notice.', - documentationUrl: 'https://ela.st/kbn-so-migration-v2', - correctiveActions: { - manualSteps: [`Remove "migrations.enableV2" from your kibana configs.`], - }, - }); - } - return settings; - }, -]; +const migrationDeprecations: ConfigDeprecationProvider = ({ unused }) => [unused('enableV2')]; export const savedObjectsMigrationConfig: ServiceConfigDescriptor = { diff --git a/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts b/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts index fa065b02b805..16e3ba9495f0 100644 --- a/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts +++ b/src/core/server/saved_objects/service/lib/get_index_for_type.test.ts @@ -18,63 +18,27 @@ describe('getIndexForType', () => { typeRegistry = typeRegistryMock.create(); }); - describe('when migV2 is enabled', () => { - const migV2Enabled = true; - - it('returns the correct index for a type specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => `.${type}-index`); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.foo-index_8.0.0'); - }); - - it('returns the correct index for a type not specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => undefined); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.kibana_8.0.0'); - }); + it('returns the correct index for a type specifying a custom index', () => { + typeRegistry.getIndex.mockImplementation((type) => `.${type}-index`); + expect( + getIndexForType({ + type: 'foo', + typeRegistry, + defaultIndex, + kibanaVersion, + }) + ).toEqual('.foo-index_8.0.0'); }); - describe('when migV2 is disabled', () => { - const migV2Enabled = false; - - it('returns the correct index for a type specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => `.${type}-index`); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.foo-index'); - }); - - it('returns the correct index for a type not specifying a custom index', () => { - typeRegistry.getIndex.mockImplementation((type) => undefined); - expect( - getIndexForType({ - type: 'foo', - typeRegistry, - defaultIndex, - kibanaVersion, - migV2Enabled, - }) - ).toEqual('.kibana'); - }); + it('returns the correct index for a type not specifying a custom index', () => { + typeRegistry.getIndex.mockImplementation((type) => undefined); + expect( + getIndexForType({ + type: 'foo', + typeRegistry, + defaultIndex, + kibanaVersion, + }) + ).toEqual('.kibana_8.0.0'); }); }); diff --git a/src/core/server/saved_objects/service/lib/get_index_for_type.ts b/src/core/server/saved_objects/service/lib/get_index_for_type.ts index cef477e6dd84..ae34e6063e0a 100644 --- a/src/core/server/saved_objects/service/lib/get_index_for_type.ts +++ b/src/core/server/saved_objects/service/lib/get_index_for_type.ts @@ -11,7 +11,6 @@ import { ISavedObjectTypeRegistry } from '../../saved_objects_type_registry'; interface GetIndexForTypeOptions { type: string; typeRegistry: ISavedObjectTypeRegistry; - migV2Enabled: boolean; kibanaVersion: string; defaultIndex: string; } @@ -19,18 +18,8 @@ interface GetIndexForTypeOptions { export const getIndexForType = ({ type, typeRegistry, - migV2Enabled, defaultIndex, kibanaVersion, }: GetIndexForTypeOptions): string => { - // TODO migrationsV2: Remove once we remove migrations v1 - // This is a hacky, but it required the least amount of changes to - // existing code to support a migrations v2 index. Long term we would - // want to always use the type registry to resolve a type's index - // (including the default index). - if (migV2Enabled) { - return `${typeRegistry.getIndex(type) || defaultIndex}_${kibanaVersion}`; - } else { - return typeRegistry.getIndex(type) || defaultIndex; - } + return `${typeRegistry.getIndex(type) || defaultIndex}_${kibanaVersion}`; }; diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 82a0dd71700f..45865a2fb988 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -612,12 +612,18 @@ describe('SavedObjectsRepository', () => { it(`should use default index`, async () => { await bulkCreateSuccess([obj1, obj2]); - expectClientCallArgsAction([obj1, obj2], { method: 'create', _index: '.kibana-test' }); + expectClientCallArgsAction([obj1, obj2], { + method: 'create', + _index: '.kibana-test_8.0.0-testing', + }); }); it(`should use custom index`, async () => { await bulkCreateSuccess([obj1, obj2].map((x) => ({ ...x, type: CUSTOM_INDEX_TYPE }))); - expectClientCallArgsAction([obj1, obj2], { method: 'create', _index: 'custom' }); + expectClientCallArgsAction([obj1, obj2], { + method: 'create', + _index: 'custom_8.0.0-testing', + }); }); it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => { @@ -2091,7 +2097,7 @@ describe('SavedObjectsRepository', () => { it(`should use default index`, async () => { await createSuccess(type, attributes, { id }); expect(client.create).toHaveBeenCalledWith( - expect.objectContaining({ index: '.kibana-test' }), + expect.objectContaining({ index: '.kibana-test_8.0.0-testing' }), expect.anything() ); }); @@ -2099,7 +2105,7 @@ describe('SavedObjectsRepository', () => { it(`should use custom index`, async () => { await createSuccess(CUSTOM_INDEX_TYPE, attributes, { id }); expect(client.create).toHaveBeenCalledWith( - expect.objectContaining({ index: 'custom' }), + expect.objectContaining({ index: 'custom_8.0.0-testing' }), expect.anything() ); }); @@ -2679,7 +2685,9 @@ describe('SavedObjectsRepository', () => { it(`should use all indices for types that are not namespace-agnostic`, async () => { await deleteByNamespaceSuccess(namespace); expect(client.updateByQuery).toHaveBeenCalledWith( - expect.objectContaining({ index: ['.kibana-test', 'custom'] }), + expect.objectContaining({ + index: ['.kibana-test_8.0.0-testing', 'custom_8.0.0-testing'], + }), expect.anything() ); }); @@ -2769,7 +2777,7 @@ describe('SavedObjectsRepository', () => { await removeReferencesToSuccess(); expect(client.updateByQuery).toHaveBeenCalledWith( expect.objectContaining({ - index: ['.kibana-test', 'custom'], + index: ['.kibana-test_8.0.0-testing', 'custom_8.0.0-testing'], }), expect.anything() ); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index c74092faad96..26ebef4d12ea 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -2077,7 +2077,6 @@ export class SavedObjectsRepository { defaultIndex: this._index, typeRegistry: this._registry, kibanaVersion: this._migrator.kibanaVersion, - migV2Enabled: this._migrator.soMigrationsConfig.enableV2, }); } diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker index 7d486e4717e5..73ed445d272a 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker @@ -109,7 +109,6 @@ kibana_vars=( map.tilemap.url migrations.batchSize migrations.maxBatchSizeBytes - migrations.enableV2 migrations.pollInterval migrations.retryAttempts migrations.scrollDuration diff --git a/x-pack/plugins/uptime/e2e/config.ts b/x-pack/plugins/uptime/e2e/config.ts index c5d573afccd9..d2c7a691e0a4 100644 --- a/x-pack/plugins/uptime/e2e/config.ts +++ b/x-pack/plugins/uptime/e2e/config.ts @@ -43,7 +43,6 @@ async function config({ readConfigFile }: FtrConfigProviderContext) { `--uiSettings.overrides.theme:darkMode=true`, `--elasticsearch.username=kibana_system`, `--elasticsearch.password=changeme`, - '--migrations.enableV2=false', '--xpack.reporting.enabled=false', ], }, diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index ad1f897debe3..3bc03eb5b425 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -11,22 +11,13 @@ import { createTestConfig, CreateTestConfig } from '../common/config'; const apmFtrConfigs = { basic: { license: 'basic' as const, - kibanaConfig: { - // disable v2 migrations to prevent issue where kibana index is deleted - // during a migration - 'migrations.enableV2': 'false', - }, }, trial: { license: 'trial' as const, - kibanaConfig: { - 'migrations.enableV2': 'false', - }, }, rules: { license: 'trial' as const, kibanaConfig: { - 'migrations.enableV2': 'false', 'xpack.ruleRegistry.write.enabled': 'true', }, },