From 8861c86cb51f875d134712f9686554c5f5e8e818 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Fri, 9 Apr 2021 01:07:40 +0200 Subject: [PATCH 1/4] migrationsv2: ignore fleet agent events and tsvb telemetry --- .../migrationsv2/actions/index.test.ts | 3 +- .../migrationsv2/actions/index.ts | 17 ++++- .../integration_tests/actions.test.ts | 75 +++++++++++++++---- .../migrations_state_action_machine.test.ts | 28 +++++++ .../saved_objects/migrationsv2/model.test.ts | 7 ++ .../saved_objects/migrationsv2/model.ts | 6 ++ .../server/saved_objects/migrationsv2/next.ts | 12 ++- .../saved_objects/migrationsv2/types.ts | 5 ++ 8 files changed, 135 insertions(+), 18 deletions(-) diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.test.ts b/src/core/server/saved_objects/migrationsv2/actions/index.test.ts index 14ca73e7fcca0..bee17f42d7bdb 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.test.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.test.ts @@ -85,7 +85,8 @@ describe('actions', () => { 'my_source_index', 'my_target_index', Option.none, - false + false, + Option.none ); try { await task(); diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index 8ac683a29d657..d759c0c9be20e 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -14,6 +14,7 @@ import { errors as EsErrors } from '@elastic/elasticsearch'; import type { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/lib/errors'; import { pipe } from 'fp-ts/lib/pipeable'; import { flow } from 'fp-ts/lib/function'; +import { QueryContainer } from '@elastic/eui/src/components/search_bar/query/ast_to_es_query_dsl'; import { ElasticsearchClient } from '../../../elasticsearch'; import { IndexMapping } from '../../mappings'; import { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '../../serialization'; @@ -436,7 +437,12 @@ export const reindex = ( sourceIndex: string, targetIndex: string, reindexScript: Option.Option, - requireAlias: boolean + requireAlias: boolean, + /* When reindexing we use a source query to exclude saved objects types which + * are no longer used. These saved objects will still be kept in the outdated + * index for backup purposes, but won't be availble in the upgraded index. + */ + unusedTypesToExclude: Option.Option ): TaskEither.TaskEither => () => { return client .reindex({ @@ -450,6 +456,15 @@ export const reindex = ( index: sourceIndex, // Set reindex batch size size: BATCH_SIZE, + // Exclude saved object types + query: Option.fold( + () => undefined, + (types) => ({ + bool: { + must_not: types.map((type) => ({ term: { type } })), + }, + }) + )(unusedTypesToExclude), }, dest: { index: targetIndex, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts index aa9a5ea92ac11..3ed3ace416990 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts @@ -66,7 +66,8 @@ describe('migration actions', () => { { _source: { title: 'doc 1' } }, { _source: { title: 'doc 2' } }, { _source: { title: 'doc 3' } }, - { _source: { title: 'saved object 4' } }, + { _source: { title: 'saved object 4', type: 'another_unused_type' } }, + { _source: { title: 'f-agent-event 5', type: 'f_agent_event' } }, ] as unknown) as SavedObjectsRawDoc[]; await bulkOverwriteTransformedDocuments(client, 'existing_index_with_docs', sourceDocs)(); @@ -343,7 +344,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -364,6 +366,37 @@ describe('migration actions', () => { "doc 2", "doc 3", "saved object 4", + "f-agent-event 5", + ] + `); + }); + it('resolves right and excludes all unusedTypesToExclude documents', async () => { + const res = (await reindex( + client, + 'existing_index_with_docs', + 'reindex_target_excluded_docs', + Option.none, + false, + Option.some(['f_agent_event', 'another_unused_type']) + )()) as Either.Right; + const task = waitForReindexTask(client, res.right.taskId, '10s'); + await expect(task()).resolves.toMatchInlineSnapshot(` + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); + + const results = ((await searchForOutdatedDocuments(client, { + batchSize: 1000, + targetIndex: 'reindex_target_excluded_docs', + outdatedDocumentsQuery: undefined, + })()) as Either.Right).right.outdatedDocuments; + expect(results.map((doc) => doc._source.title)).toMatchInlineSnapshot(` + Array [ + "doc 1", + "doc 2", + "doc 3", ] `); }); @@ -374,7 +407,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_2', Option.some(`ctx._source.title = ctx._source.title + '_updated'`), - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -394,6 +428,7 @@ describe('migration actions', () => { "doc 2_updated", "doc 3_updated", "saved object 4_updated", + "f-agent-event 5_updated", ] `); }); @@ -405,7 +440,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_3', Option.some(`ctx._source.title = ctx._source.title + '_updated'`), - false + false, + Option.none )()) as Either.Right; let task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -421,7 +457,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_3', Option.none, - false + false, + Option.none )()) as Either.Right; task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -443,6 +480,7 @@ describe('migration actions', () => { "doc 2_updated", "doc 3_updated", "saved object 4_updated", + "f-agent-event 5_updated", ] `); }); @@ -469,7 +507,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_4', Option.some(`ctx._source.title = ctx._source.title + '_updated'`), - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -491,6 +530,7 @@ describe('migration actions', () => { "doc 2", "doc 3_updated", "saved object 4_updated", + "f-agent-event 5_updated", ] `); }); @@ -517,7 +557,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_5', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, reindexTaskId, '10s'); @@ -551,7 +592,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_6', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, reindexTaskId, '10s'); @@ -571,7 +613,8 @@ describe('migration actions', () => { 'no_such_index', 'reindex_target', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` @@ -591,7 +634,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'existing_index_with_write_block', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); @@ -612,7 +656,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'existing_index_with_write_block', Option.none, - true + true, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); @@ -633,7 +678,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target', Option.none, - false + false, + Option.none )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '0s'); @@ -659,7 +705,8 @@ describe('migration actions', () => { 'existing_index_with_docs', 'reindex_target_7', Option.none, - false + false, + Option.none )()) as Either.Right; await waitForReindexTask(client, res.right.taskId, '10s')(); @@ -714,7 +761,7 @@ describe('migration actions', () => { targetIndex: 'existing_index_with_docs', outdatedDocumentsQuery: undefined, })()) as Either.Right).right.outdatedDocuments; - expect(resultsWithoutQuery.length).toBe(4); + expect(resultsWithoutQuery.length).toBe(5); }); it('resolves with _id, _source, _seq_no and _primary_term', async () => { expect.assertions(1); diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index d4ce7b74baa5f..2c2cd0032abfd 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -249,6 +249,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, @@ -310,6 +317,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, @@ -456,6 +470,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, @@ -512,6 +533,13 @@ describe('migrationsStateActionMachine', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", }, diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index f9bf3418c0ab6..626687970c152 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -1242,6 +1242,13 @@ describe('migrations v2 model', () => { }, }, }, + "unusedTypesToExclude": Object { + "_tag": "Some", + "value": Array [ + "fleet-agent-events", + "tsvb-validation-telemetry", + ], + }, "versionAlias": ".kibana_task_manager_8.1.0", "versionIndex": ".kibana_task_manager_8.1.0_001", } diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index e62bd108faea0..2353452a6a51b 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -768,6 +768,11 @@ export const createInitialState = ({ }, }; + const unusedTypesToExclude = Option.some([ + 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 + 'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617 + ]); + const initialState: InitState = { controlState: 'INIT', indexPrefix, @@ -786,6 +791,7 @@ export const createInitialState = ({ retryAttempts: migrationsConfig.retryAttempts, batchSize: migrationsConfig.batchSize, logs: [], + unusedTypesToExclude, }; return initialState; }; diff --git a/src/core/server/saved_objects/migrationsv2/next.ts b/src/core/server/saved_objects/migrationsv2/next.ts index 5c159f4f24e22..67b2004a4b31a 100644 --- a/src/core/server/saved_objects/migrationsv2/next.ts +++ b/src/core/server/saved_objects/migrationsv2/next.ts @@ -61,7 +61,14 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra CREATE_REINDEX_TEMP: (state: CreateReindexTempState) => Actions.createIndex(client, state.tempIndex, state.tempIndexMappings), REINDEX_SOURCE_TO_TEMP: (state: ReindexSourceToTempState) => - Actions.reindex(client, state.sourceIndex.value, state.tempIndex, Option.none, false), + Actions.reindex( + client, + state.sourceIndex.value, + state.tempIndex, + Option.none, + false, + state.unusedTypesToExclude + ), SET_TEMP_WRITE_BLOCK: (state: SetTempWriteBlock) => Actions.setWriteBlock(client, state.tempIndex), REINDEX_SOURCE_TO_TEMP_WAIT_FOR_TASK: (state: ReindexSourceToTempWaitForTaskState) => @@ -104,7 +111,8 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra state.legacyIndex, state.sourceIndex.value, state.preMigrationScript, - false + false, + state.unusedTypesToExclude ), LEGACY_REINDEX_WAIT_FOR_TASK: (state: LegacyReindexWaitForTaskState) => Actions.waitForReindexTask(client, state.legacyReindexTaskId, '60s'), diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index 8d6fe3f030eb3..cc4aa18171843 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -89,6 +89,11 @@ export interface BaseState extends ControlState { * prevents lost deletes e.g. `.kibana_7.11.0_reindex`. */ readonly tempIndex: string; + /* When reindexing we use a source query to exclude saved objects types which + * are no longer used. These saved objects will still be kept in the outdated + * index for backup purposes, but won't be availble in the upgraded index. + */ + readonly unusedTypesToExclude: Option.Option; } export type InitState = BaseState & { From 259308c60f832edf8a50d5ddd05ccd1ae67d5d31 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Fri, 9 Apr 2021 01:27:22 +0200 Subject: [PATCH 2/4] migrationsv1: ignore tsvb-validation-telemetry --- .../migrations/core/elastic_index.test.ts | 15 ++++++++---- .../migrations/core/elastic_index.ts | 23 +++++++++++-------- .../migrations/kibana/kibana_migrator.test.ts | 6 ++--- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts index 5cb2a88c4733f..2fc78fc619cab 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts @@ -414,11 +414,18 @@ describe('ElasticIndex', () => { size: 100, query: { bool: { - must_not: { - term: { - type: 'fleet-agent-events', + must_not: [ + { + term: { + type: 'fleet-agent-events', + }, }, - }, + { + term: { + type: 'tsvb-validation-telemetry', + }, + }, + ], }, }, }, diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index a5f3cb36e736b..462425ff6e3e0 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -70,16 +70,19 @@ export function reader( let scrollId: string | undefined; // When migrating from the outdated index we use a read query which excludes - // saved objects which are no longer used. These saved objects will still be - // kept in the outdated index for backup purposes, but won't be availble in - // the upgraded index. - const excludeUnusedTypes = { + // saved object types which are no longer used. These saved objects will + // still be kept in the outdated index for backup purposes, but won't be + // availble in the upgraded index. + const EXCLUDE_UNUSED_TYPES = [ + 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 + 'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617 + ]; + + const excludeUnusedTypesQuery = { bool: { - must_not: { - term: { - type: 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 - }, - }, + must_not: EXCLUDE_UNUSED_TYPES.map((type) => ({ + term: { type }, + })), }, }; @@ -92,7 +95,7 @@ export function reader( : client.search>({ body: { size: batchSize, - query: excludeUnusedTypes, + query: excludeUnusedTypesQuery, }, index, scroll, diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index 40d18c3b5063a..221e78e3e12e2 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -321,7 +321,7 @@ describe('KibanaMigrator', () => { options.client.tasks.get.mockReturnValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ completed: true, - error: { type: 'elatsicsearch_exception', reason: 'task failed with an error' }, + error: { type: 'elasticsearch_exception', reason: 'task failed with an error' }, failures: [], task: { description: 'task description' } as any, }) @@ -331,11 +331,11 @@ describe('KibanaMigrator', () => { migrator.prepareMigrations(); await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(` [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}] + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(` [Error: Reindex failed with the following error: - {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}] + {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}] `); }); }); From 1ac27e7da6256f126fc8c2532d0cddf11ce52e25 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Fri, 9 Apr 2021 15:19:59 +0200 Subject: [PATCH 3/4] Skip fleet test that depends on fleet-agent-events --- x-pack/test/fleet_api_integration/apis/agents_setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/fleet_api_integration/apis/agents_setup.ts b/x-pack/test/fleet_api_integration/apis/agents_setup.ts index 91d6ca0119d1d..700a06750d2f4 100644 --- a/x-pack/test/fleet_api_integration/apis/agents_setup.ts +++ b/x-pack/test/fleet_api_integration/apis/agents_setup.ts @@ -101,7 +101,7 @@ export default function (providerContext: FtrProviderContext) { ); }); - it('should create or update the fleet_enroll user if called multiple times with forceRecreate flag', async () => { + it.skip('should create or update the fleet_enroll user if called multiple times with forceRecreate flag', async () => { await supertest.post(`/api/fleet/agents/setup`).set('kbn-xsrf', 'xxxx').expect(200); const { body: userResponseFirstTime } = await es.security.getUser({ From 2eeac2adecc95d7b4fe826f50575fa1063eb34ac Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Fri, 9 Apr 2021 15:20:55 +0200 Subject: [PATCH 4/4] Fix typescript errors --- src/core/server/saved_objects/migrationsv2/model.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index 626687970c152..4fd9b7cbb3df4 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -69,6 +69,7 @@ describe('migrations v2 model', () => { versionAlias: '.kibana_7.11.0', versionIndex: '.kibana_7.11.0_001', tempIndex: '.kibana_7.11.0_reindex_temp', + unusedTypesToExclude: Option.some(['unused-fleet-agent-events']), }; describe('exponential retry delays for retryable_es_client_error', () => {