diff --git a/docs/setup/upgrade/resolving-migration-failures.asciidoc b/docs/setup/upgrade/resolving-migration-failures.asciidoc index 3cbfb4c9c2abe..f90a9f541f3eb 100644 --- a/docs/setup/upgrade/resolving-migration-failures.asciidoc +++ b/docs/setup/upgrade/resolving-migration-failures.asciidoc @@ -171,7 +171,7 @@ Upgrade migrations fail because routing allocation is disabled or restricted (`c [source,sh] -------------------------------------------- -Unable to complete saved object migrations for the [.kibana] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}} +Unable to complete saved object migrations for the [.kibana] index: [unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}} -------------------------------------------- To get around the issue, remove the transient and persisted routing allocation settings: diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts index 74549f4e32b57..14fd80c3a8552 100644 --- a/packages/kbn-doc-links/src/get_doc_links.ts +++ b/packages/kbn-doc-links/src/get_doc_links.ts @@ -648,6 +648,8 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => { }, kibanaUpgradeSavedObjects: { resolveMigrationFailures: `${KIBANA_DOCS}resolve-migrations-failures.html`, + repeatedTimeoutRequests: `${KIBANA_DOCS}resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail`, + routingAllocationDisabled: `${KIBANA_DOCS}resolve-migrations-failures.html#routing-allocation-disabled`, }, }); }; diff --git a/packages/kbn-doc-links/src/types.ts b/packages/kbn-doc-links/src/types.ts index bf378c4cf2dd1..4a5a9fdeb9576 100644 --- a/packages/kbn-doc-links/src/types.ts +++ b/packages/kbn-doc-links/src/types.ts @@ -403,5 +403,7 @@ export interface DocLinks { }; readonly kibanaUpgradeSavedObjects: { readonly resolveMigrationFailures: string; + readonly repeatedTimeoutRequests: string; + readonly routingAllocationDisabled: string; }; } diff --git a/src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap b/src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap index b1dcd51bbdd0d..d26021d28b0e5 100644 --- a/src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap +++ b/src/core/server/saved_objects/migrations/__snapshots__/migrations_state_action_machine.test.ts.snap @@ -33,7 +33,9 @@ Object { ], "maxBatchSizeBytes": 100000000, "migrationDocLinks": Object { + "repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail", "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", + "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { @@ -197,7 +199,9 @@ Object { ], "maxBatchSizeBytes": 100000000, "migrationDocLinks": Object { + "repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail", "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", + "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { @@ -365,7 +369,9 @@ Object { ], "maxBatchSizeBytes": 100000000, "migrationDocLinks": Object { + "repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail", "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", + "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { @@ -537,7 +543,9 @@ Object { ], "maxBatchSizeBytes": 100000000, "migrationDocLinks": Object { + "repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail", "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", + "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, "outdatedDocuments": Array [], "outdatedDocumentsQuery": Object { @@ -735,7 +743,9 @@ Object { ], "maxBatchSizeBytes": 100000000, "migrationDocLinks": Object { + "repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail", "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", + "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, "outdatedDocuments": Array [ Object { @@ -910,7 +920,9 @@ Object { ], "maxBatchSizeBytes": 100000000, "migrationDocLinks": Object { + "repeatedTimeoutRequests": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail", "resolveMigrationFailures": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html", + "routingAllocationDisabled": "https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled", }, "outdatedDocuments": Array [ Object { diff --git a/src/core/server/saved_objects/migrations/actions/initialize_action.ts b/src/core/server/saved_objects/migrations/actions/initialize_action.ts index 73502382c9ca0..281e3a0a4f3e0 100644 --- a/src/core/server/saved_objects/migrations/actions/initialize_action.ts +++ b/src/core/server/saved_objects/migrations/actions/initialize_action.ts @@ -29,6 +29,7 @@ export interface InitActionParams { export interface UnsupportedClusterRoutingAllocation { type: 'unsupported_cluster_routing_allocation'; + message: string; } export const checkClusterRoutingAllocationEnabledTask = @@ -53,7 +54,11 @@ export const checkClusterRoutingAllocationEnabledTask = [...clusterRoutingAllocations].every((s: string) => s === 'all'); // if set, only allow 'all' if (!clusterRoutingAllocationEnabled) { - return Either.left({ type: 'unsupported_cluster_routing_allocation' as const }); + return Either.left({ + type: 'unsupported_cluster_routing_allocation' as const, + message: + '[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.', + }); } else { return Either.right({}); } diff --git a/src/core/server/saved_objects/migrations/actions/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrations/actions/integration_tests/actions.test.ts index 5e840d87ea1ab..9846e5f48dc21 100644 --- a/src/core/server/saved_objects/migrations/actions/integration_tests/actions.test.ts +++ b/src/core/server/saved_objects/migrations/actions/integration_tests/actions.test.ts @@ -167,6 +167,7 @@ describe('migration actions', () => { Object { "_tag": "Left", "left": Object { + "message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.", "type": "unsupported_cluster_routing_allocation", }, } @@ -187,6 +188,7 @@ describe('migration actions', () => { Object { "_tag": "Left", "left": Object { + "message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.", "type": "unsupported_cluster_routing_allocation", }, } @@ -207,6 +209,7 @@ describe('migration actions', () => { Object { "_tag": "Left", "left": Object { + "message": "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue.", "type": "unsupported_cluster_routing_allocation", }, } @@ -395,14 +398,14 @@ describe('migration actions', () => { timeout: '1s', }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "message": "[index_not_yellow_timeout] Timeout waiting for the status of the [red_index] index to become 'yellow'", - "type": "index_not_yellow_timeout", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "message": "[index_not_yellow_timeout] Timeout waiting for the status of the [red_index] index to become 'yellow'", + "type": "index_not_yellow_timeout", + }, + } + `); }); }); @@ -422,14 +425,14 @@ describe('migration actions', () => { }); expect.assertions(1); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": Object { - "acknowledged": true, - "shardsAcknowledged": true, - }, - } - `); + Object { + "_tag": "Right", + "right": Object { + "acknowledged": true, + "shardsAcknowledged": true, + }, + } + `); }); it('resolves right after waiting for index status to be yellow if clone target already existed', async () => { expect.assertions(2); @@ -488,14 +491,14 @@ describe('migration actions', () => { expect.assertions(1); const task = cloneIndex({ client, source: 'no_such_index', target: 'clone_target_3' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "index": "no_such_index", - "type": "index_not_found_exception", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "index": "no_such_index", + "type": "index_not_found_exception", + }, + } + `); }); it('resolves left with a index_not_yellow_timeout if clone target already exists but takes longer than the specified timeout before turning yellow', async () => { // Create a red index @@ -524,14 +527,14 @@ describe('migration actions', () => { })(); await expect(cloneIndexPromise).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "message": "[index_not_yellow_timeout] Timeout waiting for the status of the [clone_red_index] index to become 'yellow'", - "type": "index_not_yellow_timeout", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "message": "[index_not_yellow_timeout] Timeout waiting for the status of the [clone_red_index] index to become 'yellow'", + "type": "index_not_yellow_timeout", + }, + } + `); // Now that we know timeouts work, make the index yellow again and call cloneIndex a second time to verify that it completes @@ -552,14 +555,14 @@ describe('migration actions', () => { })(); await expect(cloneIndexPromise2).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": Object { - "acknowledged": true, - "shardsAcknowledged": true, - }, - } - `); + Object { + "_tag": "Right", + "right": Object { + "acknowledged": true, + "shardsAcknowledged": true, + }, + } + `); }); }); @@ -577,11 +580,11 @@ describe('migration actions', () => { })()) as Either.Right; const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); const results = ( (await searchForOutdatedDocuments(client, { @@ -617,11 +620,11 @@ describe('migration actions', () => { })()) as Either.Right; const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); const results = ( (await searchForOutdatedDocuments(client, { @@ -650,11 +653,11 @@ describe('migration actions', () => { })()) as Either.Right; const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); const results = ( (await searchForOutdatedDocuments(client, { batchSize: 1000, @@ -685,11 +688,11 @@ describe('migration actions', () => { })()) as Either.Right; let task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); // reindex without a script res = (await reindex({ @@ -702,11 +705,11 @@ describe('migration actions', () => { })()) as Either.Right; task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); // Assert that documents weren't overridden by the second, unscripted reindex const results = ( @@ -761,11 +764,11 @@ describe('migration actions', () => { })()) as Either.Right; const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "reindex_succeeded", + } + `); // Assert that existing documents weren't overridden, but that missing // documents were added by the reindex const results = ( @@ -818,13 +821,13 @@ describe('migration actions', () => { const task = waitForReindexTask({ client, taskId: reindexTaskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "incompatible_mapping_exception", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "incompatible_mapping_exception", + }, + } + `); }); it('resolves left incompatible_mapping_exception if all reindex failures are due to a mapper_parsing_exception', async () => { expect.assertions(1); @@ -857,13 +860,13 @@ describe('migration actions', () => { const task = waitForReindexTask({ client, taskId: reindexTaskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "incompatible_mapping_exception", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "incompatible_mapping_exception", + }, + } + `); }); it('resolves left index_not_found_exception if source index does not exist', async () => { expect.assertions(1); @@ -879,14 +882,14 @@ describe('migration actions', () => { })()) as Either.Right; const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "index": "no_such_index", - "type": "index_not_found_exception", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "index": "no_such_index", + "type": "index_not_found_exception", + }, + } + `); }); it('resolves left target_index_had_write_block if all failures are due to a write block', async () => { expect.assertions(1); @@ -902,13 +905,13 @@ describe('migration actions', () => { const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "target_index_had_write_block", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "target_index_had_write_block", + }, + } + `); }); it('resolves left if requireAlias=true and the target is not an alias', async () => { expect.assertions(1); @@ -924,14 +927,14 @@ describe('migration actions', () => { const task = waitForReindexTask({ client, taskId: res.right.taskId, timeout: '10s' }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "index": "existing_index_with_write_block", - "type": "index_not_found_exception", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "index": "existing_index_with_write_block", + "type": "index_not_found_exception", + }, + } + `); }); it('resolves left wait_for_task_completion_timeout when the task does not finish within the timeout', async () => { @@ -983,11 +986,11 @@ describe('migration actions', () => { targetIndex: 'reindex_target_7', }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "verify_reindex_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "verify_reindex_succeeded", + } + `); }); it('resolves left if source and target indices have different amount of documents', async () => { expect.assertions(1); @@ -997,13 +1000,13 @@ describe('migration actions', () => { targetIndex: 'existing_index_2', }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "verify_reindex_failed", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "verify_reindex_failed", + }, + } + `); }); it('rejects if source or target index does not exist', async () => { expect.assertions(2); @@ -1630,11 +1633,11 @@ describe('migration actions', () => { }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Right", - "right": "bulk_index_succeeded", - } - `); + Object { + "_tag": "Right", + "right": "bulk_index_succeeded", + } + `); }); it('resolves right even if there were some version_conflict_engine_exception', async () => { const existingDocs = ( @@ -1675,13 +1678,13 @@ describe('migration actions', () => { refresh: 'wait_for', })() ).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "target_index_had_write_block", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "target_index_had_write_block", + }, + } + `); }); it('resolves left request_entity_too_large_exception when the payload is too large', async () => { @@ -1697,13 +1700,13 @@ describe('migration actions', () => { transformedDocs: newDocs, }); await expect(task()).resolves.toMatchInlineSnapshot(` - Object { - "_tag": "Left", - "left": Object { - "type": "request_entity_too_large_exception", - }, - } - `); + Object { + "_tag": "Left", + "left": Object { + "type": "request_entity_too_large_exception", + }, + } + `); }); }); }); diff --git a/src/core/server/saved_objects/migrations/initial_state.test.ts b/src/core/server/saved_objects/migrations/initial_state.test.ts index 0fff4ddb06895..2ad3dc38e6d65 100644 --- a/src/core/server/saved_objects/migrations/initial_state.test.ts +++ b/src/core/server/saved_objects/migrations/initial_state.test.ts @@ -116,6 +116,10 @@ describe('createInitialState', () => { migrationDocLinks: { resolveMigrationFailures: 'https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html', + repeatedTimeoutRequests: + 'https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#_repeated_time_out_requests_that_eventually_fail', + routingAllocationDisabled: + 'https://www.elastic.co/guide/en/kibana/test-branch/resolve-migrations-failures.html#routing-allocation-disabled', }, }); }); diff --git a/src/core/server/saved_objects/migrations/integration_tests/cluster_routing_allocation_disabled.test.ts b/src/core/server/saved_objects/migrations/integration_tests/cluster_routing_allocation_disabled.test.ts index ea70478d6ce7b..37b278fe9ccf0 100644 --- a/src/core/server/saved_objects/migrations/integration_tests/cluster_routing_allocation_disabled.test.ts +++ b/src/core/server/saved_objects/migrations/integration_tests/cluster_routing_allocation_disabled.test.ts @@ -114,7 +114,7 @@ describe('unsupported_cluster_routing_allocation', () => { await root.setup(); await expect(root.start()).rejects.toThrowError( - /Unable to complete saved object migrations for the \[\.kibana.*\] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {"transient": {"cluster\.routing\.allocation\.enable": null}, "persistent": {"cluster\.routing\.allocation\.enable": null}}/ + /Unable to complete saved object migrations for the \[\.kibana\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {\"transient\": {\"cluster\.routing\.allocation\.enable\": null}, \"persistent\": {\"cluster\.routing\.allocation\.enable\": null}}\. Refer to https:\/\/www.elastic.co\/guide\/en\/kibana\/master\/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue\./ ); await retryAsync( @@ -126,7 +126,7 @@ describe('unsupported_cluster_routing_allocation', () => { .map((str) => JSON5.parse(str)) as LogRecord[]; expect( records.find((rec) => - /^Unable to complete saved object migrations for the \[\.kibana.*\] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\./.test( + /^Unable to complete saved object migrations for the \[\.kibana.*\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\./.test( rec.message ) ) @@ -149,7 +149,7 @@ describe('unsupported_cluster_routing_allocation', () => { await root.setup(); await expect(root.start()).rejects.toThrowError( - /Unable to complete saved object migrations for the \[\.kibana.*\] index: The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {"transient": {"cluster\.routing\.allocation\.enable": null}, "persistent": {"cluster\.routing\.allocation\.enable": null}}/ + /Unable to complete saved object migrations for the \[\.kibana\] index: \[unsupported_cluster_routing_allocation\] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue\. To proceed, please remove the cluster routing allocation settings with PUT \/_cluster\/settings {\"transient\": {\"cluster\.routing\.allocation\.enable\": null}, \"persistent\": {\"cluster\.routing\.allocation\.enable\": null}}\. Refer to https:\/\/www.elastic.co\/guide\/en\/kibana\/master\/resolve-migrations-failures.html#routing-allocation-disabled for more information on how to resolve the issue\./ ); }); }); diff --git a/src/core/server/saved_objects/migrations/model/extract_errors.test.ts b/src/core/server/saved_objects/migrations/model/extract_errors.test.ts index a028c40ca6597..e434a5001a6ae 100644 --- a/src/core/server/saved_objects/migrations/model/extract_errors.test.ts +++ b/src/core/server/saved_objects/migrations/model/extract_errors.test.ts @@ -6,7 +6,11 @@ * Side Public License, v 1. */ -import { extractUnknownDocFailureReason } from './extract_errors'; +import { + extractUnknownDocFailureReason, + fatalReasonClusterRoutingAllocationUnsupported, + fatalReasonDocumentExceedsMaxBatchSizeBytes, +} from './extract_errors'; describe('extractUnknownDocFailureReason', () => { it('generates the correct error message', () => { @@ -37,3 +41,32 @@ describe('extractUnknownDocFailureReason', () => { `); }); }); + +describe('fatalReasonDocumentExceedsMaxBatchSizeBytes', () => { + it('generate the correct error message', () => { + expect( + fatalReasonDocumentExceedsMaxBatchSizeBytes({ + _id: 'abc', + docSizeBytes: 106954752, + maxBatchSizeBytes: 104857600, + }) + ).toMatchInlineSnapshot( + `"The document with _id \\"abc\\" is 106954752 bytes which exceeds the configured maximum batch size of 104857600 bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value."` + ); + }); +}); + +describe('fatalReasonClusterRoutingAllocationUnsupported', () => { + it('generates the correct error message', () => { + const errorMessages = fatalReasonClusterRoutingAllocationUnsupported({ + errorMessage: '[some-error] message', + docSectionLink: 'linkToDocsSection', + }); + expect(errorMessages.fatalReason).toMatchInlineSnapshot( + `"[some-error] message To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}. Refer to linkToDocsSection for more information on how to resolve the issue."` + ); + expect(errorMessages.logsErrorMessage).toMatchInlineSnapshot( + `"[some-error] message Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'. Refer to linkToDocsSection for more information on how to resolve the issue."` + ); + }); +}); diff --git a/src/core/server/saved_objects/migrations/model/extract_errors.ts b/src/core/server/saved_objects/migrations/model/extract_errors.ts index 95d10603caa80..f41009ab2127c 100644 --- a/src/core/server/saved_objects/migrations/model/extract_errors.ts +++ b/src/core/server/saved_objects/migrations/model/extract_errors.ts @@ -51,3 +51,32 @@ export function extractUnknownDocFailureReason( `'` ); } + +/** + * Constructs migration failure message string for doc exceeds max batch size in bytes + */ +export const fatalReasonDocumentExceedsMaxBatchSizeBytes = ({ + _id, + docSizeBytes, + maxBatchSizeBytes, +}: { + _id: string; + docSizeBytes: number; + maxBatchSizeBytes: number; +}) => + `The document with _id "${_id}" is ${docSizeBytes} bytes which exceeds the configured maximum batch size of ${maxBatchSizeBytes} bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`; + +/** + * Constructs migration failure message and logs message strings when an unsupported cluster routing allocation is configured. + * The full errorMessage is "[unsupported_cluster_routing_allocation] The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue." + */ +export const fatalReasonClusterRoutingAllocationUnsupported = ({ + errorMessage, + docSectionLink, +}: { + errorMessage: string; + docSectionLink: string; +}) => ({ + fatalReason: `${errorMessage} To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}. Refer to ${docSectionLink} for more information on how to resolve the issue.`, + logsErrorMessage: `${errorMessage} Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'. Refer to ${docSectionLink} for more information on how to resolve the issue.`, +}); diff --git a/src/core/server/saved_objects/migrations/model/model.test.ts b/src/core/server/saved_objects/migrations/model/model.test.ts index b80e2bceae846..e44995ac8d30e 100644 --- a/src/core/server/saved_objects/migrations/model/model.test.ts +++ b/src/core/server/saved_objects/migrations/model/model.test.ts @@ -96,6 +96,8 @@ describe('migrations v2 model', () => { excludeFromUpgradeFilterHooks: {}, migrationDocLinks: { resolveMigrationFailures: 'resolveMigrationFailures', + repeatedTimeoutRequests: 'repeatedTimeoutRequests', + routingAllocationDisabled: 'routingAllocationDisabled', }, }; @@ -283,12 +285,13 @@ describe('migrations v2 model', () => { test('INIT -> FATAL when cluster routing allocation is not enabled', () => { const res: ResponseType<'INIT'> = Either.left({ type: 'unsupported_cluster_routing_allocation', + message: '[unsupported_cluster_routing_allocation]', }); const newState = model(initState, res) as FatalState; expect(newState.controlState).toEqual('FATAL'); expect(newState.reason).toMatchInlineSnapshot( - `"The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}"` + `"[unsupported_cluster_routing_allocation] To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {\\"transient\\": {\\"cluster.routing.allocation.enable\\": null}, \\"persistent\\": {\\"cluster.routing.allocation.enable\\": null}}. Refer to routingAllocationDisabled for more information on how to resolve the issue."` ); }); test("INIT -> FATAL when .kibana points to newer version's index", () => { diff --git a/src/core/server/saved_objects/migrations/model/model.ts b/src/core/server/saved_objects/migrations/model/model.ts index e711f62bcd8d6..cff23f0eeda65 100644 --- a/src/core/server/saved_objects/migrations/model/model.ts +++ b/src/core/server/saved_objects/migrations/model/model.ts @@ -21,7 +21,12 @@ import { setProgressTotal, } from './progress'; import { delayRetryState, resetRetryState } from './retry_state'; -import { extractTransformFailuresReason, extractUnknownDocFailureReason } from './extract_errors'; +import { + extractTransformFailuresReason, + extractUnknownDocFailureReason, + fatalReasonDocumentExceedsMaxBatchSizeBytes, + fatalReasonClusterRoutingAllocationUnsupported, +} from './extract_errors'; import type { ExcludeRetryableEsError } from './types'; import { getAliases, @@ -33,17 +38,7 @@ import { } from './helpers'; import { createBatches } from './create_batches'; -const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; -const fatalReasonDocumentExceedsMaxBatchSizeBytes = ({ - _id, - docSizeBytes, - maxBatchSizeBytes, -}: { - _id: string; - docSizeBytes: number; - maxBatchSizeBytes: number; -}) => - `The document with _id "${_id}" is ${docSizeBytes} bytes which exceeds the configured maximum batch size of ${maxBatchSizeBytes} bytes. To proceed, please increase the 'migrations.maxBatchSizeBytes' Kibana configuration option and ensure that the Elasticsearch 'http.max_content_length' configuration option is set to an equal or larger value.`; +export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; export const model = (currentState: State, resW: ResponseType): State => { // The action response `resW` is weakly typed, the type includes all action @@ -73,15 +68,19 @@ export const model = (currentState: State, resW: ResponseType): if (Either.isLeft(res)) { const left = res.left; if (isLeftTypeof(left, 'unsupported_cluster_routing_allocation')) { + const initErrorMessages = fatalReasonClusterRoutingAllocationUnsupported({ + errorMessage: left.message, + docSectionLink: stateP.migrationDocLinks.routingAllocationDisabled, + }); return { ...stateP, controlState: 'FATAL', - reason: `The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. To proceed, please remove the cluster routing allocation settings with PUT /_cluster/settings {"transient": {"cluster.routing.allocation.enable": null}, "persistent": {"cluster.routing.allocation.enable": null}}`, + reason: initErrorMessages.fatalReason, logs: [ ...stateP.logs, { level: 'error', - message: `The elasticsearch cluster has cluster routing allocation incorrectly set for migrations to continue. Ensure that the persistent and transient Elasticsearch configuration option 'cluster.routing.allocation.enable' is not set or set it to a value of 'all'.`, + message: initErrorMessages.logsErrorMessage, }, ], }; @@ -244,7 +243,7 @@ export const model = (currentState: State, resW: ResponseType): // we get a response. // If the cluster hit the low watermark for disk usage the LEGACY_CREATE_REINDEX_TARGET action will // continue to timeout and eventually lead to a failed migration. - const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`; + const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts); } else { return throwBadResponse(stateP, left); @@ -366,7 +365,7 @@ export const model = (currentState: State, resW: ResponseType): // we get a response. // In the event of retries running out, we link to the docs to help with diagnosing // the problem. - const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`; + const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts); } else { return throwBadResponse(stateP, left); @@ -461,7 +460,7 @@ export const model = (currentState: State, resW: ResponseType): // // If there is a problem CREATE_REINDEX_TEMP action will // continue to timeout and eventually lead to a failed migration. - const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`; + const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts); } else { return throwBadResponse(stateP, left); @@ -696,7 +695,7 @@ export const model = (currentState: State, resW: ResponseType): // `_cluster/allocation/explain?index=${targetIndex}` API. // Unless the root cause is identified and addressed, the request will // continue to timeout and eventually lead to a failed migration. - const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`; + const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts); } else { throwBadResponse(stateP, left); @@ -951,7 +950,7 @@ export const model = (currentState: State, resW: ResponseType): // If the cluster hit the low watermark for disk usage the action will continue to timeout. // Unless the disk space is addressed, the LEGACY_CREATE_REINDEX_TARGET action will // continue to timeout and eventually lead to a failed migration. - const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.resolveMigrationFailures} for information on how to resolve the issue.`; + const retryErrorMessage = `${left.message} Refer to ${stateP.migrationDocLinks.repeatedTimeoutRequests} for information on how to resolve the issue.`; return delayRetryState(stateP, retryErrorMessage, stateP.retryAttempts); } else { return throwBadResponse(stateP, left); diff --git a/src/core/server/saved_objects/migrations/state.ts b/src/core/server/saved_objects/migrations/state.ts index 6630b5ee57808..dee6839d6b902 100644 --- a/src/core/server/saved_objects/migrations/state.ts +++ b/src/core/server/saved_objects/migrations/state.ts @@ -8,6 +8,7 @@ import * as Option from 'fp-ts/lib/Option'; import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { DocLinks } from '@kbn/doc-links'; import { ControlState } from './state_action_machine'; import { AliasAction } from './actions'; import { IndexMapping } from '../mappings'; @@ -125,7 +126,7 @@ export interface BaseState extends ControlState { /** * DocLinks for savedObjects. to reference online documentation */ - readonly migrationDocLinks: Record; + readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects']; } export interface InitState extends BaseState {