From 35252ee11782cedcb83b765a5df0c04eec5cbcd4 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Mon, 27 Sep 2021 15:28:52 +0200 Subject: [PATCH 01/11] Remove server side logic --- .../server/lib/reindexing/reindex_actions.ts | 2 - .../lib/reindexing/reindex_service.test.ts | 528 +----------------- .../server/lib/reindexing/reindex_service.ts | 121 ---- .../reindex_indices/reindex_indices.test.ts | 21 +- .../routes/reindex_indices/reindex_indices.ts | 2 - 5 files changed, 2 insertions(+), 672 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts index 8e63367802cd9..f2cf16bd7c4bd 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts @@ -92,8 +92,6 @@ export interface ReindexActions { withTypeName?: boolean ): Promise; - // ----- Functions below are for enforcing locks around groups of indices like ML or Watcher - /** * Atomically increments the number of reindex operations running for an index group. */ diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 7a5bf1c187698..848bf5e425063 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -14,7 +14,6 @@ import { elasticsearchServiceMock, loggingSystemMock } from 'src/core/server/moc import { ScopedClusterClientMock } from 'src/core/server/elasticsearch/client/mocks'; import { - IndexGroup, ReindexOperation, ReindexSavedObject, ReindexStatus, @@ -28,12 +27,7 @@ import { getMockVersionInfo } from '../__fixtures__/version'; import { esIndicesStateCheck } from '../es_indices_state_check'; import { versionService } from '../version'; -import { - isMlIndex, - isWatcherIndex, - ReindexService, - reindexServiceFactory, -} from './reindex_service'; +import { ReindexService, reindexServiceFactory } from './reindex_service'; const asApiResponse = (body: T): RequestEvent => ({ @@ -129,31 +123,6 @@ describe('reindexService', () => { }); }); - it('includes manage_ml for ML indices', async () => { - clusterClient.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce( - // @ts-expect-error not full interface - asApiResponse({ has_all_requested: true }) - ); - - await service.hasRequiredPrivileges('.ml-anomalies'); - expect(clusterClient.asCurrentUser.security.hasPrivileges).toHaveBeenCalledWith({ - body: { - cluster: ['manage', 'manage_ml'], - index: [ - { - names: ['.ml-anomalies', `.reindexed-v${currentMajor}-ml-anomalies`], - allow_restricted_indices: true, - privileges: ['all'], - }, - { - names: ['.tasks'], - privileges: ['read', 'delete'], - }, - ], - }, - }); - }); - it('includes checking for permissions on the baseName which could be an alias', async () => { clusterClient.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce( // @ts-expect-error not full interface @@ -183,33 +152,6 @@ describe('reindexService', () => { }, }); }); - - it('includes manage_watcher for watcher indices', async () => { - clusterClient.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce( - // @ts-expect-error not full interface - asApiResponse({ - has_all_requested: true, - }) - ); - - await service.hasRequiredPrivileges('.watches'); - expect(clusterClient.asCurrentUser.security.hasPrivileges).toHaveBeenCalledWith({ - body: { - cluster: ['manage', 'manage_watcher'], - index: [ - { - names: ['.watches', `.reindexed-v${currentMajor}-watches`], - allow_restricted_indices: true, - privileges: ['all'], - }, - { - names: ['.tasks'], - privileges: ['read', 'delete'], - }, - ], - }, - }); - }); }); describe('detectReindexWarnings', () => { @@ -496,40 +438,6 @@ describe('reindexService', () => { }); }); - describe('isMlIndex', () => { - it('is false for non-ml indices', () => { - expect(isMlIndex('.literally-anything')).toBe(false); - }); - - it('is true for ML indices', () => { - expect(isMlIndex('.ml-state')).toBe(true); - expect(isMlIndex('.ml-anomalies')).toBe(true); - expect(isMlIndex('.ml-config')).toBe(true); - }); - - it('is true for ML re-indexed indices', () => { - expect(isMlIndex(`.reindexed-v${prevMajor}-ml-state`)).toBe(true); - expect(isMlIndex(`.reindexed-v${prevMajor}-ml-anomalies`)).toBe(true); - expect(isMlIndex(`.reindexed-v${prevMajor}-ml-config`)).toBe(true); - }); - }); - - describe('isWatcherIndex', () => { - it('is false for non-watcher indices', () => { - expect(isWatcherIndex('.literally-anything')).toBe(false); - }); - - it('is true for watcher indices', () => { - expect(isWatcherIndex('.watches')).toBe(true); - expect(isWatcherIndex('.triggered-watches')).toBe(true); - }); - - it('is true for watcher re-indexed indices', () => { - expect(isWatcherIndex(`.reindexed-v${prevMajor}-watches`)).toBe(true); - expect(isWatcherIndex(`.reindexed-v${prevMajor}-triggered-watches`)).toBe(true); - }); - }); - describe('state machine, lastCompletedStep ===', () => { const defaultAttributes = { indexName: 'myIndex', @@ -541,241 +449,6 @@ describe('reindexService', () => { mappings: { _doc: { properties: { timestampl: { type: 'date' } } } }, }; - describe('created', () => { - const reindexOp = { - id: '1', - attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.created }, - } as ReindexSavedObject; - - describe('ml behavior', () => { - const mlReindexOp = { - id: '2', - attributes: { ...reindexOp.attributes, indexName: '.ml-anomalies' }, - } as ReindexSavedObject; - - it('does nothing if index is not an ML index', async () => { - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(actions.incrementIndexGroupReindexes).not.toHaveBeenCalled(); - expect(actions.runWhileIndexGroupLocked).not.toHaveBeenCalled(); - expect(clusterClient.asCurrentUser.nodes.info).not.toHaveBeenCalled(); - }); - - it('supports an already migrated ML index', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f() - ); - clusterClient.asCurrentUser.nodes.info.mockResolvedValueOnce( - // @ts-expect-error not full interface - asApiResponse({ nodes: { nodeX: { version: '6.7.0-alpha' } } }) - ); - clusterClient.asCurrentUser.ml.setUpgradeMode.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - - const mlReindexedOp = { - id: '2', - attributes: { - ...reindexOp.attributes, - indexName: `.reindexed-v${prevMajor}-ml-anomalies`, - }, - } as ReindexSavedObject; - const updatedOp = await service.processNextStep(mlReindexedOp); - - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(actions.incrementIndexGroupReindexes).toHaveBeenCalled(); - expect(actions.runWhileIndexGroupLocked).toHaveBeenCalled(); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).toHaveBeenCalledWith({ - enabled: true, - }); - }); - - it('increments ML reindexes and calls ML stop endpoint', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f() - ); - - clusterClient.asCurrentUser.nodes.info.mockResolvedValueOnce( - // @ts-expect-error not full interface - asApiResponse({ nodes: { nodeX: { version: '6.7.0-alpha' } } }) - ); - clusterClient.asCurrentUser.ml.setUpgradeMode.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(actions.incrementIndexGroupReindexes).toHaveBeenCalled(); - expect(actions.runWhileIndexGroupLocked).toHaveBeenCalled(); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).toHaveBeenCalledWith({ - enabled: true, - }); - }); - - it('fails if ML reindexes cannot be incremented', async () => { - actions.incrementIndexGroupReindexes.mockRejectedValueOnce(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalledWith({ - enabled: true, - }); - }); - - it('fails if ML doc cannot be locked', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockRejectedValueOnce(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalledWith({ - enabled: true, - }); - }); - - it('fails if ML endpoint fails', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f() - ); - clusterClient.asCurrentUser.nodes.info.mockResolvedValueOnce( - // @ts-expect-error not full interface - asApiResponse({ nodes: { nodeX: { version: '6.7.0' } } }) - ); - clusterClient.asCurrentUser.ml.setUpgradeMode.mockResolvedValueOnce( - asApiResponse({ acknowledged: false }) - ); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect( - updatedOp.attributes.errorMessage!.includes('Could not stop ML jobs') - ).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).toHaveBeenCalledWith({ - enabled: true, - }); - }); - - it('fails if not all nodes have been upgraded to 6.7.0', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f() - ); - clusterClient.asCurrentUser.nodes.info.mockResolvedValueOnce( - // @ts-expect-error not full interface - asApiResponse({ nodes: { nodeX: { version: '6.6.0' } } }) - ); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect( - updatedOp.attributes.errorMessage!.includes('Some nodes are not on minimum version') - ).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - // Should not have called ML endpoint at all - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalledWith({ - enabled: true, - }); - }); - }); - - describe('watcher behavior', () => { - const watcherReindexOp = { - id: '2', - attributes: { ...reindexOp.attributes, indexName: '.watches' }, - } as ReindexSavedObject; - - it('does nothing if index is not a watcher index', async () => { - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(actions.incrementIndexGroupReindexes).not.toHaveBeenCalled(); - expect(actions.runWhileIndexGroupLocked).not.toHaveBeenCalled(); - expect(clusterClient.asCurrentUser.watcher.stop).not.toHaveBeenCalled(); - }); - - it('increments ML reindexes and calls watcher stop endpoint', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (type: string, f: any) => - f() - ); - clusterClient.asCurrentUser.watcher.stop.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(actions.incrementIndexGroupReindexes).toHaveBeenCalledWith(IndexGroup.watcher); - expect(actions.runWhileIndexGroupLocked).toHaveBeenCalled(); - expect(clusterClient.asCurrentUser.watcher.stop).toHaveBeenCalled(); - }); - - it('fails if watcher reindexes cannot be incremented', async () => { - actions.incrementIndexGroupReindexes.mockRejectedValueOnce(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.watcher.stop).not.toHaveBeenCalledWith({ - enabled: true, - }); - }); - - it('fails if watcher doc cannot be locked', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockRejectedValueOnce(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.watcher.stop).not.toHaveBeenCalled(); - }); - - it('fails if watcher endpoint fails', async () => { - actions.incrementIndexGroupReindexes.mockResolvedValueOnce(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (type: string, f: any) => - f() - ); - clusterClient.asCurrentUser.watcher.stop.mockResolvedValueOnce( - asApiResponse({ acknowledged: false }) - ); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect( - updatedOp.attributes.errorMessage!.includes('Could not stop Watcher') - ).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.watcher.stop).toHaveBeenCalled(); - }); - }); - }); - describe('indexConsumersStopped', () => { const reindexOp = { id: '1', @@ -1128,205 +801,6 @@ describe('reindexService', () => { }); }); - describe('aliasCreated', () => { - const reindexOp = { - id: '1', - attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.aliasCreated }, - } as ReindexSavedObject; - - describe('ml behavior', () => { - const mlReindexOp = { - id: '2', - attributes: { ...reindexOp.attributes, indexName: '.ml-anomalies' }, - } as ReindexSavedObject; - - it('does nothing if index is not an ML index', async () => { - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStarted - ); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalled(); - }); - - it('decrements ML reindexes and calls ML start endpoint if no remaining ML jobs', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f({ attributes: { runningReindexCount: 0 } }) - ); - clusterClient.asCurrentUser.ml.setUpgradeMode.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(actions.decrementIndexGroupReindexes).toHaveBeenCalledWith(IndexGroup.ml); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStarted - ); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).toHaveBeenCalledWith({ - enabled: false, - }); - }); - - it('does not call ML start endpoint if there are remaining ML jobs', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f({ attributes: { runningReindexCount: 2 } }) - ); - clusterClient.asCurrentUser.ml.setUpgradeMode.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStarted - ); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalledWith({ - enabled: false, - }); - }); - - it('fails if ML reindexes cannot be decremented', async () => { - // Mock unable to lock ml doc - actions.decrementIndexGroupReindexes.mockRejectedValue(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalledWith({ - enabled: false, - }); - }); - - it('fails if ML doc cannot be locked', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - // Mock unable to lock ml doc - actions.runWhileIndexGroupLocked.mockRejectedValueOnce(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).not.toHaveBeenCalledWith({ - enabled: false, - }); - }); - - it('fails if ML endpoint fails', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f({ attributes: { runningReindexCount: 0 } }) - ); - clusterClient.asCurrentUser.ml.setUpgradeMode.mockResolvedValueOnce( - asApiResponse({ acknowledged: false }) - ); - const updatedOp = await service.processNextStep(mlReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect( - updatedOp.attributes.errorMessage!.includes('Could not resume ML jobs') - ).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.ml.setUpgradeMode).toHaveBeenCalledWith({ - enabled: false, - }); - }); - }); - - describe('watcher behavior', () => { - const watcherReindexOp = { - id: '2', - attributes: { ...reindexOp.attributes, indexName: '.watches' }, - } as ReindexSavedObject; - - it('does nothing if index is not a watcher index', async () => { - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStarted - ); - expect(clusterClient.asCurrentUser.watcher.start).not.toHaveBeenCalled(); - }); - - it('decrements watcher reindexes and calls wathcer start endpoint if no remaining watcher reindexes', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f({ attributes: { runningReindexCount: 0 } }) - ); - clusterClient.asCurrentUser.watcher.start.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(actions.decrementIndexGroupReindexes).toHaveBeenCalledWith(IndexGroup.watcher); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStarted - ); - expect(clusterClient.asCurrentUser.watcher.start).toHaveBeenCalled(); - }); - - it('does not call watcher start endpoint if there are remaining watcher reindexes', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f({ attributes: { runningReindexCount: 2 } }) - ); - clusterClient.asCurrentUser.watcher.start.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStarted - ); - expect(clusterClient.asCurrentUser.watcher.start).not.toHaveBeenCalledWith(); - }); - - it('fails if watcher reindexes cannot be decremented', async () => { - // Mock unable to lock watcher doc - actions.decrementIndexGroupReindexes.mockRejectedValue(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.watcher.start).not.toHaveBeenCalledWith(); - }); - - it('fails if watcher doc cannot be locked', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - // Mock unable to lock watcher doc - actions.runWhileIndexGroupLocked.mockRejectedValueOnce(new Error(`Can't lock!`)); - - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.watcher.start).not.toHaveBeenCalledWith(); - }); - - it('fails if watcher endpoint fails', async () => { - actions.decrementIndexGroupReindexes.mockResolvedValue(); - actions.runWhileIndexGroupLocked.mockImplementationOnce(async (group: string, f: any) => - f({ attributes: { runningReindexCount: 0 } }) - ); - - clusterClient.asCurrentUser.watcher.start.mockResolvedValueOnce( - asApiResponse({ acknowledged: false }) - ); - const updatedOp = await service.processNextStep(watcherReindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect( - updatedOp.attributes.errorMessage!.includes('Could not start Watcher') - ).toBeTruthy(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - expect(clusterClient.asCurrentUser.watcher.start).toHaveBeenCalled(); - }); - }); - }); - describe('indexGroupServicesStarted', () => { const reindexOp = { id: '1', diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index a8f0708045951..ffd36fe2b27e1 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -49,12 +49,6 @@ export interface ReindexService { */ detectReindexWarnings(indexName: string): Promise; - /** - * Returns an IndexGroup if the index belongs to one, otherwise undefined. - * @param indexName - */ - getIndexGroup(indexName: string): IndexGroup | undefined; - /** * Creates a new reindex operation for a given index. * @param indexName @@ -135,83 +129,6 @@ export const reindexServiceFactory = ( licensing: LicensingPluginSetup ): ReindexService => { // ------ Utility functions - - /** - * If the index is a ML index that will cause jobs to fail when set to readonly, - * turn on 'upgrade mode' to pause all ML jobs. - * @param reindexOp - */ - const stopMlJobs = async () => { - await actions.incrementIndexGroupReindexes(IndexGroup.ml); - await actions.runWhileIndexGroupLocked(IndexGroup.ml, async (mlDoc) => { - await validateNodesMinimumVersion(6, 7); - - const { body } = await esClient.ml.setUpgradeMode({ - enabled: true, - }); - - if (!body.acknowledged) { - throw new Error(`Could not stop ML jobs`); - } - - return mlDoc; - }); - }; - - /** - * Resumes ML jobs if there are no more remaining reindex operations. - */ - const resumeMlJobs = async () => { - await actions.decrementIndexGroupReindexes(IndexGroup.ml); - await actions.runWhileIndexGroupLocked(IndexGroup.ml, async (mlDoc) => { - if (mlDoc.attributes.runningReindexCount === 0) { - const { body } = await esClient.ml.setUpgradeMode({ - enabled: false, - }); - - if (!body.acknowledged) { - throw new Error(`Could not resume ML jobs`); - } - } - - return mlDoc; - }); - }; - - /** - * Stops Watcher in Elasticsearch. - */ - const stopWatcher = async () => { - await actions.incrementIndexGroupReindexes(IndexGroup.watcher); - await actions.runWhileIndexGroupLocked(IndexGroup.watcher, async (watcherDoc) => { - const { body } = await esClient.watcher.stop(); - - if (!body.acknowledged) { - throw new Error('Could not stop Watcher'); - } - - return watcherDoc; - }); - }; - - /** - * Starts Watcher in Elasticsearch. - */ - const startWatcher = async () => { - await actions.decrementIndexGroupReindexes(IndexGroup.watcher); - await actions.runWhileIndexGroupLocked(IndexGroup.watcher, async (watcherDoc) => { - if (watcherDoc.attributes.runningReindexCount === 0) { - const { body } = await esClient.watcher.start(); - - if (!body.acknowledged) { - throw new Error('Could not start Watcher'); - } - } - - return watcherDoc; - }); - }; - const cleanupChanges = async (reindexOp: ReindexSavedObject) => { // Cancel reindex task if it was started but not completed if (reindexOp.attributes.lastCompletedStep === ReindexStep.reindexStarted) { @@ -270,12 +187,6 @@ export const reindexServiceFactory = ( }; const stopIndexGroupServices = async (reindexOp: ReindexSavedObject) => { - if (isMlIndex(reindexOp.attributes.indexName)) { - await stopMlJobs(); - } else if (isWatcherIndex(reindexOp.attributes.indexName)) { - await stopWatcher(); - } - return actions.updateReindexOp(reindexOp, { lastCompletedStep: ReindexStep.indexGroupServicesStopped, }); @@ -477,12 +388,6 @@ export const reindexServiceFactory = ( }; const resumeIndexGroupServices = async (reindexOp: ReindexSavedObject) => { - if (isMlIndex(reindexOp.attributes.indexName)) { - await resumeMlJobs(); - } else if (isWatcherIndex(reindexOp.attributes.indexName)) { - await startWatcher(); - } - // Only change the status if we're still in-progress (this function is also called when the reindex fails or is cancelled) if (reindexOp.attributes.status === ReindexStatus.inProgress) { return actions.updateReindexOp(reindexOp, { @@ -537,14 +442,6 @@ export const reindexServiceFactory = ( ], } as any; - if (isMlIndex(indexName)) { - body.cluster = [...body.cluster, 'manage_ml']; - } - - if (isWatcherIndex(indexName)) { - body.cluster = [...body.cluster, 'manage_watcher']; - } - const { body: resp } = await esClient.security.hasPrivileges({ body, }); @@ -561,14 +458,6 @@ export const reindexServiceFactory = ( } }, - getIndexGroup(indexName: string) { - if (isMlIndex(indexName)) { - return IndexGroup.ml; - } else if (isWatcherIndex(indexName)) { - return IndexGroup.watcher; - } - }, - async createReindexOperation(indexName: string, opts?: { enqueue: boolean }) { const { body: indexExists } = await esClient.indices.exists({ index: indexName }); if (!indexExists) { @@ -767,13 +656,3 @@ export const reindexServiceFactory = ( }, }; }; - -export const isMlIndex = (indexName: string) => { - const sourceName = sourceNameForIndex(indexName); - return ML_INDICES.indexOf(sourceName) >= 0; -}; - -export const isWatcherIndex = (indexName: string) => { - const sourceName = sourceNameForIndex(indexName); - return WATCHER_INDICES.indexOf(sourceName) >= 0; -}; diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts index 44331799a160b..b88f565782f77 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts @@ -33,7 +33,7 @@ jest.mock('../../lib/reindexing', () => { }; }); -import { IndexGroup, ReindexSavedObject, ReindexStatus } from '../../../common/types'; +import { ReindexSavedObject, ReindexStatus } from '../../../common/types'; import { credentialStoreFactory } from '../../lib/reindexing/credential_store'; import { registerReindexIndicesRoutes } from './reindex_indices'; @@ -143,25 +143,6 @@ describe('reindex API', () => { expect(data.reindexOp).toBeNull(); expect(data.warnings).toBeNull(); }); - - it('returns the indexGroup for ML indices', async () => { - mockReindexService.findReindexOperation.mockResolvedValueOnce(null); - mockReindexService.detectReindexWarnings.mockResolvedValueOnce([]); - mockReindexService.getIndexGroup.mockReturnValue(IndexGroup.ml); - - const resp = await routeDependencies.router.getHandler({ - method: 'get', - pathPattern: '/api/upgrade_assistant/reindex/{indexName}', - })( - routeHandlerContextMock, - createRequestMock({ params: { indexName: 'anIndex' } }), - kibanaResponseFactory - ); - - expect(resp.status).toEqual(200); - const data = resp.payload; - expect(data.indexGroup).toEqual(IndexGroup.ml); - }); }); describe('POST /api/upgrade_assistant/reindex/{indexName}', () => { diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts index 6edc03d86cb85..01c31fe118b78 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts @@ -266,13 +266,11 @@ export function registerReindexIndicesRoutes( const warnings = hasRequiredPrivileges ? await reindexService.detectReindexWarnings(indexName) : []; - const indexGroup = reindexService.getIndexGroup(indexName); return response.ok({ body: { reindexOp: reindexOp ? reindexOp.attributes : null, warnings, - indexGroup, hasRequiredPrivileges, }, }); From 9101ccda2c850d3129f607b3ec7fb97dd4c9ae9d Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Sep 2021 09:27:08 +0200 Subject: [PATCH 02/11] Remove last traces from the UI and tests --- .../plugins/upgrade_assistant/common/types.ts | 5 - .../reindex/flyout/progress.test.tsx | 76 +-------------- .../reindex/flyout/progress.tsx | 45 +-------- .../reindex/use_reindex_state.tsx | 9 +- .../lib/reindexing/reindex_actions.test.ts | 43 -------- .../server/lib/reindexing/reindex_actions.ts | 97 ------------------- .../lib/reindexing/reindex_service.test.ts | 3 - .../server/lib/reindexing/reindex_service.ts | 1 - 8 files changed, 4 insertions(+), 275 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts index 6368e3b1e717c..4a3dd3e09bef1 100644 --- a/x-pack/plugins/upgrade_assistant/common/types.ts +++ b/x-pack/plugins/upgrade_assistant/common/types.ts @@ -114,11 +114,6 @@ export interface ReindexWarning { }; } -export enum IndexGroup { - ml = '___ML_REINDEX_LOCK___', - watcher = '___WATCHER_REINDEX_LOCK___', -} - // Telemetry types export const UPGRADE_ASSISTANT_TYPE = 'upgrade-assistant-telemetry'; export const UPGRADE_ASSISTANT_DOC_ID = 'upgrade-assistant-telemetry'; diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx index b49d816302213..4cdf600f2b465 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx @@ -8,7 +8,7 @@ import { shallow } from 'enzyme'; import React from 'react'; -import { IndexGroup, ReindexStatus, ReindexStep } from '../../../../../../../common/types'; +import { ReindexStatus, ReindexStep } from '../../../../../../../common/types'; import type { ReindexState } from '../use_reindex_state'; import { ReindexProgress } from './progress'; @@ -110,78 +110,4 @@ describe('ReindexProgress', () => { expect(reindexStep.children.type.name).toEqual('ReindexProgressBar'); expect(reindexStep.children.props.reindexState.reindexTaskPercComplete).toEqual(0.25); }); - - it('adds steps for index groups', () => { - const wrapper = shallow( - - ); - - expect(wrapper).toMatchInlineSnapshot(` -, - }, - Object { - "status": "incomplete", - "title": , - }, - Object { - "status": "incomplete", - "title": , - }, - Object { - "status": "incomplete", - "title": , - }, - Object { - "status": "incomplete", - "title": , - }, - Object { - "status": "incomplete", - "title": , - }, - ] - } -/> -`); - }); }); diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.tsx index 65a790fe96691..7a2c9943691a3 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.tsx @@ -17,7 +17,7 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { IndexGroup, ReindexStatus, ReindexStep } from '../../../../../../../common/types'; +import { ReindexStatus, ReindexStep } from '../../../../../../../common/types'; import { LoadingState } from '../../../../types'; import type { ReindexState } from '../use_reindex_state'; import { StepProgress, StepProgressStep } from './step_progress'; @@ -118,7 +118,7 @@ export const ReindexProgress: React.FunctionComponent<{ reindexState: ReindexState; cancelReindex: () => void; }> = (props) => { - const { errorMessage, indexGroup, lastCompletedStep = -1, status } = props.reindexState; + const { errorMessage, lastCompletedStep = -1, status } = props.reindexState; const stepDetails = (thisStep: ReindexStep): Pick => { const previousStep = orderedSteps[orderedSteps.indexOf(thisStep) - 1]; @@ -225,46 +225,5 @@ export const ReindexProgress: React.FunctionComponent<{ }, ]; - // If this index is part of an index group, add the approriate group services steps. - if (indexGroup === IndexGroup.ml) { - steps.unshift({ - title: ( - - ), - ...stepDetails(ReindexStep.indexGroupServicesStopped), - }); - steps.push({ - title: ( - - ), - ...stepDetails(ReindexStep.indexGroupServicesStarted), - }); - } else if (indexGroup === IndexGroup.watcher) { - steps.unshift({ - title: ( - - ), - ...stepDetails(ReindexStep.indexGroupServicesStopped), - }); - steps.push({ - title: ( - - ), - ...stepDetails(ReindexStep.indexGroupServicesStarted), - }); - } - return ; }; diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/use_reindex_state.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/use_reindex_state.tsx index e1f01be2e0174..09a108cd88977 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/use_reindex_state.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/use_reindex_state.tsx @@ -8,7 +8,6 @@ import { useRef, useCallback, useState, useEffect } from 'react'; import { - IndexGroup, ReindexOperation, ReindexStatus, ReindexStep, @@ -28,19 +27,17 @@ export interface ReindexState { errorMessage: string | null; reindexWarnings?: ReindexWarning[]; hasRequiredPrivileges?: boolean; - indexGroup?: IndexGroup; } interface StatusResponse { warnings?: ReindexWarning[]; reindexOp?: ReindexOperation; hasRequiredPrivileges?: boolean; - indexGroup?: IndexGroup; } const getReindexState = ( reindexState: ReindexState, - { reindexOp, warnings, hasRequiredPrivileges, indexGroup }: StatusResponse + { reindexOp, warnings, hasRequiredPrivileges }: StatusResponse ) => { const newReindexState = { ...reindexState, @@ -55,10 +52,6 @@ const getReindexState = ( newReindexState.hasRequiredPrivileges = hasRequiredPrivileges; } - if (indexGroup) { - newReindexState.indexGroup = indexGroup; - } - if (reindexOp) { // Prevent the UI flickering back to inProgress after cancelling newReindexState.lastCompletedStep = reindexOp.lastCompletedStep; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts index 3cfdb1fdd3167..32ae7cd9c5ccf 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts @@ -13,7 +13,6 @@ import { ScopedClusterClientMock } from 'src/core/server/elasticsearch/client/mo import moment from 'moment'; import { - IndexGroup, REINDEX_OP_TYPE, ReindexSavedObject, ReindexStatus, @@ -283,46 +282,4 @@ describe('ReindexActions', () => { await expect(actions.getFlatSettings('myIndex')).resolves.toBeNull(); }); }); - - describe('runWhileConsumerLocked', () => { - Object.entries(IndexGroup).forEach(([typeKey, consumerType]) => { - describe(`IndexConsumerType.${typeKey}`, () => { - it('creates the lock doc if it does not exist and executes callback', async () => { - expect.assertions(3); - client.get.mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError()); // mock no ML doc exists yet - client.create.mockImplementationOnce((type: any, attributes: any, { id }: any) => - Promise.resolve({ - type, - id, - attributes, - }) - ); - - let flip = false; - await actions.runWhileIndexGroupLocked(consumerType, async (mlDoc) => { - expect(mlDoc.id).toEqual(consumerType); - expect(mlDoc.attributes.runningReindexCount).toEqual(0); - flip = true; - return mlDoc; - }); - expect(flip).toEqual(true); - }); - - it('fails after 10 attempts to lock', async () => { - client.get.mockResolvedValue({ - type: REINDEX_OP_TYPE, - id: consumerType, - attributes: { mlReindexCount: 0 }, - }); - - client.update.mockRejectedValue(new Error('NO LOCKING!')); - - await expect( - actions.runWhileIndexGroupLocked(consumerType, async (m) => m) - ).rejects.toThrow('Could not acquire lock for ML jobs'); - expect(client.update).toHaveBeenCalledTimes(10); - }, 20000); - }); - }); - }); }); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts index f2cf16bd7c4bd..ba91197938935 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts @@ -13,7 +13,6 @@ import { ElasticsearchClient, } from 'src/core/server'; import { - IndexGroup, REINDEX_OP_TYPE, ReindexOperation, ReindexOptions, @@ -91,31 +90,6 @@ export interface ReindexActions { indexName: string, withTypeName?: boolean ): Promise; - - /** - * Atomically increments the number of reindex operations running for an index group. - */ - incrementIndexGroupReindexes(group: IndexGroup): Promise; - - /** - * Atomically decrements the number of reindex operations running for an index group. - */ - decrementIndexGroupReindexes(group: IndexGroup): Promise; - - /** - * Runs a callback function while locking an index group. - * @param func A function to run with the locked index group lock document. Must return a promise that resolves - * to the updated ReindexSavedObject. - */ - runWhileIndexGroupLocked( - group: IndexGroup, - func: (lockDoc: ReindexSavedObject) => Promise - ): Promise; - - /** - * Exposed only for testing, DO NOT USE. - */ - _fetchAndLockIndexGroupDoc(group: IndexGroup): Promise; } export const reindexActionsFactory = ( @@ -268,76 +242,5 @@ export const reindexActionsFactory = ( return flatSettings.body[indexName]; }, - - async _fetchAndLockIndexGroupDoc(indexGroup) { - const fetchDoc = async () => { - try { - // The IndexGroup enum value (a string) serves as the ID of the lock doc - return await client.get(REINDEX_OP_TYPE, indexGroup); - } catch (e) { - if (client.errors.isNotFoundError(e)) { - return await client.create( - REINDEX_OP_TYPE, - { - indexName: null, - newIndexName: null, - locked: null, - status: null, - lastCompletedStep: null, - reindexTaskId: null, - reindexTaskPercComplete: null, - errorMessage: null, - runningReindexCount: 0, - } as any, - { id: indexGroup } - ); - } else { - throw e; - } - } - }; - - const lockDoc = async (attempt = 1): Promise => { - try { - // Refetch the document each time to avoid version conflicts. - return await acquireLock(await fetchDoc()); - } catch (e) { - if (attempt >= 10) { - throw new Error(`Could not acquire lock for ML jobs`); - } - - await new Promise((resolve) => setTimeout(resolve, 1000)); - return lockDoc(attempt + 1); - } - }; - - return lockDoc(); - }, - - async incrementIndexGroupReindexes(indexGroup) { - this.runWhileIndexGroupLocked(indexGroup, (lockDoc) => - this.updateReindexOp(lockDoc, { - runningReindexCount: lockDoc.attributes.runningReindexCount! + 1, - }) - ); - }, - - async decrementIndexGroupReindexes(indexGroup) { - this.runWhileIndexGroupLocked(indexGroup, (lockDoc) => - this.updateReindexOp(lockDoc, { - runningReindexCount: lockDoc.attributes.runningReindexCount! - 1, - }) - ); - }, - - async runWhileIndexGroupLocked(indexGroup, func) { - let lockDoc = await this._fetchAndLockIndexGroupDoc(indexGroup); - - try { - lockDoc = await func(lockDoc); - } finally { - await releaseLock(lockDoc); - } - }, }; }; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 848bf5e425063..2cb2ff8eeed53 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -63,9 +63,6 @@ describe('reindexService', () => { findAllByStatus: jest.fn(unimplemented('findAllInProgressOperations')), getFlatSettings: jest.fn(unimplemented('getFlatSettings')), cleanupChanges: jest.fn(), - incrementIndexGroupReindexes: jest.fn(unimplemented('incrementIndexGroupReindexes')), - decrementIndexGroupReindexes: jest.fn(unimplemented('decrementIndexGroupReindexes')), - runWhileIndexGroupLocked: jest.fn(async (group: string, f: any) => f({ attributes: {} })), }; clusterClient = elasticsearchServiceMock.createScopedClusterClient(); log = loggingSystemMock.create().get(); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index ffd36fe2b27e1..07adc420f067f 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -11,7 +11,6 @@ import { first } from 'rxjs/operators'; import { LicensingPluginSetup } from '../../../../licensing/server'; import { - IndexGroup, ReindexSavedObject, ReindexStatus, ReindexStep, From a74dbd2d7c5dc56ed02d503feab2abb388609093 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Sep 2021 09:28:32 +0200 Subject: [PATCH 03/11] remove indices list --- .../upgrade_assistant/server/lib/reindexing/reindex_service.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index 07adc420f067f..4a3364a4ea7ff 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -31,8 +31,6 @@ import { ReindexActions } from './reindex_actions'; import { error } from './error'; const VERSION_REGEX = new RegExp(/^([1-9]+)\.([0-9]+)\.([0-9]+)/); -const ML_INDICES = ['.ml-state', '.ml-anomalies', '.ml-config']; -const WATCHER_INDICES = ['.watches', '.triggered-watches']; export interface ReindexService { /** From 9f26bded99582af7e1aa1d2c8242f818155e367f Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Sep 2021 09:42:00 +0200 Subject: [PATCH 04/11] Remove unused translations --- x-pack/plugins/translations/translations/ja-JP.json | 4 ---- x-pack/plugins/translations/translations/zh-CN.json | 4 ---- .../server/routes/reindex_indices/reindex_indices.test.ts | 2 -- 3 files changed, 10 deletions(-) diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 3109ab42a452a..4162eb3bf0c15 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -25820,12 +25820,8 @@ "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.cancelButton.cancellingLabel": "キャンセル中…", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.cancelButton.errorLabel": "キャンセルできませんでした", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.createIndexStepTitle": "新規インデックスを作成中", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.pauseMlStepTitle": "機械学習ジョブを一時停止中", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.readonlyStepTitle": "古いインデックスを読み込み専用に設定中", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.reindexingDocumentsStepTitle": "ドキュメントを再インデックス中", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.resumeMlStepTitle": "機械学習ジョブを再開中", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.resumeWatcherStepTitle": "Watcher を再開中", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.stopWatcherStepTitle": "Watcher を停止中", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklistTitle": "プロセスを再インデックス中", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails": "このインデックスは現在閉じています。アップグレードアシスタントが開き、再インデックスを実行してからインデックスを閉じます。 {reindexingMayTakeLongerEmph}。詳細については {docs} をご覧ください。", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails.reindexingTakesLongerEmphasis": "再インデックスには通常よりも時間がかかることがあります", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 1f4164db5403c..0cd2cd7464643 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -26254,12 +26254,8 @@ "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.cancelButton.cancellingLabel": "正在取消……", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.cancelButton.errorLabel": "无法取消", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.createIndexStepTitle": "正在创建新索引", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.pauseMlStepTitle": "正在暂停 Machine Learning 作业", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.readonlyStepTitle": "正在将旧索引设置为只读", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.reindexingDocumentsStepTitle": "正在重新索引文档", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.resumeMlStepTitle": "正在恢复 Machine Learning 作业", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.resumeWatcherStepTitle": "正在恢复 Watcher", - "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklist.stopWatcherStepTitle": "正在停止 Watcher", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.checklistStep.reindexingChecklistTitle": "重新索引过程", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails": "此索引当前已关闭。升级助手将打开索引，重新索引，然后关闭索引。{reindexingMayTakeLongerEmph}。请参阅文档{docs}以了解更多信息。", "xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails.reindexingTakesLongerEmphasis": "重新索引可能比通常花费更多的时间", diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts index b88f565782f77..d0ceb6883e702 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts @@ -15,7 +15,6 @@ import { createRequestMock } from '../__mocks__/request.mock'; const mockReindexService = { hasRequiredPrivileges: jest.fn(), detectReindexWarnings: jest.fn(), - getIndexGroup: jest.fn(), createReindexOperation: jest.fn(), findAllInProgressOperations: jest.fn(), findReindexOperation: jest.fn(), @@ -66,7 +65,6 @@ describe('reindex API', () => { mockReindexService.hasRequiredPrivileges.mockResolvedValue(true); mockReindexService.detectReindexWarnings.mockReset(); - mockReindexService.getIndexGroup.mockReset(); mockReindexService.createReindexOperation.mockReset(); mockReindexService.findAllInProgressOperations.mockReset(); mockReindexService.findReindexOperation.mockReset(); From 8438791c6870087d3dfa54fac2d3ab7cc47f7d20 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Sep 2021 09:46:15 +0200 Subject: [PATCH 05/11] Fix test --- .../reindex/flyout/progress.test.tsx | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx index 4cdf600f2b465..2809d664b183e 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx @@ -110,4 +110,61 @@ describe('ReindexProgress', () => { expect(reindexStep.children.type.name).toEqual('ReindexProgressBar'); expect(reindexStep.children.props.reindexState.reindexTaskPercComplete).toEqual(0.25); }); + + it('adds steps for index groups', () => { + const wrapper = shallow( + + ); + + expect(wrapper).toMatchInlineSnapshot(` +, + }, + Object { + "status": "incomplete", + "title": , + }, + Object { + "status": "incomplete", + "title": , + }, + Object { + "status": "incomplete", + "title": , + }, + ] + } +/> +`); + }); }); From 128c7d3050172eb1ec1d24d4e7d97f6e3e234920 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 28 Sep 2021 10:01:27 +0200 Subject: [PATCH 06/11] Remove unused function --- .../server/lib/reindexing/reindex_service.ts | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index 4a3364a4ea7ff..8ca9a2922ce8f 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -30,8 +30,6 @@ import { ReindexActions } from './reindex_actions'; import { error } from './error'; -const VERSION_REGEX = new RegExp(/^([1-9]+)\.([0-9]+)\.([0-9]+)/); - export interface ReindexService { /** * Checks whether or not the user has proper privileges required to reindex this index. @@ -163,26 +161,6 @@ export const reindexServiceFactory = ( // ------ Functions used to process the state machine - const validateNodesMinimumVersion = async (minMajor: number, minMinor: number) => { - const { body: nodesResponse } = await esClient.nodes.info(); - - const outDatedNodes = Object.values(nodesResponse.nodes).filter((node: any) => { - const matches = node.version.match(VERSION_REGEX); - const major = parseInt(matches[1], 10); - const minor = parseInt(matches[2], 10); - - // All ES nodes must be >= 6.7.0 to pause ML jobs - return !(major > minMajor || (major === minMajor && minor >= minMinor)); - }); - - if (outDatedNodes.length > 0) { - const nodeList = JSON.stringify(outDatedNodes.map((n: any) => n.name)); - throw new Error( - `Some nodes are not on minimum version (${minMajor}.${minMinor}.0) required: ${nodeList}` - ); - } - }; - const stopIndexGroupServices = async (reindexOp: ReindexSavedObject) => { return actions.updateReindexOp(reindexOp, { lastCompletedStep: ReindexStep.indexGroupServicesStopped, From c39788398806066687411645276717c3929a603c Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Thu, 30 Sep 2021 17:16:20 +0200 Subject: [PATCH 07/11] Remove indexGroupServices start/stop from the state machine --- .../plugins/upgrade_assistant/common/types.ts | 2 - .../server/lib/reindexing/reindex_actions.ts | 5 -- .../lib/reindexing/reindex_service.test.ts | 56 ++----------------- .../server/lib/reindexing/reindex_service.ts | 29 +--------- 4 files changed, 7 insertions(+), 85 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts index 94e1b0cba2fae..f9a13ce75d5c3 100644 --- a/x-pack/plugins/upgrade_assistant/common/types.ts +++ b/x-pack/plugins/upgrade_assistant/common/types.ts @@ -16,13 +16,11 @@ export type DeprecationSource = 'Kibana' | 'Elasticsearch'; export enum ReindexStep { // Enum values are spaced out by 10 to give us room to insert steps in between. created = 0, - indexGroupServicesStopped = 10, readonly = 20, newIndexCreated = 30, reindexStarted = 40, reindexCompleted = 50, aliasCreated = 60, - indexGroupServicesStarted = 70, } export enum ReindexStatus { diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts index ba91197938935..2faa493e122b2 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts @@ -32,11 +32,6 @@ export const LOCK_WINDOW = moment.duration(90, 'seconds'); * This is NOT intended to be used by any other code. */ export interface ReindexActions { - /** - * Namespace for ML-specific actions. - */ - // ml: MlActions; - /** * Creates a new reindexOp, does not perform any pre-flight checks. * @param indexName diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 2cb2ff8eeed53..4316ef5e69d63 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -446,52 +446,6 @@ describe('reindexService', () => { mappings: { _doc: { properties: { timestampl: { type: 'date' } } } }, }; - describe('indexConsumersStopped', () => { - const reindexOp = { - id: '1', - attributes: { - ...defaultAttributes, - lastCompletedStep: ReindexStep.indexGroupServicesStopped, - }, - } as ReindexSavedObject; - - it('blocks writes and updates lastCompletedStep', async () => { - clusterClient.asCurrentUser.indices.putSettings.mockResolvedValueOnce( - asApiResponse({ acknowledged: true }) - ); - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.readonly); - expect(clusterClient.asCurrentUser.indices.putSettings).toHaveBeenCalledWith({ - index: 'myIndex', - body: { settings: { blocks: { write: true } } }, - }); - }); - - it('fails if setting updates are not acknowledged', async () => { - clusterClient.asCurrentUser.indices.putSettings.mockResolvedValueOnce( - asApiResponse({ acknowledged: false }) - ); - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage).not.toBeNull(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - }); - - it('fails if setting updates fail', async () => { - clusterClient.asCurrentUser.indices.putSettings.mockRejectedValueOnce(new Error('blah!')); - const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.lastCompletedStep).toEqual( - ReindexStep.indexGroupServicesStopped - ); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed); - expect(updatedOp.attributes.errorMessage).not.toBeNull(); - expect(log.error).toHaveBeenCalledWith(expect.any(String)); - }); - }); - describe('readonly', () => { const reindexOp = { id: '1', @@ -798,18 +752,20 @@ describe('reindexService', () => { }); }); - describe('indexGroupServicesStarted', () => { + describe('aliasCreated', () => { const reindexOp = { id: '1', attributes: { ...defaultAttributes, - lastCompletedStep: ReindexStep.indexGroupServicesStarted, + lastCompletedStep: ReindexStep.aliasCreated, }, } as ReindexSavedObject; - it('sets to completed', async () => { + it('sets reindex status as complete', async () => { const updatedOp = await service.processNextStep(reindexOp); - expect(updatedOp.attributes.status).toEqual(ReindexStatus.completed); + expect(actions.updateReindexOp).toHaveBeenCalledWith(reindexOp, { + status: ReindexStatus.completed, + }); }); }); }); diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index 8ca9a2922ce8f..6140111374864 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -151,22 +151,11 @@ export const reindexServiceFactory = ( }); } - // Resume consumers if we ever got past this point. - if (reindexOp.attributes.lastCompletedStep >= ReindexStep.indexGroupServicesStopped) { - await resumeIndexGroupServices(reindexOp); - } - return reindexOp; }; // ------ Functions used to process the state machine - const stopIndexGroupServices = async (reindexOp: ReindexSavedObject) => { - return actions.updateReindexOp(reindexOp, { - lastCompletedStep: ReindexStep.indexGroupServicesStopped, - }); - }; - /** * Sets the original index as readonly so new data can be indexed until the reindex * is completed. @@ -362,17 +351,6 @@ export const reindexServiceFactory = ( }); }; - const resumeIndexGroupServices = async (reindexOp: ReindexSavedObject) => { - // Only change the status if we're still in-progress (this function is also called when the reindex fails or is cancelled) - if (reindexOp.attributes.status === ReindexStatus.inProgress) { - return actions.updateReindexOp(reindexOp, { - lastCompletedStep: ReindexStep.indexGroupServicesStarted, - }); - } else { - return reindexOp; - } - }; - // ------ The service itself return { @@ -500,9 +478,6 @@ export const reindexServiceFactory = ( try { switch (lockedReindexOp.attributes.lastCompletedStep) { case ReindexStep.created: - lockedReindexOp = await stopIndexGroupServices(lockedReindexOp); - break; - case ReindexStep.indexGroupServicesStopped: lockedReindexOp = await setReadonly(lockedReindexOp); break; case ReindexStep.readonly: @@ -518,12 +493,10 @@ export const reindexServiceFactory = ( lockedReindexOp = await switchAlias(lockedReindexOp); break; case ReindexStep.aliasCreated: - lockedReindexOp = await resumeIndexGroupServices(lockedReindexOp); - break; - case ReindexStep.indexGroupServicesStarted: lockedReindexOp = await actions.updateReindexOp(lockedReindexOp, { status: ReindexStatus.completed, }); + break; default: break; } From 9dcec2e0890e330fb58ab074e50509f493448290 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Thu, 30 Sep 2021 17:21:02 +0200 Subject: [PATCH 08/11] Remove no longer used indexGroup param --- .../es_deprecations/reindex_deprecation_flyout.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/upgrade_assistant/__jest__/client_integration/es_deprecations/reindex_deprecation_flyout.test.ts b/x-pack/plugins/upgrade_assistant/__jest__/client_integration/es_deprecations/reindex_deprecation_flyout.test.ts index dce2fa6e18ac9..ca9253c1140b0 100644 --- a/x-pack/plugins/upgrade_assistant/__jest__/client_integration/es_deprecations/reindex_deprecation_flyout.test.ts +++ b/x-pack/plugins/upgrade_assistant/__jest__/client_integration/es_deprecations/reindex_deprecation_flyout.test.ts @@ -52,7 +52,6 @@ describe('Reindex deprecation flyout', () => { httpRequestsMockHelpers.setReindexStatusResponse({ reindexOp: null, warnings: [], - indexGroup: null, hasRequiredPrivileges: true, }); From ba3ec72de1c3fc36a02f2a564b17354785f486f0 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Thu, 30 Sep 2021 18:42:39 +0200 Subject: [PATCH 09/11] Fix TS error --- .../server/lib/reindexing/reindex_service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 4316ef5e69d63..e49fe4139a1a5 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -762,7 +762,7 @@ describe('reindexService', () => { } as ReindexSavedObject; it('sets reindex status as complete', async () => { - const updatedOp = await service.processNextStep(reindexOp); + await service.processNextStep(reindexOp); expect(actions.updateReindexOp).toHaveBeenCalledWith(reindexOp, { status: ReindexStatus.completed, }); From c25a569c66c3bd0d7c536e25ba5c1c8bcde87f6c Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Thu, 30 Sep 2021 20:38:15 +0200 Subject: [PATCH 10/11] Fix snapshot --- .../deprecation_types/reindex/flyout/progress.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx index 2809d664b183e..7d9a885649d0c 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx @@ -33,7 +33,7 @@ describe('ReindexProgress', () => { steps={ Array [ Object { - "status": "incomplete", + "status": "inProgress", "title": { steps={ Array [ Object { - "status": "incomplete", + "status": "inProgress", "title": Date: Fri, 1 Oct 2021 10:43:46 +0200 Subject: [PATCH 11/11] Remove no longer needed test --- .../reindex/flyout/progress.test.tsx | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx index 7d9a885649d0c..c58230b3563f3 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/es_deprecations/deprecation_types/reindex/flyout/progress.test.tsx @@ -110,61 +110,4 @@ describe('ReindexProgress', () => { expect(reindexStep.children.type.name).toEqual('ReindexProgressBar'); expect(reindexStep.children.props.reindexState.reindexTaskPercComplete).toEqual(0.25); }); - - it('adds steps for index groups', () => { - const wrapper = shallow( - - ); - - expect(wrapper).toMatchInlineSnapshot(` -, - }, - Object { - "status": "incomplete", - "title": , - }, - Object { - "status": "incomplete", - "title": , - }, - Object { - "status": "incomplete", - "title": , - }, - ] - } -/> -`); - }); });