From 48dc40c429fb732a71e52ccd23ea62132e245656 Mon Sep 17 00:00:00 2001 From: doubleface Date: Thu, 23 Nov 2023 16:21:37 +0100 Subject: [PATCH] feat: Handle MAIN_FOLDER_REMOVED error on saveFiles When this even occurs, it means that the main destination folder of the konnector has been removed during konnector execution. In this case, we : - Regenerate the files index - create the destination folder according to the manifest - update the trigger with the new message.folder_to_save attribute - retry saveFiles see https://github.com/konnectors/libs/pull/973 --- package.json | 2 +- src/libs/Launcher.js | 70 +++++++++++++++++++-- src/libs/Launcher.spec.js | 128 ++++++++++++++++++++++++++++++++++++++ yarn.lock | 8 +-- 4 files changed, 198 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 27a59493b..512d9ea87 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "@sentry/react-native": "3.4.3", "base-64": "^1.0.0", "cozy-client": "^44.0.0", - "cozy-clisk": "^0.26.0", + "cozy-clisk": "^0.27.0", "cozy-device-helper": "^2.7.0", "cozy-flags": "^2.11.0", "cozy-intent": "^2.18.0", diff --git a/src/libs/Launcher.js b/src/libs/Launcher.js index 7cccda4c5..7979c07d5 100644 --- a/src/libs/Launcher.js +++ b/src/libs/Launcher.js @@ -402,8 +402,8 @@ export default class Launcher { * * @return {Promise>} - index of existing files */ - async getExistingFilesIndex() { - if (this.existingFilesIndex) { + async getExistingFilesIndex(reset = false) { + if (!reset && this.existingFilesIndex) { return this.existingFilesIndex } @@ -472,7 +472,7 @@ export default class Launcher { const existingFilesIndex = await this.getExistingFilesIndex() - const result = await saveFiles(client, entries, folderPath, { + const saveFilesOptions = { ...options, manifest: konnector, // @ts-ignore @@ -485,10 +485,70 @@ export default class Launcher { dataUri: await this.worker.call('downloadFileInWorker', entry) }), existingFilesIndex + } + + try { + const result = await saveFiles( + client, + entries, + folderPath, + saveFilesOptions + ) + log.debug(result, 'saveFiles result') + return result + } catch (err) { + if ( + (err instanceof Error && err.message !== 'MAIN_FOLDER_REMOVED') || + !(err instanceof Error) // instanceof Error is here to fix typing error + ) { + throw err + } + // main destination folder has been removed during the execution of the konnector. Trying one time to reset all and relaunch saveFiles + return await this.retrySaveFiles(entries, saveFilesOptions) + } + } + + /** + * Rerun the saveFiles function after have reindexed files and created the destination folder + * @param {Array} entries - list of file entries to save + * @param {object} options - options object + * @returns {Promise>} list of saved files + */ + async retrySaveFiles(entries, options) { + const { + launcherClient: client, + account, + trigger, + konnector + } = this.getStartContext() || {} + + const folderPath = await this.getFolderPath(trigger.message?.folder_to_save) + this.log({ + level: 'warning', + namespace: 'Launcher', + label: 'saveFiles', + message: + 'Destination folder removed during konnector execution, trying again' + }) + const folder = await models.konnectorFolder.ensureKonnectorFolder(client, { + konnector: { ...konnector, _id: konnector.id }, // _id attribute is missing in konnector object, which causes the reference to the konnector in the destination folder to be null + account, + lang: client.getInstanceOptions().locale }) - log.debug(result, 'saveFiles result') + trigger.message.folder_to_save = folder._id + const { data: triggerResult } = await client.save(trigger) - return result + this.setStartContext({ + ...this.getStartContext(), + trigger: triggerResult + }) + + const updatedFilesIndex = await this.getExistingFilesIndex(true) // update files index since the destination folder was removed + + return await saveFiles(client, entries, folderPath, { + ...options, + existingFilesIndex: updatedFilesIndex + }) } /** diff --git a/src/libs/Launcher.spec.js b/src/libs/Launcher.spec.js index c7db90157..fbf3962cb 100644 --- a/src/libs/Launcher.spec.js +++ b/src/libs/Launcher.spec.js @@ -290,5 +290,133 @@ describe('Launcher', () => { sourceAccountIdentifier: 'testsourceaccountidentifier' }) }) + it('should retry on MAIN_FOLDER_REMOVED error', async () => { + const launcher = new Launcher() + launcher.setUserData({ + sourceAccountIdentifier: 'testsourceaccountidentifier' + }) + + const konnector = { + slug: 'testkonnectorslug', + id: 'testkonnectorid', + name: 'Test Konnector' + } + const trigger = { + _type: 'io.cozy.triggers', + message: { + folder_to_save: 'testfolderid', + account: 'testaccountid' + } + } + const job = { + message: { account: 'testaccountid', folder_to_save: 'testfolderid' } + } + const client = { + collection: doctype => { + if (doctype === 'io.cozy.permissions') { + return { add: jest.fn() } + } + return client + }, + addReferencesTo: jest.fn(), + statByPath: jest + .fn() + .mockImplementation(path => ({ data: { _id: path } })), + findReferencedBy: jest + .fn() + .mockResolvedValue({ included: existingMagicFolder }), + getInstanceOptions: jest.fn().mockReturnValue(() => ({ locale: 'fr' })), + queryAll: jest.fn().mockResolvedValue([ + { + _id: 'tokeep', + metadata: { + fileIdAttributes: 'fileidattribute' + } + }, + { + _id: 'toignore' + } + ]), + query: jest.fn().mockResolvedValue({ data: { path: 'folderPath' } }), + save: jest.fn().mockImplementation(() => ({ _id: 'triggerid' })) + } + launcher.setStartContext({ + konnector, + account: { _id: 'testaccountid' }, + client, + launcherClient: client, + trigger, + job + }) + saveFiles.mockRejectedValueOnce(new Error('MAIN_FOLDER_REMOVED')) + + await launcher.saveFiles([{}], {}) + + expect(saveFiles).toHaveBeenNthCalledWith(1, client, [{}], 'folderPath', { + downloadAndFormatFile: expect.any(Function), + manifest: expect.any(Object), + existingFilesIndex: new Map([ + [ + 'fileidattribute', + { + _id: 'tokeep', + metadata: { fileIdAttributes: 'fileidattribute' } + } + ] + ]), + sourceAccount: 'testaccountid', + sourceAccountIdentifier: 'testsourceaccountidentifier' + }) + expect(saveFiles).toHaveBeenNthCalledWith(2, client, [{}], 'folderPath', { + downloadAndFormatFile: expect.any(Function), + manifest: expect.any(Object), + existingFilesIndex: new Map([ + [ + 'fileidattribute', + { + _id: 'tokeep', + metadata: { fileIdAttributes: 'fileidattribute' } + } + ] + ]), + sourceAccount: 'testaccountid', + sourceAccountIdentifier: 'testsourceaccountidentifier' + }) + expect(client.save).toHaveBeenCalledTimes(1) + expect(client.save).toHaveBeenNthCalledWith(1, { + _type: 'io.cozy.triggers', + message: { + account: 'testaccountid', + folder_to_save: '/Administratif/Test Konnector/testaccountid' + } + }) + expect(client.queryAll).toHaveBeenCalledTimes(2) + expect(client.queryAll).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + doctype: 'io.cozy.files', + selector: { + cozyMetadata: { + createdByApp: 'testkonnectorslug', + sourceAccountIdentifier: 'testsourceaccountidentifier' + }, + trashed: false + } + }) + ) + expect(client.queryAll).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + doctype: 'io.cozy.files', + selector: { + cozyMetadata: { + createdByApp: 'testkonnectorslug', + sourceAccountIdentifier: 'testsourceaccountidentifier' + }, + trashed: false + } + }) + ) + }) }) }) diff --git a/yarn.lock b/yarn.lock index f845828cd..48767f7a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6400,10 +6400,10 @@ cozy-client@^44.0.0: sift "^6.0.0" url-search-params-polyfill "^8.0.0" -cozy-clisk@^0.26.0: - version "0.26.0" - resolved "https://registry.yarnpkg.com/cozy-clisk/-/cozy-clisk-0.26.0.tgz#b63734b5678a9ed8c992554da6031c1e6ac1d805" - integrity sha512-E1V3vcrQyrcq6Uxn2B8YqVdufIaI3MTJ//TAwnIhlBYgebo1MRIRoaRYpK8kNXB7EsmQXY5DlgLMOA+1A/YZTQ== +cozy-clisk@^0.27.0: + version "0.27.0" + resolved "https://registry.yarnpkg.com/cozy-clisk/-/cozy-clisk-0.27.0.tgz#49853c67a4e89e950d08fef0f9239f379c4edc29" + integrity sha512-uJR61dQSnXJgGHDhO+6OKo8uj7haGJNnK7gCEhWwj9KlePCajsRm0UuEwh3tl7Kk9bOeteT/LG5wGkw3I2EZyQ== dependencies: "@cozy/minilog" "^1.0.0" bluebird-retry "^0.11.0"