Skip to content

Commit

Permalink
All, Server: Add support for faster built-in sync locks (laurent22#5662)
Browse files Browse the repository at this point in the history
  • Loading branch information
@laurent22
laurent22 authored Nov 3, 2021
1 parent 630a400 commit 47a31c4
Show file tree
Hide file tree
Showing 19 changed files with 675 additions and 181 deletions.
8 changes: 6 additions & 2 deletions packages/lib/JoplinServerApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ export default class JoplinServerApi {
}

if (sessionId) headers['X-API-AUTH'] = sessionId;
headers['X-API-MIN-VERSION'] = '2.1.4';
headers['X-API-MIN-VERSION'] = '2.6.0'; // Need server 2.6 for new lock support

const fetchOptions: any = {};
fetchOptions.headers = headers;
Expand Down Expand Up @@ -253,8 +253,12 @@ export default class JoplinServerApi {
const output = await loadResponseJson();
return output;
} catch (error) {
if (error.code !== 404) {
// Don't print error info for file not found (handled by the
// driver), or lock-acquisition errors because it's handled by
// LockHandler.
if (![404, 'hasExclusiveLock', 'hasSyncLock'].includes(error.code)) {
logger.warn(this.requestToCurl_(url, fetchOptions));
logger.warn('Code:', error.code);
logger.warn(error);
}

Expand Down
42 changes: 32 additions & 10 deletions packages/lib/Synchronizer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import Logger from './Logger';
import LockHandler, { LockType } from './services/synchronizer/LockHandler';
import Setting from './models/Setting';
import LockHandler, { hasActiveLock, LockClientType, LockType } from './services/synchronizer/LockHandler';
import Setting, { AppType } from './models/Setting';
import shim from './shim';
import MigrationHandler from './services/synchronizer/MigrationHandler';
import eventManager from './eventManager';
Expand Down Expand Up @@ -66,6 +66,7 @@ export default class Synchronizer {
private resourceService_: ResourceService = null;
private syncTargetIsLocked_: boolean = false;
private shareService_: ShareService = null;
private lockClientType_: LockClientType = null;

// Debug flags are used to test certain hard-to-test conditions
// such as cancelling in the middle of a loop.
Expand Down Expand Up @@ -120,9 +121,21 @@ export default class Synchronizer {
return this.lockHandler_;
}

private lockClientType(): LockClientType {
if (this.lockClientType_) return this.lockClientType_;

if (this.appType_ === AppType.Desktop) this.lockClientType_ = LockClientType.Desktop;
if (this.appType_ === AppType.Mobile) this.lockClientType_ = LockClientType.Mobile;
if (this.appType_ === AppType.Cli) this.lockClientType_ = LockClientType.Cli;

if (!this.lockClientType_) throw new Error(`Invalid client type: ${this.appType_}`);

return this.lockClientType_;
}

migrationHandler() {
if (this.migrationHandler_) return this.migrationHandler_;
this.migrationHandler_ = new MigrationHandler(this.api(), this.db(), this.lockHandler(), this.appType_, this.clientId_);
this.migrationHandler_ = new MigrationHandler(this.api(), this.db(), this.lockHandler(), this.lockClientType(), this.clientId_);
return this.migrationHandler_;
}

Expand Down Expand Up @@ -164,6 +177,12 @@ export default class Synchronizer {
return !!report && !!report.errors && !!report.errors.length;
}

private static completionTime(report: any): string {
const duration = report.completedTime - report.startTime;
if (duration > 1000) return `${Math.round(duration / 1000)}s`;
return `${duration}ms`;
}

static reportToLines(report: any) {
const lines = [];
if (report.createLocal) lines.push(_('Created local items: %d.', report.createLocal));
Expand All @@ -174,7 +193,7 @@ export default class Synchronizer {
if (report.deleteRemote) lines.push(_('Deleted remote items: %d.', report.deleteRemote));
if (report.fetchingTotal && report.fetchingProcessed) lines.push(_('Fetched items: %d/%d.', report.fetchingProcessed, report.fetchingTotal));
if (report.cancelling && !report.completedTime) lines.push(_('Cancelling...'));
if (report.completedTime) lines.push(_('Completed: %s (%s)', time.formatMsToLocal(report.completedTime), `${Math.round((report.completedTime - report.startTime) / 1000)}s`));
if (report.completedTime) lines.push(_('Completed: %s (%s)', time.formatMsToLocal(report.completedTime), this.completionTime(report)));
if (this.reportHasErrors(report)) lines.push(_('Last error: %s', report.errors[report.errors.length - 1].toString().substr(0, 500)));

return lines;
Expand Down Expand Up @@ -298,10 +317,13 @@ export default class Synchronizer {
}

async lockErrorStatus_() {
const hasActiveExclusiveLock = await this.lockHandler().hasActiveLock(LockType.Exclusive);
const locks = await this.lockHandler().locks();
const currentDate = await this.lockHandler().currentDate();

const hasActiveExclusiveLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Exclusive);
if (hasActiveExclusiveLock) return 'hasExclusiveLock';

const hasActiveSyncLock = await this.lockHandler().hasActiveLock(LockType.Sync, this.appType_, this.clientId_);
const hasActiveSyncLock = await hasActiveLock(locks, currentDate, this.lockHandler().lockTtl, LockType.Sync, this.lockClientType(), this.clientId_);
if (!hasActiveSyncLock) return 'syncLockGone';

return '';
Expand Down Expand Up @@ -446,10 +468,10 @@ export default class Synchronizer {
const previousE2EE = localInfo.e2ee;
logger.info('Sync target info differs between local and remote - merging infos: ', newInfo.toObject());

await this.lockHandler().acquireLock(LockType.Exclusive, this.appType_, this.clientId_, { clearExistingSyncLocksFromTheSameClient: true });
await this.lockHandler().acquireLock(LockType.Exclusive, this.lockClientType(), this.clientId_, { clearExistingSyncLocksFromTheSameClient: true });
await uploadSyncInfo(this.api(), newInfo);
await saveLocalSyncInfo(newInfo);
await this.lockHandler().releaseLock(LockType.Exclusive, this.appType_, this.clientId_);
await this.lockHandler().releaseLock(LockType.Exclusive, this.lockClientType(), this.clientId_);

// console.info('NEW', newInfo);

Expand All @@ -473,7 +495,7 @@ export default class Synchronizer {
throw error;
}

syncLock = await this.lockHandler().acquireLock(LockType.Sync, this.appType_, this.clientId_);
syncLock = await this.lockHandler().acquireLock(LockType.Sync, this.lockClientType(), this.clientId_);

this.lockHandler().startAutoLockRefresh(syncLock, (error: any) => {
logger.warn('Could not refresh lock - cancelling sync. Error was:', error);
Expand Down Expand Up @@ -1084,7 +1106,7 @@ export default class Synchronizer {

if (syncLock) {
this.lockHandler().stopAutoLockRefresh(syncLock);
await this.lockHandler().releaseLock(LockType.Sync, this.appType_, this.clientId_);
await this.lockHandler().releaseLock(LockType.Sync, this.lockClientType(), this.clientId_);
}

this.syncTargetIsLocked_ = false;
Expand Down
51 changes: 51 additions & 0 deletions packages/lib/file-api-driver-joplinServer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { MultiPutItem } from './file-api';
import JoplinError from './JoplinError';
import JoplinServerApi from './JoplinServerApi';
import { trimSlashes } from './path-utils';
import { Lock, LockClientType, LockType } from './services/synchronizer/LockHandler';

// All input paths should be in the format: "path/to/file". This is converted to
// "root:/path/to/file:" when doing the API call.
Expand Down Expand Up @@ -40,6 +41,10 @@ export default class FileApiDriverJoplinServer {
return true;
}

public get supportsLocks() {
return true;
}

public requestRepeatCount() {
return 3;
}
Expand Down Expand Up @@ -196,13 +201,59 @@ export default class FileApiDriverJoplinServer {
throw new Error('Not supported');
}

// private lockClientTypeToId(clientType:AppType):number {
// if (clientType === AppType.Desktop) return 1;
// if (clientType === AppType.Mobile) return 2;
// if (clientType === AppType.Cli) return 3;
// throw new Error('Invalid client type: ' + clientType);
// }

// private lockTypeToId(lockType:LockType):number {
// if (lockType === LockType.None) return 0; // probably not possible?
// if (lockType === LockType.Sync) return 1;
// if (lockType === LockType.Exclusive) return 2;
// throw new Error('Invalid lock type: ' + lockType);
// }

// private lockClientIdTypeToType(clientType:number):AppType {
// if (clientType === 1) return AppType.Desktop;
// if (clientType === 2) return AppType.Mobile;
// if (clientType === 3) return AppType.Cli;
// throw new Error('Invalid client type: ' + clientType);
// }

// private lockIdToType(lockType:number):LockType {
// if (lockType === 0) return LockType.None; // probably not possible?
// if (lockType === 1) return LockType.Sync;
// if (lockType === 2) return LockType.Exclusive;
// throw new Error('Invalid lock type: ' + lockType);
// }

public async acquireLock(type: LockType, clientType: LockClientType, clientId: string): Promise<Lock> {
return this.api().exec('POST', 'api/locks', null, {
type,
clientType,
clientId: clientId,
});
}

public async releaseLock(type: LockType, clientType: LockClientType, clientId: string) {
await this.api().exec('DELETE', `api/locks/${type}_${clientType}_${clientId}`);
}

public async listLocks() {
return this.api().exec('GET', 'api/locks');
}

public async clearRoot(path: string) {
const response = await this.list(path);

for (const item of response.items) {
await this.delete(item.path);
}

await this.api().exec('POST', 'api/debug', null, { action: 'clearKeyValues' });

if (response.has_more) throw new Error('has_more support not implemented');
}
}
23 changes: 22 additions & 1 deletion packages/lib/file-api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import time from './time';

const { isHidden } = require('./path-utils');
import JoplinError from './JoplinError';
import { Lock, LockClientType, LockType } from './services/synchronizer/LockHandler';
const ArrayUtils = require('./ArrayUtils');
const { sprintf } = require('sprintf-js');
const Mutex = require('async-mutex').Mutex;
Expand Down Expand Up @@ -36,7 +37,7 @@ export interface RemoteItem {

export interface PaginatedList {
items: RemoteItem[];
has_more: boolean;
hasMore: boolean;
context: any;
}

Expand Down Expand Up @@ -130,6 +131,10 @@ class FileApi {
return !!this.driver().supportsAccurateTimestamp;
}

public get supportsLocks(): boolean {
return !!this.driver().supportsLocks;
}

async fetchRemoteDateOffset_() {
const tempFile = `${this.tempDirName()}/timeCheck${Math.round(Math.random() * 1000000)}.txt`;
const startTime = Date.now();
Expand Down Expand Up @@ -349,6 +354,22 @@ class FileApi {
logger.debug(`delta ${this.fullPath(path)}`);
return tryAndRepeat(() => this.driver_.delta(this.fullPath(path), options), this.requestRepeatCount());
}

public async acquireLock(type: LockType, clientType: LockClientType, clientId: string): Promise<Lock> {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.acquireLock(type, clientType, clientId), this.requestRepeatCount());
}

public async releaseLock(type: LockType, clientType: LockClientType, clientId: string) {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.releaseLock(type, clientType, clientId), this.requestRepeatCount());
}

public async listLocks() {
if (!this.supportsLocks) throw new Error('Sync target does not support built-in locks');
return tryAndRepeat(() => this.driver_.listLocks(), this.requestRepeatCount());
}

}

function basicDeltaContextFromOptions_(options: any) {
Expand Down
Loading

0 comments on commit 47a31c4

Please sign in to comment.