Skip to content

Commit

Permalink
optimize the upload procedure
Browse files Browse the repository at this point in the history
- closes #2538
  • Loading branch information
foxriver76 committed Jan 25, 2024
1 parent 7758afb commit 927d2cd
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 101 deletions.
21 changes: 4 additions & 17 deletions packages/cli/src/lib/setup/setupUpload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,6 @@ export class Upload {
isAdmin: boolean,
files: string[],
id: string,
rev: any,
logger: Logger | typeof console
): Promise<string> {
const uploadID = `system.adapter.${adapter}.upload`;
Expand Down Expand Up @@ -442,18 +441,8 @@ export class Upload {
}

try {
await new Promise<void>((resolve, reject) => {
const stream = fs.createReadStream(file);
stream.on('error', e => reject(e));
stream.pipe(
this.objects.insert(id, attName, null, mimeType || {}, { rev }, err => {
if (err) {
console.log(err);
}
resolve();
})
);
});
const content = await fs.readFile(file);
await this.objects.writeFileAsync(id, attName, content, { mimeType: mimeType || undefined });
} catch (e) {
console.error(`Error: Cannot upload ${file}: ${e.message}`);
}
Expand Down Expand Up @@ -627,11 +616,9 @@ export class Upload {
if (!isAdmin) {
await this.checkRestartOther(adapter);
await new Promise<void>(resolve => setTimeout(() => resolve(), 25));
// @ts-expect-error TODO rev is not required and should not exist on an object?
await this.upload(adapter, isAdmin, files, id, result?.rev, logger);
await this.upload(adapter, isAdmin, files, id, logger);
} else {
// @ts-expect-error TODO rev is not required and should not exist on an object?
await this.upload(adapter, isAdmin, files, id, result?.rev, logger);
await this.upload(adapter, isAdmin, files, id, logger);
}
}
return adapter;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,7 @@ import path from 'path';
import crypto from 'crypto';
import { isDeepStrictEqual } from 'util';
import deepClone from 'deep-clone';
import type {
ACLObject,
FileObject,
CheckFileRightsCallback,
GetUserGroupPromiseReturn,
WMStrm
} from './objectsUtils.js';
import type { ACLObject, FileObject, CheckFileRightsCallback, GetUserGroupPromiseReturn } from './objectsUtils.js';
import * as utils from './objectsUtils.js';
import semver from 'semver';
import * as CONSTS from './constants';
Expand Down Expand Up @@ -1028,17 +1022,6 @@ export class ObjectsInRedisClient {
});
}

insert(
id: string,
attName: string,
ignore: any,
options: string | Record<string, any>,
obj: any,
callback: (err?: Error | null) => void
): WMStrm {
return utils.insert(this, id, attName, ignore, options, obj, callback);
}

private async _writeFile(
id: string,
name: string,
Expand Down
67 changes: 1 addition & 66 deletions packages/db-objects-redis/src/lib/objects/objectsUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
*
*/

import { Writable, type WritableOptions } from 'stream';
import path from 'path';
import path from 'node:path';
import deepClone from 'deep-clone';
import { tools } from '@iobroker/db-base';
import * as CONSTS from './constants';
Expand All @@ -20,7 +19,6 @@ export const REG_CHECK_ID = CONSTS.REG_CHECK_ID;

const USER_STARTS_WITH = CONSTS.USER_STARTS_WITH;
const GROUP_STARTS_WITH = CONSTS.GROUP_STARTS_WITH;
const memStore: Record<string, Buffer> = {};

export interface FileMimeInformation {
/** the mime type, of the file */
Expand Down Expand Up @@ -126,69 +124,6 @@ export function getMimeType(ext: string, isTextData: boolean): FileMimeInformati
}
}

/**
* Writable memory stream
*/
export class WMStrm extends Writable {
private readonly key: string;

constructor(key: string, options: WritableOptions) {
super(options); // init super
this.key = key; // save key
memStore[key] = Buffer.alloc(0); // empty
}

_write(chunk: string | Buffer, enc: BufferEncoding, cb: () => void): void {
if (chunk) {
// our memory store stores things in buffers
const buffer = Buffer.isBuffer(chunk)
? chunk // already is Buffer use it
: Buffer.from(chunk, enc); // string, convert

// concatenate to the buffer already there
if (!memStore[this.key]) {
memStore[this.key] = Buffer.alloc(0);
console.log(`memstore for ${this.key} is null`);
}
memStore[this.key] = Buffer.concat([memStore[this.key], buffer]);
}
if (!cb) {
throw new Error('Callback is empty');
}
cb();
}
}

export function insert(
objects: any,
id: string,
attName: string,
_ignore: any,
options: Record<string, any> | string,
_obj: any,
callback: (err?: Error | null) => void
): WMStrm {
if (typeof options === 'string') {
options = { mimeType: options };
}

// return pipe for write into redis
const strm = new WMStrm(`${id}/${attName}`, {});
strm.on('finish', () => {
let error: null | string = null;
if (!memStore[`${id}/${attName}`]) {
error = `File ${id} / ${attName} is empty`;
}
objects.writeFile(id, attName, memStore[`${id}/${attName}`] || '', options, () => {
if (memStore[`${id}/${attName}`] !== undefined) {
delete memStore[`${id}/${attName}`];
}
return tools.maybeCallbackWithError(callback, error);
});
});
return strm;
}

export function checkFile(
fileOptions: Record<string, any>,
options: Record<string, any>,
Expand Down

0 comments on commit 927d2cd

Please sign in to comment.