Skip to content

Commit

Permalink
feat: Enable storage of remote file cache in a separate object storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@caipira113
caipira113 committed Jul 17, 2023
1 parent 3c95399 commit 0d93433
Show file tree
Hide file tree
Showing 11 changed files with 494 additions and 102 deletions.
36 changes: 36 additions & 0 deletions packages/backend/migration/1689580926821-ObjectStorageRemoteSetting.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
export class ObjectStorageRemoteSetting1689580926821 {
name = 'ObjectStorageRemoteSetting1689580926821'

async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "useObjectStorageRemote" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteBucket" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemotePrefix" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteBaseUrl" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteEndpoint" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteRegion" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteAccessKey" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteSecretKey" character varying(1024)`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemotePort" integer`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteUseSSL" boolean NOT NULL DEFAULT true`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteUseProxy" boolean NOT NULL DEFAULT true`, undefined);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteSetPublicRead" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRemoteS3ForcePathStyle" boolean NOT NULL DEFAULT true`);
}

async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteS3ForcePathStyle"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteSetPublicRead"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteUseProxy"`, undefined);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteUseSSL"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemotePort"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteSecretKey"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteAccessKey"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteRegion"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteEndpoint"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteBaseUrl"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemotePrefix"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRemoteBucket"`);
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useObjectStorageRemote"`);
}

}
58 changes: 37 additions & 21 deletions packages/backend/src/core/DriveService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,10 @@ export class DriveService {
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param isRemote If true, file is remote file
*/
@bindThis
private async save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<DriveFile> {
private async save(file: DriveFile, path: string, name: string, type: string, hash: string, size: number, isRemote: boolean): Promise<DriveFile> {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);

Expand All @@ -158,11 +159,19 @@ export class DriveService {
ext = '';
}

const baseUrl = meta.objectStorageBaseUrl
?? `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`;
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const objectStorageBaseUrl = useObjectStorageRemote ? meta.objectStorageRemoteBaseUrl : meta.objectStorageBaseUrl;
const objectStorageUseSSL = useObjectStorageRemote ? meta.objectStorageRemoteUseSSL : meta.objectStorageUseSSL;
const objectStorageEndpoint = useObjectStorageRemote ? meta.objectStorageRemoteEndpoint : meta.objectStorageEndpoint;
const objectStoragePort = useObjectStorageRemote ? meta.objectStorageRemotePort : meta.objectStoragePort;
const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket;
const objectStoragePrefix = useObjectStorageRemote ? meta.objectStorageRemotePrefix : meta.objectStoragePrefix;

const baseUrl = objectStorageBaseUrl
?? `${ objectStorageUseSSL ? 'https' : 'http' }://${ objectStorageEndpoint }${ objectStoragePort ? `:${objectStoragePort}` : '' }/${ objectStorageBucket }`;

// for original
const key = `${meta.objectStoragePrefix}/${randomUUID()}${ext}`;
const key = `${objectStoragePrefix}/${randomUUID()}${ext}`;
const url = `${ baseUrl }/${ key }`;

// for alts
Expand All @@ -175,23 +184,23 @@ export class DriveService {
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
this.upload(key, fs.createReadStream(path), type, isRemote, null, name),
];

if (alts.webpublic) {
webpublicKey = `${meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicKey = `${objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;

this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, isRemote, alts.webpublic.ext, name));
}

if (alts.thumbnail) {
thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailKey = `${objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;

this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, isRemote, alts.thumbnail.ext, `${name}.thumbnail`));
}

await Promise.all(uploads);
Expand Down Expand Up @@ -360,14 +369,18 @@ export class DriveService {
* Upload to ObjectStorage
*/
@bindThis
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, ext?: string | null, filename?: string) {
private async upload(key: string, stream: fs.ReadStream | Buffer, type: string, isRemote: boolean, ext?: string | null, filename?: string) {
if (type === 'image/apng') type = 'image/png';
if (!FILE_TYPE_BROWSERSAFE.includes(type)) type = 'application/octet-stream';

const meta = await this.metaService.fetch();

const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket;
const objectStorageSetPublicRead = useObjectStorageRemote ? meta.objectStorageRemoteSetPublicRead : meta.objectStorageSetPublicRead;

const params = {
Bucket: meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
Body: stream,
ContentType: type,
Expand All @@ -380,9 +393,9 @@ export class DriveService {
// 許可されているファイル形式でしか拡張子をつけない
ext ? correctFilename(filename, ext) : filename,
);
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
if (objectStorageSetPublicRead) params.ACL = 'public-read';

await this.s3Service.upload(meta, params)
await this.s3Service.upload(meta, params, isRemote)
.then(
result => {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
Expand Down Expand Up @@ -618,7 +631,8 @@ export class DriveService {
}
}
} else {
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size));
const isRemote = user ? this.userEntityService.isRemoteUser(user) : false;
file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size, isRemote));
}

this.registerLogger.succ(`drive file has been created ${file.id}`);
Expand Down Expand Up @@ -672,7 +686,7 @@ export class DriveService {
}

@bindThis
public async deleteFileSync(file: DriveFile, isExpired = false) {
public async deleteFileSync(file: DriveFile, isExpired = false, isRemote: boolean) {
if (file.storedInternal) {
this.internalStorageService.del(file.accessKey!);

Expand All @@ -686,14 +700,14 @@ export class DriveService {
} else if (!file.isLink) {
const promises = [];

promises.push(this.deleteObjectStorageFile(file.accessKey!));
promises.push(this.deleteObjectStorageFile(file.accessKey!, isRemote));

if (file.thumbnailUrl) {
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!));
promises.push(this.deleteObjectStorageFile(file.thumbnailAccessKey!, isRemote));
}

if (file.webpublicUrl) {
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!));
promises.push(this.deleteObjectStorageFile(file.webpublicAccessKey!, isRemote));
}

await Promise.all(promises);
Expand Down Expand Up @@ -733,15 +747,17 @@ export class DriveService {
}

@bindThis
public async deleteObjectStorageFile(key: string) {
public async deleteObjectStorageFile(key: string, isRemote: boolean) {
const meta = await this.metaService.fetch();
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;
const objectStorageBucket = useObjectStorageRemote ? meta.objectStorageRemoteBucket : meta.objectStorageBucket;
try {
const param = {
Bucket: meta.objectStorageBucket,
Bucket: objectStorageBucket,
Key: key,
} as DeleteObjectCommandInput;

await this.s3Service.delete(meta, param);
await this.s3Service.delete(meta, param, isRemote);
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
Expand Down
44 changes: 27 additions & 17 deletions packages/backend/src/core/S3Service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,35 +23,45 @@ export class S3Service {
}

@bindThis
public getS3Client(meta: Meta): S3Client {
const u = meta.objectStorageEndpoint
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
public getS3Client(meta: Meta, isRemote: boolean): S3Client {
const useObjectStorageRemote = isRemote && meta.useObjectStorageRemote;

const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
const objectStorageEndpoint = useObjectStorageRemote ? meta.objectStorageRemoteEndpoint : meta.objectStorageEndpoint;
const objectStorageUseSSL = useObjectStorageRemote ? meta.objectStorageRemoteUseSSL : meta.objectStorageUseSSL;
const objectStorageUseProxy = useObjectStorageRemote ? meta.objectStorageRemoteUseProxy : meta.objectStorageUseProxy;
const objectStorageAccessKey = useObjectStorageRemote ? meta.objectStorageRemoteAccessKey : meta.objectStorageAccessKey;
const objectStorageSecretKey = useObjectStorageRemote ? meta.objectStorageRemoteSecretKey : meta.objectStorageSecretKey;
const objectStorageRegion = useObjectStorageRemote ? meta.objectStorageRemoteRegion : meta.objectStorageRegion;
const objectStorageS3ForcePathStyle = useObjectStorageRemote ? meta.objectStorageRemoteS3ForcePathStyle : meta.objectStorageS3ForcePathStyle;

const u = objectStorageEndpoint
? `${objectStorageUseSSL ? 'https' : 'http'}://${objectStorageEndpoint}`
: `${objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent

const agent = this.httpRequestService.getAgentByUrl(new URL(u), !objectStorageUseProxy);
const handlerOption: NodeHttpHandlerOptions = {};
if (meta.objectStorageUseSSL) {
if (objectStorageUseSSL) {
handlerOption.httpsAgent = agent as https.Agent;
} else {
handlerOption.httpAgent = agent as http.Agent;
}

return new S3Client({
endpoint: meta.objectStorageEndpoint ? u : undefined,
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
accessKeyId: meta.objectStorageAccessKey,
secretAccessKey: meta.objectStorageSecretKey,
endpoint: objectStorageEndpoint ? u : undefined,
credentials: (objectStorageAccessKey !== null && objectStorageSecretKey !== null) ? {
accessKeyId: objectStorageAccessKey,
secretAccessKey: objectStorageSecretKey,
} : undefined,
region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, // 空文字列もundefinedにするため ?? は使わない
tls: meta.objectStorageUseSSL,
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
region: objectStorageRegion ? objectStorageRegion : undefined, // empty string is converted to undefined
tls: objectStorageUseSSL,
forcePathStyle: objectStorageEndpoint ? objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
requestHandler: new NodeHttpHandler(handlerOption),
});
}

@bindThis
public async upload(meta: Meta, input: PutObjectCommandInput) {
const client = this.getS3Client(meta);
public async upload(meta: Meta, input: PutObjectCommandInput, isRemote: boolean) {
const client = this.getS3Client(meta, isRemote);
return new Upload({
client,
params: input,
Expand All @@ -62,8 +72,8 @@ export class S3Service {
}

@bindThis
public delete(meta: Meta, input: DeleteObjectCommandInput) {
const client = this.getS3Client(meta);
public delete(meta: Meta, input: DeleteObjectCommandInput, isRemote: boolean) {
const client = this.getS3Client(meta, isRemote);
return client.send(new DeleteObjectCommand(input));
}
}
72 changes: 72 additions & 0 deletions packages/backend/src/models/entities/Meta.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,78 @@ export class Meta {
})
public objectStorageS3ForcePathStyle: boolean;

@Column('boolean', {
default: false,
})
public useObjectStorageRemote: boolean;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteBucket: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemotePrefix: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteBaseUrl: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteEndpoint: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteRegion: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteAccessKey: string | null;

@Column('varchar', {
length: 1024,
nullable: true,
})
public objectStorageRemoteSecretKey: string | null;

@Column('integer', {
nullable: true,
})
public objectStorageRemotePort: number | null;

@Column('boolean', {
default: true,
})
public objectStorageRemoteUseSSL: boolean;

@Column('boolean', {
default: true,
})
public objectStorageRemoteUseProxy: boolean;

@Column('boolean', {
default: false,
})
public objectStorageRemoteSetPublicRead: boolean;

@Column('boolean', {
default: true,
})
public objectStorageRemoteS3ForcePathStyle: boolean;

@Column('boolean', {
default: false,
})
Expand Down
2 changes: 1 addition & 1 deletion packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ export class CleanRemoteFilesProcessorService {

cursor = files.at(-1)?.id ?? null;

await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true)));
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true, true)));

deletedCount += 8;

Expand Down
5 changes: 4 additions & 1 deletion packages/backend/src/queue/processors/DeleteAccountProcessorService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Note } from '@/models/entities/Note.js';
import { EmailService } from '@/core/EmailService.js';
import { bindThis } from '@/decorators.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
Expand All @@ -34,6 +35,7 @@ export class DeleteAccountProcessorService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,

private userEntityService: UserEntityService,
private driveService: DriveService,
private emailService: EmailService,
private queueLoggerService: QueueLoggerService,
Expand All @@ -47,6 +49,7 @@ export class DeleteAccountProcessorService {
this.logger.info(`Deleting account of ${job.data.user.id} ...`);

const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
const isRemote = user ? this.userEntityService.isRemoteUser(user) : false;
if (user == null) {
return;
}
Expand Down Expand Up @@ -104,7 +107,7 @@ export class DeleteAccountProcessorService {
cursor = files.at(-1)?.id ?? null;

for (const file of files) {
await this.driveService.deleteFileSync(file);
await this.driveService.deleteFileSync(file, false, isRemote);
}
}

Expand Down
Loading

0 comments on commit 0d93433

Please sign in to comment.