From 9775e53d9ad445f345f4b0ea85ffa097010a4495 Mon Sep 17 00:00:00 2001 From: Edison J Padilla Date: Thu, 11 Jan 2024 01:36:11 -0400 Subject: [PATCH] PB-845: Calculate folder size and expose it via shared & Drive --- src/common/base-http.exception.ts | 20 +--- src/modules/file/file.usecase.spec.ts | 6 +- ...calculate-folder-size-timeout.exception.ts | 12 +++ src/modules/folder/folder.controller.spec.ts | 50 ++++++++++ src/modules/folder/folder.controller.ts | 12 ++- src/modules/folder/folder.repository.spec.ts | 95 +++++++++++++++++++ src/modules/folder/folder.repository.ts | 62 +++++++++++- src/modules/folder/folder.usecase.spec.ts | 32 +++++++ src/modules/folder/folder.usecase.ts | 11 ++- src/modules/send/send.usecase.spec.ts | 2 +- src/modules/share/share.usecase.spec.ts | 2 +- src/modules/trash/trash.usecase.spec.ts | 8 +- 12 files changed, 276 insertions(+), 36 deletions(-) create mode 100644 src/modules/folder/exception/calculate-folder-size-timeout.exception.ts create mode 100644 src/modules/folder/folder.controller.spec.ts create mode 100644 src/modules/folder/folder.repository.spec.ts diff --git a/src/common/base-http.exception.ts b/src/common/base-http.exception.ts index 39a858589..285cc42c7 100644 --- a/src/common/base-http.exception.ts +++ b/src/common/base-http.exception.ts @@ -1,25 +1,11 @@ import { HttpStatus } from '@nestjs/common'; export class BaseHttpException extends Error { - private readonly _statusCode: number; - private readonly _code: string; - constructor( - message: string, - statusCode = HttpStatus.INTERNAL_SERVER_ERROR, - code?: string, + public readonly message: string, + public readonly statusCode = HttpStatus.INTERNAL_SERVER_ERROR, + public readonly code?: string, ) { super(message); - this.message = message; - this._statusCode = statusCode; - this._code = code; - } - - get statusCode(): number { - return this._statusCode; - } - - get code(): string { - return this._code; } } diff --git a/src/modules/file/file.usecase.spec.ts b/src/modules/file/file.usecase.spec.ts index 80277f27d..76494bf1c 100644 --- a/src/modules/file/file.usecase.spec.ts +++ b/src/modules/file/file.usecase.spec.ts @@ -15,10 +15,8 @@ import { SequelizeShareRepository, ShareModel, } from '../share/share.repository'; -import { - FolderModel, - SequelizeFolderRepository, -} from '../folder/folder.repository'; +import { SequelizeFolderRepository } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { SequelizeUserRepository, UserModel } from '../user/user.repository'; import { BridgeModule } from '../../externals/bridge/bridge.module'; import { BridgeService } from '../../externals/bridge/bridge.service'; diff --git a/src/modules/folder/exception/calculate-folder-size-timeout.exception.ts b/src/modules/folder/exception/calculate-folder-size-timeout.exception.ts new file mode 100644 index 000000000..87a72ea97 --- /dev/null +++ b/src/modules/folder/exception/calculate-folder-size-timeout.exception.ts @@ -0,0 +1,12 @@ +import { HttpStatus } from '@nestjs/common'; +import { BaseHttpException } from '../../../common/base-http.exception'; + +export class CalculateFolderSizeTimeoutException extends BaseHttpException { + constructor( + message = 'Calculate folder size timeout', + code = 'CALCULATE_FOLDER_SIZE_TIMEOUT', + statusCode = HttpStatus.UNPROCESSABLE_ENTITY, + ) { + super(message, statusCode, code); + } +} diff --git a/src/modules/folder/folder.controller.spec.ts b/src/modules/folder/folder.controller.spec.ts new file mode 100644 index 000000000..7239801f1 --- /dev/null +++ b/src/modules/folder/folder.controller.spec.ts @@ -0,0 +1,50 @@ +import { createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { newFolder } from '../../../test/fixtures'; +import { FileUseCases } from '../file/file.usecase'; +import { FolderController } from './folder.controller'; +import { Folder } from './folder.domain'; +import { FolderUseCases } from './folder.usecase'; +import { CalculateFolderSizeTimeoutException } from './exception/calculate-folder-size-timeout.exception'; + +describe('FolderController', () => { + let folderController: FolderController; + let folderUseCases: FolderUseCases; + let folder: Folder; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [FolderController], + providers: [ + { provide: FolderUseCases, useValue: createMock() }, + { provide: FileUseCases, useValue: createMock() }, + ], + }).compile(); + + folderController = module.get(FolderController); + folderUseCases = module.get(FolderUseCases); + folder = newFolder(); + }); + + describe('get folder size', () => { + it('When get folder size is requested, then return the folder size', async () => { + const expectedSize = 100; + jest + .spyOn(folderUseCases, 'getFolderSizeByUuid') + .mockResolvedValue(expectedSize); + + const result = await folderController.getFolderSize(folder.uuid); + expect(result).toEqual({ size: expectedSize }); + }); + + it('When get folder size times out, then throw an exception', async () => { + jest + .spyOn(folderUseCases, 'getFolderSizeByUuid') + .mockRejectedValue(new CalculateFolderSizeTimeoutException()); + + await expect(folderController.getFolderSize(folder.uuid)).rejects.toThrow( + CalculateFolderSizeTimeoutException, + ); + }); + }); +}); diff --git a/src/modules/folder/folder.controller.ts b/src/modules/folder/folder.controller.ts index 81d8ae4ac..6505c2a44 100644 --- a/src/modules/folder/folder.controller.ts +++ b/src/modules/folder/folder.controller.ts @@ -9,6 +9,7 @@ import { NotImplementedException, Param, Query, + UseFilters, } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { FolderUseCases } from './folder.usecase'; @@ -19,6 +20,7 @@ import { Folder, SortableFolderAttributes } from './folder.domain'; import { FileStatus, SortableFileAttributes } from '../file/file.domain'; import logger from '../../externals/logger'; import { validate } from 'uuid'; +import { HttpExceptionFilter } from '../../lib/http/http-exception.filter'; const foldersStatuses = ['ALL', 'EXISTS', 'TRASHED', 'DELETED'] as const; @@ -255,7 +257,7 @@ export class FolderController { @UserDecorator() user: User, @Query('limit') limit: number, @Query('offset') offset: number, - @Query('status') status: typeof foldersStatuses[number], + @Query('status') status: (typeof foldersStatuses)[number], @Query('updatedAt') updatedAt?: string, ) { if (!status) { @@ -418,4 +420,12 @@ export class FolderController { }); } } + + @UseFilters(new HttpExceptionFilter()) + @Get(':uuid/size') + async getFolderSize(@Param('uuid') folderUuid: Folder['uuid']) { + const size = await this.folderUseCases.getFolderSizeByUuid(folderUuid); + + return { size }; + } } diff --git a/src/modules/folder/folder.repository.spec.ts b/src/modules/folder/folder.repository.spec.ts new file mode 100644 index 000000000..9aebca3ec --- /dev/null +++ b/src/modules/folder/folder.repository.spec.ts @@ -0,0 +1,95 @@ +import { createMock } from '@golevelup/ts-jest'; +import { CalculateFolderSizeTimeoutException } from './exception/calculate-folder-size-timeout.exception'; +import { SequelizeFolderRepository } from './folder.repository'; +import { FolderModel } from './folder.model'; +import { Folder } from './folder.domain'; +import { newFolder } from '../../../test/fixtures'; + +jest.mock('./folder.model', () => ({ + FolderModel: { + sequelize: { + query: jest.fn(() => Promise.resolve([[{ totalsize: 100 }]])), + }, + }, +})); + +describe('SequelizeFolderRepository', () => { + const TIMEOUT_SECONDS = 15; + const TIMEOUT_ERROR_CODE = '57014'; + + let repository: SequelizeFolderRepository; + let folderModel: typeof FolderModel; + let folder: Folder; + + beforeEach(async () => { + folderModel = createMock(); + + repository = new SequelizeFolderRepository(folderModel); + + folder = newFolder(); + }); + + describe('calculate folder size', () => { + it('When calculate folder size is requested, then it works', async () => { + const calculateSizeQuery = ` + BEGIN; + SET LOCAL statement_timeout TO :timeoutSeconds; + + WITH RECURSIVE folder_recursive AS ( + SELECT + fl1.uuid, + fl1.parent_uuid, + f1.size AS filesize + FROM folders fl1 + LEFT JOIN files f1 ON f1.folder_uuid = fl1.uuid + WHERE fl1.uuid = :folderUuid + + UNION ALL + + SELECT + fl2.uuid, + fl2.parent_uuid, + f2.size AS filesize + FROM folders fl2 + INNER JOIN files f2 ON f2.folder_uuid = fl2.uuid + INNER JOIN folder_recursive fr ON fr.uuid = fl2.parent_uuid + ) + + SELECT COALESCE(SUM(filesize), 0) AS total_size FROM folder_recursive; + COMMIT; + `; + + jest + .spyOn(FolderModel.sequelize, 'query') + .mockResolvedValue([[{ totalsize: 100 }]] as any); + + const size = await repository.calculateFolderSize( + folder.uuid, + TIMEOUT_SECONDS, + ); + + expect(size).toBeGreaterThanOrEqual(0); + expect(FolderModel.sequelize.query).toHaveBeenCalledWith( + calculateSizeQuery, + { + replacements: { + folderUuid: folder.uuid, + timeoutSeconds: TIMEOUT_SECONDS * 1000, + }, + }, + ); + }); + + it('When the folder size calculation times out, then throw an exception', async () => { + jest.spyOn(FolderModel.sequelize, 'query').mockRejectedValue({ + original: { + code: TIMEOUT_ERROR_CODE, + }, + }); + + await expect( + repository.calculateFolderSize(folder.uuid, TIMEOUT_SECONDS), + ).rejects.toThrow(CalculateFolderSizeTimeoutException); + }); + }); +}); diff --git a/src/modules/folder/folder.repository.ts b/src/modules/folder/folder.repository.ts index 825a4d058..0cc812fe5 100644 --- a/src/modules/folder/folder.repository.ts +++ b/src/modules/folder/folder.repository.ts @@ -6,12 +6,12 @@ import { v4 } from 'uuid'; import { Folder } from './folder.domain'; import { FolderAttributes } from './folder.attributes'; -import { UserModel } from '../user/user.model'; import { User } from '../user/user.domain'; import { UserAttributes } from '../user/user.attributes'; import { Pagination } from '../../lib/pagination'; import { FolderModel } from './folder.model'; import { SharingModel } from '../sharing/models'; +import { CalculateFolderSizeTimeoutException } from './exception/calculate-folder-size-timeout.exception'; function mapSnakeCaseToCamelCase(data) { const camelCasedObject = {}; @@ -59,6 +59,10 @@ export interface FolderRepository { ): Promise; deleteById(folderId: FolderAttributes['id']): Promise; clearOrphansFolders(userId: FolderAttributes['userId']): Promise; + calculateFolderSize( + folderUuid: string, + timeoutSeconds: number, + ): Promise; } @Injectable() @@ -66,8 +70,6 @@ export class SequelizeFolderRepository implements FolderRepository { constructor( @InjectModel(FolderModel) private folderModel: typeof FolderModel, - @InjectModel(UserModel) - private userModel: typeof UserModel, ) {} async findAllCursor( @@ -447,6 +449,59 @@ export class SequelizeFolderRepository implements FolderRepository { return folders.map((folder) => this.toDomain(folder)); } + async calculateFolderSize( + folderUuid: string, + timeoutSeconds = 15, + ): Promise { + try { + const calculateSizeQuery = ` + BEGIN; + SET LOCAL statement_timeout TO :timeoutSeconds; + + WITH RECURSIVE folder_recursive AS ( + SELECT + fl1.uuid, + fl1.parent_uuid, + f1.size AS filesize + FROM folders fl1 + LEFT JOIN files f1 ON f1.folder_uuid = fl1.uuid + WHERE fl1.uuid = :folderUuid + + UNION ALL + + SELECT + fl2.uuid, + fl2.parent_uuid, + f2.size AS filesize + FROM folders fl2 + INNER JOIN files f2 ON f2.folder_uuid = fl2.uuid + INNER JOIN folder_recursive fr ON fr.uuid = fl2.parent_uuid + ) + + SELECT COALESCE(SUM(filesize), 0) AS total_size FROM folder_recursive; + COMMIT; + `; + + const [[{ totalsize }]]: any = await FolderModel.sequelize.query( + calculateSizeQuery, + { + replacements: { + folderUuid, + timeoutSeconds: timeoutSeconds * 1000, + }, + }, + ); + + return +totalsize; + } catch (error) { + if (error.original?.code === '57014') { + throw new CalculateFolderSizeTimeoutException(); + } + + throw error; + } + } + private toDomain(model: FolderModel): Folder { return Folder.build({ ...model.toJSON(), @@ -459,4 +514,3 @@ export class SequelizeFolderRepository implements FolderRepository { return domain.toJSON(); } } -export { FolderModel }; diff --git a/src/modules/folder/folder.usecase.spec.ts b/src/modules/folder/folder.usecase.spec.ts index f6d46211d..829e7f36c 100644 --- a/src/modules/folder/folder.usecase.spec.ts +++ b/src/modules/folder/folder.usecase.spec.ts @@ -26,6 +26,8 @@ import { FolderModel } from './folder.model'; import { FileModel } from '../file/file.model'; import { SequelizeThumbnailRepository } from '../thumbnail/thumbnail.repository'; import { ThumbnailModel } from '../thumbnail/thumbnail.model'; +import { CalculateFolderSizeTimeoutException } from './exception/calculate-folder-size-timeout.exception'; +import { newFolder } from '../../../test/fixtures'; const folderId = 4; const userId = 1; @@ -492,4 +494,34 @@ describe('FolderUseCases', () => { } }); }); + + describe('get folder size', () => { + const folder = newFolder(); + + it('When the folder size is requested to be calculated, then it works', async () => { + const mockSize = 123456789; + + jest + .spyOn(folderRepository, 'calculateFolderSize') + .mockResolvedValueOnce(mockSize); + + const result = await service.getFolderSizeByUuid(folder.uuid); + + expect(result).toBe(mockSize); + expect(folderRepository.calculateFolderSize).toHaveBeenCalledTimes(1); + expect(folderRepository.calculateFolderSize).toHaveBeenCalledWith( + folder.uuid, + ); + }); + + it('When the folder size times out, then throw an exception', async () => { + jest + .spyOn(folderRepository, 'calculateFolderSize') + .mockRejectedValueOnce(new CalculateFolderSizeTimeoutException()); + + await expect(service.getFolderSizeByUuid(folder.uuid)).rejects.toThrow( + CalculateFolderSizeTimeoutException, + ); + }); + }); }); diff --git a/src/modules/folder/folder.usecase.ts b/src/modules/folder/folder.usecase.ts index fc5568b22..f34fad779 100644 --- a/src/modules/folder/folder.usecase.ts +++ b/src/modules/folder/folder.usecase.ts @@ -16,6 +16,7 @@ import { } from './folder.domain'; import { FolderAttributes } from './folder.attributes'; import { SequelizeFolderRepository } from './folder.repository'; +import { SequelizeFileRepository } from '../file/file.repository'; const invalidName = /[\\/]|^\s*$/; @@ -26,6 +27,7 @@ export class FolderUseCases { constructor( private folderRepository: SequelizeFolderRepository, private userRepository: SequelizeUserRepository, + private readonly fileRepository: SequelizeFileRepository, private readonly cryptoService: CryptoService, ) {} @@ -498,9 +500,8 @@ export class FolderUseCases { } async deleteOrphansFolders(userId: UserAttributes['id']): Promise { - let remainingFolders = await this.folderRepository.clearOrphansFolders( - userId, - ); + let remainingFolders = + await this.folderRepository.clearOrphansFolders(userId); if (remainingFolders > 0) { remainingFolders += await this.deleteOrphansFolders(userId); @@ -536,4 +537,8 @@ export class FolderUseCases { async deleteByUser(user: User, folders: Folder[]): Promise { await this.folderRepository.deleteByUser(user, folders); } + + getFolderSizeByUuid(folderUuid: Folder['uuid']): Promise { + return this.folderRepository.calculateFolderSize(folderUuid); + } } diff --git a/src/modules/send/send.usecase.spec.ts b/src/modules/send/send.usecase.spec.ts index 120b1de56..7e7054241 100644 --- a/src/modules/send/send.usecase.spec.ts +++ b/src/modules/send/send.usecase.spec.ts @@ -5,7 +5,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { Sequelize } from 'sequelize-typescript'; import { CryptoModule } from '../../externals/crypto/crypto.module'; import { NotificationService } from '../../externals/notifications/notification.service'; -import { FolderModel } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { User } from '../user/user.domain'; import { UserModel } from '../user/user.repository'; import { SendLink } from './send-link.domain'; diff --git a/src/modules/share/share.usecase.spec.ts b/src/modules/share/share.usecase.spec.ts index 03d17a0ad..edbff6f0f 100644 --- a/src/modules/share/share.usecase.spec.ts +++ b/src/modules/share/share.usecase.spec.ts @@ -16,10 +16,10 @@ import { import { FileUseCases } from '../file/file.usecase'; import { Folder } from '../folder/folder.domain'; import { - FolderModel, FolderRepository, SequelizeFolderRepository, } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { FolderUseCases } from '../folder/folder.usecase'; import { User } from '../user/user.domain'; import { SequelizeUserRepository, UserModel } from '../user/user.repository'; diff --git a/src/modules/trash/trash.usecase.spec.ts b/src/modules/trash/trash.usecase.spec.ts index 3b989c7c5..01d52991f 100644 --- a/src/modules/trash/trash.usecase.spec.ts +++ b/src/modules/trash/trash.usecase.spec.ts @@ -2,10 +2,8 @@ import { Test, TestingModule } from '@nestjs/testing'; import { TrashUseCases } from './trash.usecase'; import { SequelizeFileRepository } from '../file/file.repository'; import { File, FileAttributes } from '../file/file.domain'; -import { - FolderModel, - SequelizeFolderRepository, -} from '../folder/folder.repository'; +import { SequelizeFolderRepository } from '../folder/folder.repository'; +import { FolderModel } from '../folder/folder.model'; import { getModelToken } from '@nestjs/sequelize'; import { User } from '../user/user.domain'; import { SequelizeUserRepository, UserModel } from '../user/user.repository'; @@ -18,7 +16,7 @@ import { ShareModel, } from '../share/share.repository'; import { BridgeModule } from '../../externals/bridge/bridge.module'; -import { CryptoModule } from '../..//externals/crypto/crypto.module'; +import { CryptoModule } from '../../externals/crypto/crypto.module'; import { NotFoundException } from '@nestjs/common'; import { FileModel } from '../file/file.model'; import { ThumbnailModel } from '../thumbnail/thumbnail.model';