Add assets module (#1307)

* Add basic tests for assets module

* Add eager to createpresignedurl dto and implement CloudinaryService

* Move AppModule imports from module to dynamic resolver

* Rework assets service to return timestamp and signature

* Rework assets service to return timestamp and signature

* Add missing ApiProperty decorator for FormMetadata properties

* Add fake cloudinary environment variables to circleci config

* Add fake cloudinary secrets to circleci workflows

* Fix unit tests

* Update changelog

* Add assets migration

.circleci/config.yml

@@ -27,6 +27,8 @@ executors:
       REDIS_TLS_URL: "rediss://localhost:6379/0"
       REDIS_URL: "redis://localhost:6379/0"
       REDIS_USE_TLS: "0"
+      CLOUDINARY_SECRET: "fake_secret"
+      CLOUDINARY_KEY: "fake_key"
 
 jobs:
   setup:
@@ -76,6 +78,8 @@ jobs:
             REDIS_TLS_URL: "rediss://localhost:6379/0"
             REDIS_URL: "redis://localhost:6379/0"
             REDIS_USE_TLS: "0"
+            CLOUDINARY_SECRET: "fake_secret"
+            CLOUDINARY_KEY: "fake_key"
   build-public:
     executor: standard-node
     steps:

CHANGELOG.md

@@ -18,6 +18,7 @@ All notable changes to this project will be documented in this file. The format
   - Added "closed" to ListingStatus enum
   - Added Transform to ListingStatus field to return closed if applicationDueDate is in the past
   - Added "ohaFormat" to CSV exporter (includes OHA and HOPWA preferences) ([#1292](https://github.com/bloom-housing/bloom/pull/1292)) (Michał Plebański)
+  - `/assets` endpoints (create and createPresignedUploadMetadata)
 
 ### Frontend
 

backend/core/.env.template

@@ -9,3 +9,5 @@ THROTTLE_LIMIT=2
 EMAIL_API_KEY='SOME-LONG-SECRET-KEY'
 EMAIL_FROM_ADDRESS='Bloom Dev Housing Portal <[email protected]>'
 APP_SECRET='SOME-LONG-SECRET-KEY'
+CLOUDINARY_SECRET=CLOUDINARY_SECRET
+CLOUDINARY_KEY=CLOUDINARY_KEY

backend/core/archer.ts

@@ -301,6 +301,9 @@ export const ArcherListing: Listing = {
       // referenceType: "Listing",
       // TODO confirm not used anywhere
       // referenceId: "Uvbk5qurpB2WI9V6WnNdH",
+      id: "id",
+      createdAt: new Date(),
+      updatedAt: new Date(),
       label: "building",
       fileId:
         "https://regional-dahlia-staging.s3-us-west-1.amazonaws.com/listings/archer/archer-studios.jpg",

backend/core/package.json

@@ -23,7 +23,7 @@
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./jest-e2e.json --runInBand --forceExit",
-    "test:e2e:local": "psql -c 'DROP DATABASE IF EXISTS bloom_test' && psql -c 'CREATE DATABASE bloom_test' && ts-node ./node_modules/.bin/typeorm --config ./ormconfig.test.ts migration:run && ts-node src/seed.ts --test && jest --config ./jest-e2e.json --runInBand --forceExit",
+    "test:e2e:local": "psql -c 'DROP DATABASE IF EXISTS bloom_test' && psql -c 'CREATE DATABASE bloom_test' && ts-node ./node_modules/.bin/typeorm --config ./ormconfig.test.ts migration:run && ts-node src/seed.ts --test && yarn run test:e2e",
     "typeorm": "ts-node ./node_modules/.bin/typeorm",
     "herokusetup": "node heroku.setup.js",
     "heroku-postbuild": "rimraf dist && nest build && yarn run db:migration:run",
@@ -48,6 +48,7 @@
     "casbin": "^5.1.6",
     "class-transformer": "0.3.1",
     "class-validator": "^0.12.2",
+    "cloudinary": "^1.25.2",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
     "handlebars": "^4.7.6",

backend/core/src/app.module.ts

@@ -23,6 +23,7 @@ import { SharedModule } from "./shared/shared.module"
 import { ConfigModule, ConfigService } from "@nestjs/config"
 import { TranslationsModule } from "./translations/translations.module"
 import { Reflector } from "@nestjs/core"
+import { AssetsModule } from "./assets/assets.module"
 
 export function applicationSetup(app: INestApplication) {
   app.enableCors()
@@ -36,9 +37,7 @@ export function applicationSetup(app: INestApplication) {
   return app
 }
 
-@Module({
-  imports: [ApplicationFlaggedSetsModule],
-})
+@Module({})
 export class AppModule {
   static register(dbOptions): DynamicModule {
     /**
@@ -88,6 +87,8 @@ export class AppModule {
         AmiChartsModule,
         SharedModule,
         TranslationsModule,
+        ApplicationFlaggedSetsModule,
+        AssetsModule,
       ],
     }
   }

backend/core/src/applications/types/form-metadata/form-metadata-options.ts

@@ -1,5 +1,12 @@
 import { Expose, Type } from "class-transformer"
-import { ArrayMaxSize, IsOptional, IsString, MaxLength, ValidateNested } from "class-validator"
+import {
+  ArrayMaxSize,
+  IsBoolean,
+  IsOptional,
+  IsString,
+  MaxLength,
+  ValidateNested,
+} from "class-validator"
 import { ValidationsGroupsEnum } from "../../../shared/types/validations-groups-enum"
 import { FormMetadataExtraData } from "./form-metadata-extra-data"
 import { ApiProperty } from "@nestjs/swagger"
@@ -21,9 +28,13 @@ export class FormMetadataOptions {
 
   @Expose()
   @IsOptional({ groups: [ValidationsGroupsEnum.default] })
+  @IsBoolean({ groups: [ValidationsGroupsEnum.default] })
+  @ApiProperty()
   description?: boolean
 
   @Expose()
   @IsOptional({ groups: [ValidationsGroupsEnum.default] })
+  @IsBoolean({ groups: [ValidationsGroupsEnum.default] })
+  @ApiProperty()
   exclusive?: boolean
 }

backend/core/src/applications/types/form-metadata/form-metadata.ts

@@ -1,5 +1,12 @@
 import { Expose, Type } from "class-transformer"
-import { ArrayMaxSize, IsString, MaxLength, ValidateNested, IsOptional } from "class-validator"
+import {
+  ArrayMaxSize,
+  IsString,
+  MaxLength,
+  ValidateNested,
+  IsOptional,
+  IsBoolean,
+} from "class-validator"
 import { ValidationsGroupsEnum } from "../../../shared/types/validations-groups-enum"
 import { FormMetadataOptions } from "./form-metadata-options"
 import { ApiProperty } from "@nestjs/swagger"
@@ -20,14 +27,19 @@ export class FormMetadata {
 
   @Expose()
   @IsOptional({ groups: [ValidationsGroupsEnum.default] })
+  @IsBoolean({ groups: [ValidationsGroupsEnum.default] })
+  @ApiProperty()
   hideGenericDecline?: boolean
 
   @Expose()
-  @IsString({ groups: [ValidationsGroupsEnum.default] })
   @IsOptional({ groups: [ValidationsGroupsEnum.default] })
+  @IsString({ groups: [ValidationsGroupsEnum.default] })
+  @ApiProperty()
   customSelectText?: string
 
   @Expose()
   @IsOptional({ groups: [ValidationsGroupsEnum.default] })
+  @IsBoolean({ groups: [ValidationsGroupsEnum.default] })
+  @ApiProperty()
   hideFromListing?: boolean
 }

backend/core/src/assets/assets.controller.spec.ts

@@ -0,0 +1,24 @@
+import { Test, TestingModule } from "@nestjs/testing"
+import { AssetsController } from "./assets.controller"
+import { AuthModule } from "../auth/auth.module"
+import dbOptions = require("../../ormconfig.test")
+import { TypeOrmModule } from "@nestjs/typeorm"
+import { AssetsService } from "./services/assets.service"
+
+describe("AssetsController", () => {
+  let controller: AssetsController
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [AssetsController],
+      imports: [TypeOrmModule.forRoot(dbOptions), AuthModule],
+      providers: [{ provide: AssetsService, useValue: {} }],
+    }).compile()
+
+    controller = module.get<AssetsController>(AssetsController)
+  })
+
+  it("should be defined", () => {
+    expect(controller).toBeDefined()
+  })
+})

backend/core/src/assets/assets.controller.ts

@@ -0,0 +1,49 @@
+import { Body, Controller, Post, UseGuards, UsePipes, ValidationPipe } from "@nestjs/common"
+import { ApiBearerAuth, ApiOperation, ApiTags } from "@nestjs/swagger"
+import { mapTo } from "../shared/mapTo"
+import { ResourceType } from "../auth/decorators/resource-type.decorator"
+import { AuthzGuard } from "../auth/guards/authz.guard"
+import { defaultValidationPipeOptions } from "../shared/default-validation-pipe-options"
+import { DefaultAuthGuard } from "../auth/guards/default.guard"
+import { AssetsService } from "./services/assets.service"
+import {
+  AssetCreateDto,
+  AssetDto,
+  CreatePresignedUploadMetadataDto,
+  CreatePresignedUploadMetadataResponseDto,
+} from "./dto/asset.dto"
+
+@Controller("assets")
+@ApiTags("assets")
+@ApiBearerAuth()
+@ResourceType("asset")
+@UseGuards(DefaultAuthGuard, AuthzGuard)
+@UsePipes(
+  new ValidationPipe({
+    ...defaultValidationPipeOptions,
+  })
+)
+export class AssetsController {
+  constructor(private readonly assetsService: AssetsService) {}
+
+  @Post()
+  @ApiOperation({ summary: "Create asset", operationId: "create" })
+  async create(@Body() assetCreateDto: AssetCreateDto): Promise<AssetDto> {
+    const asset = await this.assetsService.create(assetCreateDto)
+    return mapTo(AssetDto, asset)
+  }
+
+  @Post("/presigned-upload-metadata")
+  @ApiOperation({
+    summary: "Create presigned upload metadata",
+    operationId: "createPresignedUploadMetadata",
+  })
+  async createPresignedUploadMetadata(
+    @Body() createPresignedUploadMetadataDto: CreatePresignedUploadMetadataDto
+  ): Promise<CreatePresignedUploadMetadataResponseDto> {
+    return mapTo(
+      CreatePresignedUploadMetadataResponseDto,
+      await this.assetsService.createPresignedUploadMetadata(createPresignedUploadMetadataDto)
+    )
+  }
+}

backend/core/src/assets/assets.module.ts

@@ -0,0 +1,15 @@
+import { Module } from "@nestjs/common"
+import { AssetsController } from "./assets.controller"
+import { AssetsService } from "./services/assets.service"
+import { CloudinaryService, UploadService } from "./services/upload.service"
+import { TypeOrmModule } from "@nestjs/typeorm"
+import { SharedModule } from "../shared/shared.module"
+import { Asset } from "./entities/asset.entity"
+import { AuthModule } from "../auth/auth.module"
+
+@Module({
+  controllers: [AssetsController],
+  providers: [AssetsService, { provide: UploadService, useClass: CloudinaryService }],
+  imports: [TypeOrmModule.forFeature([Asset]), AuthModule, SharedModule],
+})
+export class AssetsModule {}

backend/core/src/assets/dto/asset.dto.ts

@@ -0,0 +1,20 @@
+import { OmitType } from "@nestjs/swagger"
+import { Asset } from "../entities/asset.entity"
+import { Expose } from "class-transformer"
+import { IsDefined } from "class-validator"
+import { ValidationsGroupsEnum } from "../../shared/types/validations-groups-enum"
+
+export class AssetDto extends OmitType(Asset, [] as const) {}
+
+export class AssetCreateDto extends OmitType(AssetDto, ["id", "createdAt", "updatedAt"] as const) {}
+
+export class CreatePresignedUploadMetadataDto {
+  @Expose()
+  @IsDefined({ groups: [ValidationsGroupsEnum.default] })
+  parametersToSign: Record<string, string>
+}
+
+export class CreatePresignedUploadMetadataResponseDto {
+  @Expose()
+  signature: string
+}

backend/core/src/assets/entities/asset.entity.ts

@@ -0,0 +1,20 @@
+import { AbstractEntity } from "../../shared/entities/abstract.entity"
+import { Expose } from "class-transformer"
+import { IsString, MaxLength } from "class-validator"
+import { ValidationsGroupsEnum } from "../../shared/types/validations-groups-enum"
+import { Column, Entity } from "typeorm"
+
+@Entity({ name: "assets" })
+export class Asset extends AbstractEntity {
+  @Column({ type: "text" })
+  @Expose()
+  @IsString({ groups: [ValidationsGroupsEnum.default] })
+  @MaxLength(128, { groups: [ValidationsGroupsEnum.default] })
+  fileId: string
+
+  @Column({ type: "text" })
+  @Expose()
+  @IsString({ groups: [ValidationsGroupsEnum.default] })
+  @MaxLength(128, { groups: [ValidationsGroupsEnum.default] })
+  label: string
+}

backend/core/src/assets/services/assets.service.spec.ts

@@ -0,0 +1,25 @@
+import { Test, TestingModule } from "@nestjs/testing"
+import { AssetsService } from "./assets.service"
+import { getRepositoryToken } from "@nestjs/typeorm"
+import { Asset } from "../entities/asset.entity"
+import { UploadService } from "./upload.service"
+
+describe("AssetsService", () => {
+  let service: AssetsService
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [
+        AssetsService,
+        { provide: getRepositoryToken(Asset), useValue: {} },
+        { provide: UploadService, useValue: {} },
+      ],
+    }).compile()
+
+    service = module.get<AssetsService>(AssetsService)
+  })
+
+  it("should be defined", () => {
+    expect(service).toBeDefined()
+  })
+})

backend/core/src/assets/services/assets.service.ts

@@ -0,0 +1,30 @@
+import { Injectable } from "@nestjs/common"
+import {
+  AssetCreateDto,
+  CreatePresignedUploadMetadataDto,
+  CreatePresignedUploadMetadataResponseDto,
+} from "../dto/asset.dto"
+import { InjectRepository } from "@nestjs/typeorm"
+import { Repository } from "typeorm"
+import { Asset } from "../entities/asset.entity"
+import { UploadService } from "./upload.service"
+
+@Injectable()
+export class AssetsService {
+  constructor(
+    @InjectRepository(Asset) private readonly repository: Repository<Asset>,
+    private readonly uploadService: UploadService
+  ) {}
+
+  async create(assetCreateDto: AssetCreateDto) {
+    return await this.repository.save(assetCreateDto)
+  }
+
+  createPresignedUploadMetadata(
+    createUploadUrlDto: CreatePresignedUploadMetadataDto
+  ): Promise<CreatePresignedUploadMetadataResponseDto> {
+    return Promise.resolve(
+      this.uploadService.createPresignedUploadMetadata(createUploadUrlDto.parametersToSign)
+    )
+  }
+}

backend/core/src/assets/services/upload.service.ts

@@ -0,0 +1,25 @@
+import { Injectable } from "@nestjs/common"
+import { ConfigService } from "@nestjs/config"
+import { v2 as cloudinary } from "cloudinary"
+
+export abstract class UploadService {
+  abstract createPresignedUploadMetadata(
+    parametersToSign: Record<string, string>
+  ): { signature: string }
+}
+
+@Injectable()
+export class CloudinaryService implements UploadService {
+  constructor(private readonly configService: ConfigService) {}
+
+  createPresignedUploadMetadata(parametersToSign: Record<string, string>): { signature: string } {
+    // Based on https://cloudinary.com/documentation/upload_images#signed_upload_video_tutorial
+    const signature = cloudinary.utils.api_sign_request(
+      parametersToSign,
+      this.configService.get<string>("CLOUDINARY_SECRET")
+    )
+    return {
+      signature,
+    }
+  }
+}

backend/core/src/auth/auth.module.ts

@@ -11,7 +11,6 @@ import { SharedModule } from "../shared/shared.module"
 import { AuthzService } from "./authz.service"
 import { ConfigModule, ConfigService } from "@nestjs/config"
 import { UserModule } from "../user/user.module"
-import Joi from "joi"
 
 @Module({
   imports: [
@@ -29,11 +28,6 @@ import Joi from "joi"
     TypeOrmModule.forFeature([RevokedToken]),
     SharedModule,
     forwardRef(() => UserModule),
-    ConfigModule.forRoot({
-      validationSchema: Joi.object({
-        APP_SECRET: Joi.string().required().min(16),
-      }),
-    }),
   ],
   providers: [LocalStrategy, JwtStrategy, AuthService, AuthzService],
   exports: [AuthzService, AuthService],