From bb37290e38b89494cefd0ffd60f0befc5ac13c9f Mon Sep 17 00:00:00 2001 From: Sean Albert Date: Wed, 11 Aug 2021 12:43:48 -0700 Subject: [PATCH 1/3] Revert redis (#1659) * force quit redis connection on app close * pulls back redis changes until they're ready --- backend/core/src/app.module.ts | 2 -- backend/core/src/listings/listings.module.ts | 36 ++------------------ backend/core/src/seed.ts | 1 - 3 files changed, 3 insertions(+), 36 deletions(-) diff --git a/backend/core/src/app.module.ts b/backend/core/src/app.module.ts index 48174442e0..6851d2db56 100644 --- a/backend/core/src/app.module.ts +++ b/backend/core/src/app.module.ts @@ -48,8 +48,6 @@ export function applicationSetup(app: INestApplication) { app.useGlobalInterceptors( new ClassSerializerInterceptor(app.get(Reflector), { excludeExtraneousValues: true }) ) - // Starts listening for shutdown hooks - app.enableShutdownHooks() return app } diff --git a/backend/core/src/listings/listings.module.ts b/backend/core/src/listings/listings.module.ts index 16bf83c3ea..7528ae5ac4 100644 --- a/backend/core/src/listings/listings.module.ts +++ b/backend/core/src/listings/listings.module.ts @@ -1,6 +1,5 @@ -import { CacheModule, CACHE_MANAGER, Inject, Module, OnModuleDestroy } from "@nestjs/common" +import { CacheModule, Module } from "@nestjs/common" import { TypeOrmModule } from "@nestjs/typeorm" -import * as redisStore from "cache-manager-redis-store" import { ListingsService } from "./listings.service" import { ListingsController } from "./listings.controller" import { Listing } from "./entities/listing.entity" @@ -9,28 +8,12 @@ import { Preference } from "../preferences/entities/preference.entity" import { AuthModule } from "../auth/auth.module" import { User } from "../auth/entities/user.entity" import { Property } from "../property/entities/property.entity" -import { Store } from "cache-manager" -import Redis from "redis" - -interface RedisCache extends Cache { - store: RedisStore -} - -interface RedisStore extends Store { - name: "redis" - getClient: () => Redis.RedisClient - isCacheableValue: (value: unknown) => boolean -} @Module({ imports: [ CacheModule.register({ ttl: 24 * 60 * 60, - store: redisStore, - url: process.env.REDIS_USE_TLS === "0" ? process.env.REDIS_URL : process.env.REDIS_TLS_URL, - tls: { - rejectUnauthorized: false, - }, + max: 10, }), TypeOrmModule.forFeature([Listing, Preference, Unit, User, Property]), AuthModule, @@ -39,17 +22,4 @@ interface RedisStore extends Store { exports: [ListingsService], controllers: [ListingsController], }) -// We have to manually disconnect from redis on app close -export class ListingsModule implements OnModuleDestroy { - redisClient: Redis.RedisClient - constructor(@Inject(CACHE_MANAGER) private cacheManager: RedisCache) { - this.redisClient = this.cacheManager.store.getClient() - - this.redisClient.on("error", (error) => { - console.log("redis error = ", error) - }) - } - onModuleDestroy() { - this.redisClient.quit() - } -} +export class ListingsModule {} diff --git a/backend/core/src/seed.ts b/backend/core/src/seed.ts index c09a8b6cae..e8e3c13b41 100644 --- a/backend/core/src/seed.ts +++ b/backend/core/src/seed.ts @@ -120,7 +120,6 @@ const seedListings = async (app: INestApplicationContext) => { async function seed() { const app = await NestFactory.create(SeederModule.forRoot({ test: argv.test })) - const userService = await app.resolve(UserService) const userRepo = app.get>(getRepositoryToken(User)) From d583fab9a90d406ab2d1f637d26411e852bf5a46 Mon Sep 17 00:00:00 2001 From: Sean Albert Date: Wed, 11 Aug 2021 16:20:38 -0700 Subject: [PATCH 2/3] Force quits redis connection on app close (#1658) * force quit redis connection on app close * updates for redis config * adds enableShutdownHooks to main app --- backend/core/src/listings/listings.module.ts | 49 +++++++++++++++++--- backend/core/src/main.ts | 2 + backend/core/src/seed.ts | 2 + 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/backend/core/src/listings/listings.module.ts b/backend/core/src/listings/listings.module.ts index 7528ae5ac4..0782faae98 100644 --- a/backend/core/src/listings/listings.module.ts +++ b/backend/core/src/listings/listings.module.ts @@ -1,5 +1,6 @@ -import { CacheModule, Module } from "@nestjs/common" +import { CacheModule, CACHE_MANAGER, Inject, Module, OnModuleDestroy } from "@nestjs/common" import { TypeOrmModule } from "@nestjs/typeorm" +import * as redisStore from "cache-manager-redis-store" import { ListingsService } from "./listings.service" import { ListingsController } from "./listings.controller" import { Listing } from "./entities/listing.entity" @@ -8,13 +9,36 @@ import { Preference } from "../preferences/entities/preference.entity" import { AuthModule } from "../auth/auth.module" import { User } from "../auth/entities/user.entity" import { Property } from "../property/entities/property.entity" +import { Store } from "cache-manager" +import Redis from "redis" + +interface RedisCache extends Cache { + store: RedisStore +} + +interface RedisStore extends Store { + name: "redis" + getClient: () => Redis.RedisClient + isCacheableValue: (value: unknown) => boolean +} + +const cacheConfig = { + ttl: 24 * 60 * 60, + store: redisStore, + url: process.env.REDIS_URL, + tls: undefined, +} + +if (process.env.REDIS_USE_TLS !== "0") { + cacheConfig.url = process.env.REDIS_TLS_URL + cacheConfig.tls = { + rejectUnauthorized: false, + } +} @Module({ imports: [ - CacheModule.register({ - ttl: 24 * 60 * 60, - max: 10, - }), + CacheModule.register(cacheConfig), TypeOrmModule.forFeature([Listing, Preference, Unit, User, Property]), AuthModule, ], @@ -22,4 +46,17 @@ import { Property } from "../property/entities/property.entity" exports: [ListingsService], controllers: [ListingsController], }) -export class ListingsModule {} +// We have to manually disconnect from redis on app close +export class ListingsModule implements OnModuleDestroy { + redisClient: Redis.RedisClient + constructor(@Inject(CACHE_MANAGER) private cacheManager: RedisCache) { + this.redisClient = this.cacheManager.store.getClient() + + this.redisClient.on("error", (error) => { + console.log("redis error = ", error) + }) + } + onModuleDestroy() { + this.redisClient.quit() + } +} diff --git a/backend/core/src/main.ts b/backend/core/src/main.ts index 44607b2360..d3371e58ac 100644 --- a/backend/core/src/main.ts +++ b/backend/core/src/main.ts @@ -9,6 +9,8 @@ import dbOptions = require("../ormconfig") let app async function bootstrap() { app = await NestFactory.create(AppModule.register(dbOptions)) + // Starts listening for shutdown hooks + app.enableShutdownHooks() app = applicationSetup(app) const conn = getConnection() // showMigrations returns true if there are pending migrations diff --git a/backend/core/src/seed.ts b/backend/core/src/seed.ts index e8e3c13b41..b7428d0829 100644 --- a/backend/core/src/seed.ts +++ b/backend/core/src/seed.ts @@ -120,6 +120,8 @@ const seedListings = async (app: INestApplicationContext) => { async function seed() { const app = await NestFactory.create(SeederModule.forRoot({ test: argv.test })) + // Starts listening for shutdown hooks + app.enableShutdownHooks() const userService = await app.resolve(UserService) const userRepo = app.get>(getRepositoryToken(User)) From 77bfa7b1cbdc30d294738ff85c814da30c7e26e3 Mon Sep 17 00:00:00 2001 From: Emily Jablonski <65367387+emilyjablonski@users.noreply.github.com> Date: Wed, 11 Aug 2021 19:20:14 -0600 Subject: [PATCH 3/3] 1663/when seeding data, create application methods unique to each listing (#1662) * create an unique app method for each listing * fix importer * changelog * Update listings.e2e-spec.ts Co-authored-by: seanmalbert --- CHANGELOG.md | 1 + backend/core/scripts/listings-importer.ts | 5 ++ .../dto/application-method.dto.ts | 1 - backend/core/src/seed.ts | 50 +++++-------------- .../seeds/listings/listing-coliseum-seed.ts | 5 -- .../seeds/listings/listing-default-seed.ts | 6 --- .../src/seeds/listings/listing-triton-seed.ts | 5 -- .../core/test/listings/listings.e2e-spec.ts | 4 +- 8 files changed, 21 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 748e5a5932..9c2f2e9f14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ All notable changes to this project will be documented in this file. The format - Fixed: - Added checks for property in listing.dto transforms - Display all listings on partners with `limit=all` ([#1635](https://github.com/bloom-housing/bloom/issues/1635)) (Marcin Jędras) + - Seed data should create unique application methods ([#1662](https://github.com/bloom-housing/bloom/issues/1662)) (Emily Jablonski) ## v1.0.5 08/03/2021 diff --git a/backend/core/scripts/listings-importer.ts b/backend/core/scripts/listings-importer.ts index 06407e37f2..ce7751044b 100644 --- a/backend/core/scripts/listings-importer.ts +++ b/backend/core/scripts/listings-importer.ts @@ -36,6 +36,8 @@ const authService = new client.AuthService() const amiChartService = new client.AmiChartsService() const unitTypesService = new client.UnitTypesService() const unitAccessibilityPriorityTypesService = new client.UnitAccessibilityPriorityTypesService() +const applicationMethodsService = new client.ApplicationMethodsService() +const reservedCommunityTypesService = new client.ReservedCommunityTypesService() async function uploadEntity(entityKey, entityService, listing) { const newRecordsIds = await Promise.all( @@ -144,11 +146,14 @@ async function main() { }) const unitTypes = await unitTypesService.list() const priorityTypes = await unitAccessibilityPriorityTypesService.list() + const reservedCommunityTypes = await reservedCommunityTypesService.list() let listing = JSON.parse(fs.readFileSync(listingFilePath, "utf-8")) const relationsKeys = [] listing = reformatListing(listing, relationsKeys) listing = await uploadEntity("preferences", preferencesService, listing) + listing = await uploadEntity("applicationMethods", applicationMethodsService, listing) + listing.reservedCommunityType = findByName(reservedCommunityTypes, listing.reservedCommunityType) const amiChartName = listing.amiChart.name let chart = await getAmiChart(amiChartName) diff --git a/backend/core/src/application-methods/dto/application-method.dto.ts b/backend/core/src/application-methods/dto/application-method.dto.ts index 073e9661e2..12f861de2e 100644 --- a/backend/core/src/application-methods/dto/application-method.dto.ts +++ b/backend/core/src/application-methods/dto/application-method.dto.ts @@ -7,7 +7,6 @@ import { PaperApplicationDto } from "../../paper-applications/dto/paper-applicat import { IdDto } from "../../shared/dto/id.dto" export class ApplicationMethodDto extends OmitType(ApplicationMethod, [ - "listing", "paperApplications", ] as const) { @Expose() diff --git a/backend/core/src/seed.ts b/backend/core/src/seed.ts index b7428d0829..bced9fec00 100644 --- a/backend/core/src/seed.ts +++ b/backend/core/src/seed.ts @@ -20,9 +20,6 @@ import { ListingTritonSeed } from "./seeds/listings/listing-triton-seed" import { ListingDefaultBmrChartSeed } from "./seeds/listings/listing-default-bmr-chart-seed" import { ApplicationMethodsService } from "./application-methods/application-methods.service" import { ApplicationMethodType } from "./application-methods/types/application-method-type-enum" -import { PaperApplicationsService } from "./paper-applications/paper-applications.service" -import { Language } from "./shared/types/language-enum" -import { AssetsService } from "./assets/services/assets.service" import { AuthContext } from "./auth/types/auth-context" import { ListingDefaultReservedSeed } from "./seeds/listings/listing-default-reserved-seed" import { ListingDefaultFCFSSeed } from "./seeds/listings/listing-default-fcfs-seed" @@ -64,52 +61,29 @@ export async function createLeasingAgents(app: INestApplicationContext) { return leasingAgents } -async function createApplicationMethods(app: INestApplicationContext) { - const assetsService = await app.resolve(AssetsService) - const englishFileAsset = await assetsService.create({ - fileId: "englishFileId", - label: "English paper application", - }) - const paperApplicationsService = await app.resolve( - PaperApplicationsService - ) - const englishPaperApplication = await paperApplicationsService.create({ - language: Language.en, - file: englishFileAsset, - }) - const applicationMethodsService = await app.resolve( - ApplicationMethodsService - ) - - await applicationMethodsService.create({ - type: ApplicationMethodType.FileDownload, - acceptsPostmarkedApplications: false, - externalReference: "https://bit.ly/2wH6dLF", - label: "English", - paperApplications: [englishPaperApplication], - }) - - await applicationMethodsService.create({ - type: ApplicationMethodType.Internal, - acceptsPostmarkedApplications: false, - externalReference: "", - label: "Label", - paperApplications: [], - }) -} - const seedListings = async (app: INestApplicationContext) => { const seeds = [] const leasingAgents = await createLeasingAgents(app) - await createApplicationMethods(app) const allSeeds = listingSeeds.map((listingSeed) => app.get(listingSeed)) const listingRepository = app.get>(getRepositoryToken(Listing)) + const applicationMethodsService = await app.resolve( + ApplicationMethodsService + ) for (const [index, listingSeed] of allSeeds.entries()) { const everyOtherAgent = index % 2 ? leasingAgents[0] : leasingAgents[1] const listing = await listingSeed.seed() listing.leasingAgents = [everyOtherAgent] + const applicationMethods = await applicationMethodsService.create({ + type: ApplicationMethodType.Internal, + acceptsPostmarkedApplications: false, + externalReference: "", + label: "Label", + paperApplications: [], + listing: listing, + }) + listing.applicationMethods = [applicationMethods] await listingRepository.save(listing) seeds.push(listing) diff --git a/backend/core/src/seeds/listings/listing-coliseum-seed.ts b/backend/core/src/seeds/listings/listing-coliseum-seed.ts index 83a1c90ef1..d8d3977bad 100644 --- a/backend/core/src/seeds/listings/listing-coliseum-seed.ts +++ b/backend/core/src/seeds/listings/listing-coliseum-seed.ts @@ -15,7 +15,6 @@ import { Listing } from "../../listings/entities/listing.entity" import { BaseEntity, DeepPartial } from "typeorm" import { UnitCreateDto } from "../../units/dto/unit.dto" import { ListingDefaultSeed } from "./listing-default-seed" -import { ApplicationMethodType } from "../../application-methods/types/application-method-type-enum" import { UnitStatus } from "../../units/types/unit-status-enum" const coliseumProperty: PropertySeedType = { @@ -1011,9 +1010,6 @@ export class ListingColiseumSeed extends ListingDefaultSeed { } await this.unitsRepository.save(unitsToBeCreated) - const applicationMethods = await this.applicationMethodRepository.find({ - type: ApplicationMethodType.Internal, - }) const listingCreateDto: Omit< DeepPartial, @@ -1027,7 +1023,6 @@ export class ListingColiseumSeed extends ListingDefaultSeed { { ...getPbvPreference(), ordinal: 2, page: 2 }, { ...getHopwaPreference(), ordinal: 3, page: 3 }, ], - applicationMethods: applicationMethods, events: [], } diff --git a/backend/core/src/seeds/listings/listing-default-seed.ts b/backend/core/src/seeds/listings/listing-default-seed.ts index 728d2b9ba0..dbbc62124d 100644 --- a/backend/core/src/seeds/listings/listing-default-seed.ts +++ b/backend/core/src/seeds/listings/listing-default-seed.ts @@ -21,7 +21,6 @@ import { getLiveWorkPreference, } from "./shared" import { ApplicationMethod } from "../../application-methods/entities/application-method.entity" -import { ApplicationMethodType } from "../../application-methods/types/application-method-type-enum" export class ListingDefaultSeed { constructor( @@ -69,11 +68,7 @@ export class ListingDefaultSeed { unitsToBeCreated[1].priorityType = priorityTypeMobilityAndHearing unitsToBeCreated[0].unitType = unitTypeOneBdrm unitsToBeCreated[1].unitType = unitTypeTwoBdrm - await this.unitsRepository.save(unitsToBeCreated) - const applicationMethods = await this.applicationMethodRepository.find({ - type: ApplicationMethodType.Internal, - }) const listingCreateDto: Omit< DeepPartial, @@ -84,7 +79,6 @@ export class ListingDefaultSeed { property: property, assets: getDefaultAssets(), preferences: [getLiveWorkPreference(), { ...getDisplaceePreference(), ordinal: 2 }], - applicationMethods: applicationMethods, events: getDefaultListingEvents(), } diff --git a/backend/core/src/seeds/listings/listing-triton-seed.ts b/backend/core/src/seeds/listings/listing-triton-seed.ts index 0c0de4405a..573a206071 100644 --- a/backend/core/src/seeds/listings/listing-triton-seed.ts +++ b/backend/core/src/seeds/listings/listing-triton-seed.ts @@ -4,7 +4,6 @@ import { getDefaultAmiChart, getDate, getDefaultAssets, getLiveWorkPreference } import { ListingStatus } from "../../listings/types/listing-status-enum" import { CountyCode } from "../../shared/types/county-code" import { CSVFormattingType } from "../../csv/types/csv-formatting-type-enum" -import { ApplicationMethodType } from "../../application-methods/types/application-method-type-enum" import { AmiChart } from "../../ami-charts/entities/ami-chart.entity" import { ListingDefaultSeed } from "./listing-default-seed" import { UnitCreateDto } from "../../units/dto/unit.dto" @@ -782,9 +781,6 @@ export class ListingTritonSeed extends ListingDefaultSeed { unitsToBeCreated[4].unitType = unitTypeOneBdrm await this.unitsRepository.save(unitsToBeCreated) - const applicationMethods = await this.applicationMethodRepository.find({ - type: ApplicationMethodType.FileDownload, - }) const listingCreateDto: Omit< DeepPartial, @@ -794,7 +790,6 @@ export class ListingTritonSeed extends ListingDefaultSeed { property: property, assets: getDefaultAssets(), preferences: [getLiveWorkPreference()], - applicationMethods: applicationMethods, events: [], } diff --git a/backend/core/test/listings/listings.e2e-spec.ts b/backend/core/test/listings/listings.e2e-spec.ts index 8fee7d451b..220ba8de7d 100644 --- a/backend/core/test/listings/listings.e2e-spec.ts +++ b/backend/core/test/listings/listings.e2e-spec.ts @@ -16,6 +16,7 @@ import { PaperApplicationsModule } from "../../src/paper-applications/paper-appl import { ListingEventCreateDto } from "../../src/listings/dto/listing-event.dto" import { ListingEventType } from "../../src/listings/types/listing-event-type-enum" import { getSeedListingsCount } from "../../src/seed" +import { Listing } from "../../src/listings/entities/listing.entity" // eslint-disable-next-line @typescript-eslint/no-var-requires const dbOptions = require("../../ormconfig.test") @@ -152,7 +153,7 @@ describe("Listings", () => { it("should add/overwrite application methods in existing listing", async () => { const res = await supertest(app.getHttpServer()).get("/listings").expect(200) - const listing: ListingUpdateDto = { ...res.body.items[0] } + const listing: Listing = { ...res.body.items[0] } const adminAccessToken = await getUserAccessToken(app, "admin@example.com", "abcdef") @@ -176,6 +177,7 @@ describe("Listings", () => { const am: ApplicationMethodCreateDto = { type: ApplicationMethodType.FileDownload, paperApplications: [{ id: paperApplication.body.id }], + listing: listing, } const applicationMethod = await supertest(app.getHttpServer())