Skip to content

Commit

Permalink
fix(backend): import scripts (bloom-housing#1031)
Browse files Browse the repository at this point in the history 
* fix(backend): import scripts

* style: linter

Co-authored-by: Michal Plebanski <[email protected]>
Co-authored-by: Sean Albert <[email protected]>
  • Loading branch information
@pbn4 @seanmalbert
3 people authored Mar 10, 2022
1 parent 9d61638 commit 7b101f8
Show file tree
Hide file tree
Showing 6 changed files with 184 additions and 69 deletions.
3 changes: 3 additions & 0 deletions backend/core/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"@nestjs/typeorm": "^7.1.0",
"@types/cache-manager": "^3.4.0",
"@types/cron": "^1.7.3",
"@types/mapbox": "^1.6.42",
"async-retry": "^1.3.1",
"axios": "0.21.1",
"bull": "^4.1.0",
Expand All @@ -78,6 +79,7 @@
"ioredis": "^4.24.4",
"joi": "^17.3.0",
"jwt-simple": "^0.5.6",
"mapbox": "^1.0.0-beta10",
"nanoid": "^3.1.12",
"nestjs-throttler-storage-redis": "^0.1.11",
"nestjs-typeorm-paginate": "^3.1.3",
Expand Down Expand Up @@ -115,6 +117,7 @@
"dotenv": "^8.2.0",
"fishery": "^0.3.0",
"jest": "^26.5.3",
"mapbox": "^1.0.0-beta10",
"supertest": "^4.0.2",
"swagger-axios-codegen": "0.11.16",
"ts-jest": "26.4.1",
Expand Down
37 changes: 25 additions & 12 deletions backend/core/scripts/import-listings-basic.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,15 @@ import dbOptions = require("../ormconfig")
import { Program } from "../src/program/entities/program.entity"
import { AddressCreateDto } from "../src/shared/dto/address.dto"

// TODO where are multiple yearbuilt values that are string in format number/number
var MapboxClient = require('mapbox');

if (!process.env["MAPBOX_TOKEN"]) {
throw new Error("environment variable MAPBOX_TOKEN is undefined")
}
const args = process.argv.slice(2)

const client = new MapboxClient(process.env["MAPBOX_TOKEN"])

const filePath = args[0]
if (typeof filePath !== "string" && !fs.existsSync(filePath)) {
throw new Error(`usage: ts-node import-unit-groups.ts csv-file-path`)
Expand Down Expand Up @@ -59,6 +64,7 @@ export class HeaderConstants {
public static readonly RequiredDocuments: string = "Required Documents"
public static readonly ImportantProgramRules: string = "Important Program Rules"
public static readonly SpecialNotes: string = "Special Notes"

}

async function fetchDetroitJurisdiction(connection: Connection): Promise<Jurisdiction> {
Expand Down Expand Up @@ -98,31 +104,37 @@ function destructureYearBuilt(yearBuilt: string): number {
}

if (yearBuilt.includes("/")) {
const [year1, _] = yearBuilt.split("/")
return Number.parseInt(year1)
const [year1, year2] = yearBuilt.split("/")
return Number.parseInt(year2)
}

return Number.parseInt(yearBuilt)
}

function destructureAddressString(addressString: string): AddressCreateDto {
async function destructureAddressString(addressString: string): Promise<AddressCreateDto> {
if (!addressString) {
return {
street: undefined,
city: undefined,
state: undefined,
zipCode: undefined,
}
return null
}

const tokens = addressString.split(",").map((addressString) => addressString.trim())

let latitude
let longitude

const res = await client.geocodeForward(addressString)
if (res.entity?.features?.length) {
latitude = res.entity.features[0].center[0]
longitude = res.entity.features[0].center[1]
}

if (tokens.length === 1) {
return {
street: tokens[0],
city: undefined,
state: undefined,
zipCode: undefined,
latitude,
longitude
}
}

Expand All @@ -133,6 +145,8 @@ function destructureAddressString(addressString: string): AddressCreateDto {
city: tokens[1],
state,
zipCode,
latitude,
longitude
}
}

Expand Down Expand Up @@ -165,7 +179,6 @@ async function main() {
temporaryListingId: row[HeaderConstants.TemporaryId],
assets: [],
name: row[HeaderConstants.Name],
// TODO should displayWaitlistSize be false?
displayWaitlistSize: false,
property: {
developer: row[HeaderConstants.Developer],
Expand Down Expand Up @@ -193,7 +206,7 @@ async function main() {
leasingAgentEmail: row[HeaderConstants.LeasingAgentEmail],
leasingAgentPhone: row[HeaderConstants.LeasingAgentPhone],
managementWebsite: row[HeaderConstants.ManagementWebsite],
leasingAgentAddress: destructureAddressString(row[HeaderConstants.LeasingAgentAddress]),
leasingAgentAddress: await destructureAddressString(row[HeaderConstants.LeasingAgentAddress]),
applicationFee: row[HeaderConstants.ApplicationFee],
depositMin: row[HeaderConstants.DepositMin],
depositMax: row[HeaderConstants.DepositMax],
Expand Down
161 changes: 106 additions & 55 deletions backend/core/scripts/import-unit-groups.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,7 @@ import { MSHDA2021 } from "../src/seeder/seeds/ami-charts/MSHDA2021"
import { MonthlyRentDeterminationType } from "../src/units-summary/types/monthly-rent-determination.enum"
import dbOptions = require("../ormconfig")

// TODO how to solve 4+BR

type AmiChartNameType = "MSHDA" | "HUD"
type TAmiChartLevel = {
amiChartName: AmiChartNameType
amiPercentage: number
}

const args = process.argv.slice(2)

Expand All @@ -37,32 +31,22 @@ export class HeaderConstants {
public static readonly WaitlistOpen: string = "Waitlist Open"
public static readonly AMIChart: string = "AMI Chart"
public static readonly AmiChartPercentage: string = "Percent AMIs"
}

function generateAmiChartLevels(
amiChartsColumns: string,
amiPercentagesColumn: string | number
): Array<TAmiChartLevel> {
const amiChartLevels = []

for (const amiChartName of amiChartsColumns.split("/")) {
// TODO remove && amiPercentagesColumn when empty AMI percentage column problem is solved
if (typeof amiPercentagesColumn === "string" && amiPercentagesColumn) {
for (const amiPercentage of amiPercentagesColumn.split(",").map((s) => s.trim())) {
amiChartLevels.push({
amiChartName,
amiPercentage: Number.parseInt(amiPercentage),
})
}
} else if (typeof amiPercentagesColumn === "number") {
amiChartLevels.push({
amiChartName,
amiPercentage: amiPercentagesColumn,
})
}
}

return amiChartLevels
public static readonly Value20: string = "20% (Value)"
public static readonly Value25: string = "25% (Value)"
public static readonly Value30: string = "30% (Value)"
public static readonly Value35: string = "35% (Value)"
public static readonly Value40: string = "40% (Value)"
public static readonly Value45: string = "45% (Value)"
public static readonly Value50: string = "50% (Value)"
public static readonly Value55: string = "55% (Value)"
public static readonly Value60: string = "60% (Value)"
public static readonly Value70: string = "70% (Value)"
public static readonly Value80: string = "80% (Value)"
public static readonly Value100: string = "100% (Value)"
public static readonly Value120: string = "120% (Value)"
public static readonly Value125: string = "125% (Value)"
public static readonly Value140: string = "140% (Value)"
public static readonly Value150: string = "150% (Value)"
}

function findAmiChartByName(
Expand All @@ -79,32 +63,99 @@ function findAmiChartByName(
)
}

function getFlatRentValueForAmiChart(amiChart: AmiChart, amiPercentage: number) {
return amiChart.items.find((item) => item.percentOfAmi === amiPercentage).percentOfAmi
function parseAmiStringValue(value: string | number) {
if (typeof value === "number") {
return value
} else if (typeof value === "string") {
const retval = Number.parseInt(value.replace(/\$/, "").replace(/,/, ""))
if (!retval) {
console.log("dollar value")
console.log(value)
throw new Error("Failed to parse $ (dolar) value")
}
return retval
} else {
throw new Error("Unknown ami value type")
}
}

function getAmiValueFromColumn(row, amiPercentage: number, type: "percentage" | "flat") {
const mapAmiPercentageToColumnName = {
20: HeaderConstants.Value20,
25: HeaderConstants.Value25,
30: HeaderConstants.Value30,
35: HeaderConstants.Value35,
40: HeaderConstants.Value40,
45: HeaderConstants.Value45,
50: HeaderConstants.Value50,
55: HeaderConstants.Value55,
60: HeaderConstants.Value60,
70: HeaderConstants.Value70,
80: HeaderConstants.Value80,
100: HeaderConstants.Value100,
120: HeaderConstants.Value120,
125: HeaderConstants.Value125,
140: HeaderConstants.Value140,
150: HeaderConstants.Value150,
}
const value = row[mapAmiPercentageToColumnName[amiPercentage]]

if (value) {
const splitValues = value.split("/")

if (splitValues.length === 1) {
return parseAmiStringValue(value)
} else if (splitValues.length === 2) {
return type === "flat"
? parseAmiStringValue(splitValues[0])
: parseAmiStringValue(splitValues[1])
}

throw new Error("This part should not be reached")
}
}

function generateUnitsSummaryAmiLevels(
amiCharts: Array<AmiChart>,
inputAmiChartLevels: Array<TAmiChartLevel>
row,
amiChartEntities: Array<AmiChart>,
amiChartString: string,
amiChartPercentagesString: string
) {
const amiCharts = amiChartString.split("/")

let amiPercentages: Array<number> = []
if (amiChartPercentagesString && typeof amiChartPercentagesString === "string") {
amiPercentages = amiChartPercentagesString
.split(",")
.map((s) => s.trim())
.map((s) => Number.parseInt(s))
} else if (amiChartPercentagesString && typeof amiChartPercentagesString === "number") {
amiPercentages = [amiChartPercentagesString]
}

const amiChartLevels: Array<DeepPartial<UnitGroupAmiLevel>> = []

for (const inputAmiChartLevel of inputAmiChartLevels) {
const amiChart = findAmiChartByName(amiCharts, inputAmiChartLevel.amiChartName)
for (const amiChartName of amiCharts) {
const amiChartEntity = findAmiChartByName(amiChartEntities, amiChartName as AmiChartNameType)
const monthlyRentDeterminationType =
inputAmiChartLevel.amiChartName === "MSHDA"
amiChartName === "MSHDA"
? MonthlyRentDeterminationType.flatRent
: MonthlyRentDeterminationType.percentageOfIncome

amiChartLevels.push({
amiChart: amiChart,
amiPercentage: inputAmiChartLevel.amiPercentage,
monthlyRentDeterminationType,
flatRentValue:
monthlyRentDeterminationType === MonthlyRentDeterminationType.flatRent
? getFlatRentValueForAmiChart(amiChart, inputAmiChartLevel.amiPercentage)
: null,
})
for (const amiPercentage of amiPercentages) {
amiChartLevels.push({
amiChart: amiChartEntity,
amiPercentage:
monthlyRentDeterminationType === MonthlyRentDeterminationType.percentageOfIncome
? getAmiValueFromColumn(row, amiPercentage, "percentage")
: null,
monthlyRentDeterminationType,
flatRentValue:
monthlyRentDeterminationType === MonthlyRentDeterminationType.flatRent
? getAmiValueFromColumn(row, amiPercentage, "flat")
: null,
})
}
}

return amiChartLevels
Expand Down Expand Up @@ -170,25 +221,25 @@ async function main() {
unitTypes.push(unitType)
}

const inputAmiChartLevels = generateAmiChartLevels(
row[HeaderConstants.AMIChart],
row[HeaderConstants.AmiChartPercentage]
)

const newUnitsSummary: DeepPartial<UnitGroup> = {
minOccupancy: row[HeaderConstants.MinOccupancy]
? row[HeaderConstants.MinOccupancy]
: null,
maxOccupancy: row[HeaderConstants.MaxOccupancy]
? row[HeaderConstants.MinOccupancy]
? row[HeaderConstants.MaxOccupancy]
: null,
totalCount: row[HeaderConstants.TotalCount] ? row[HeaderConstants.TotalCount] : null,
totalAvailable: row[HeaderConstants.TotalAvailable]
? row[HeaderConstants.TotalAvailable]
: null,
openWaitlist: getOpenWaitlistValue(row),
unitType: unitTypes,
amiLevels: generateUnitsSummaryAmiLevels(amiCharts, inputAmiChartLevels),
amiLevels: generateUnitsSummaryAmiLevels(
row,
amiCharts,
row[HeaderConstants.AMIChart],
row[HeaderConstants.AmiChartPercentage]
),
}
listing.unitGroups.push(newUnitsSummary)

Expand Down
15 changes: 15 additions & 0 deletions ...end/core/src/migration/1646908736734-make-unit-group-ami-level-ami-percentage-optional.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import {MigrationInterface, QueryRunner} from "typeorm";

export class makeUnitGroupAmiLevelAmiPercentageOptional1646908736734 implements MigrationInterface {
name = 'makeUnitGroupAmiLevelAmiPercentageOptional1646908736734'

public async up(queryRunner: QueryRunner): Promise<void> {

await queryRunner.query(`ALTER TABLE "unit_group_ami_levels" ALTER COLUMN "ami_percentage" DROP NOT NULL`);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "unit_group_ami_levels" ALTER COLUMN "ami_percentage" SET NOT NULL`);
}

}
5 changes: 3 additions & 2 deletions backend/core/src/units-summary/entities/unit-group-ami-level.entity.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ export class UnitGroupAmiLevel {
@ManyToOne(() => UnitGroup, (unitGroup: UnitGroup) => unitGroup.amiLevels)
unitGroup: UnitGroup

@Column({ type: "integer", nullable: false })
@Column({ type: "integer", nullable: true })
@Expose()
@IsOptional({ groups: [ValidationsGroupsEnum.default] })
@IsNumber({}, { groups: [ValidationsGroupsEnum.default] })
amiPercentage: number
amiPercentage: number | null

@Column({ type: "enum", enum: MonthlyRentDeterminationType, nullable: false })
@Expose()
Expand Down
Loading

0 comments on commit 7b101f8

Please sign in to comment.