Merge branch 'dev' of github.com:microsoft/community-organization-ope…

…rations-suite into bugfix/report-refresh-no-refrences

.gitignore

@@ -40,4 +40,5 @@ deploy.zip
 .env*
 
 # LogFiles
+packages/webapp/test-output.html
 *-*.log*

package.json

@@ -8,15 +8,16 @@
     "predeploy": "az account set -s Resilience",
     "clean:": "yarn workspaces foreach -pv run clean",
     "build:": "yarn workspaces foreach -pvt run build",
+    "assets:": "yarn workspaces foreach -pvt run assets",
     "test:": "yarn workspaces foreach -pv run test",
     "start:": "yarn workspaces foreach -piv run start",
     "prepare_for_deployment": "yarn build:schema && yarn build:api && yarn archive:api && yarn migrate:db:up",
     "bundle:": "yarn workspaces foreach -piv run bundle",
     "lint:": "essex lint --fix --strict",
     "git_is_clean": "essex git-is-clean",
     "prettify": "essex prettify",
-    "build_schema": "yarn build:schema",
-    "ci": "run-s build_schema lint: test: build: git_is_clean",
+    "build_schema:": "yarn build:schema",
+    "ci": "run-s build_schema: lint: assets: test: build: git_is_clean",
     "encode_firebase_config": "node encodeFirebaseConfig.mjs"
   },
   "devDependencies": {

packages/api/.gitignore

@@ -2,5 +2,4 @@ mock_data/
 seed_data/
 
 # Ignore development config
-config/development.json
-config/local.json
+config/local.json

packages/api/config/default.json

@@ -12,6 +12,9 @@
 		"pretty": true
 	},
 	"db": {
+		"automigrate": false,
+		"seedMockData": false,
+		"seedConnectionString": null,
 		"connectionString": null,
 		"database": "greenlight",
 		"usersCollection": "users",

packages/api/config/development.json

@@ -0,0 +1,8 @@
+{
+	"db": {
+		"automigrate": true,
+		"seedMockData": true,
+		"seedConnectionString": "mongodb://localhost:27017",
+		"connectionString": "mongodb://localhost:27017"
+	}
+}

packages/api/createMockData.ts

@@ -10,6 +10,7 @@ import faker from 'faker'
 import { v4 } from 'uuid'
 import { DbOrganization, DbUser, DbContact, DbAction, DbEngagement, DbTag } from './src/db/types'
 import { EngagementStatus, RoleType } from '@cbosuite/schema/dist/provider-types'
+import _ from 'lodash'
 
 const engagementStatusList: EngagementStatus[] = [
 	EngagementStatus.NotStarted,
@@ -64,7 +65,7 @@ ORG_NAMES.forEach((name) => {
 			last_name: lastName,
 			user_name: `${firstName}.${lastName}`.toLowerCase(),
 			password: bcrypt.hashSync('test', 10),
-			email: `${firstName}.${lastName}@${name}.com`.toLowerCase(),
+			email: `${userIndex === 0 ? 'admin' : `${firstName}.${lastName}`}@${name}.com`.toLowerCase(),
 			roles: [{ org_id: orgId, role_type: RoleType.User }],
 			description: `Working part-time as a ${faker.name.jobTitle()}, likes to listen to ${faker.music.genre()}.`,
 			additional_info: `Completed training(s): ${faker.name.title()}, ${faker.name.title()} and ${faker.name.title()}`,
@@ -87,6 +88,7 @@ ORG_NAMES.forEach((name) => {
 		if (!uniqueTags.includes(word)) {
 			orgTags.push({
 				id: v4(),
+				org_id: orgId,
 				label: word
 			})
 			uniqueTags.push(word)
@@ -98,7 +100,7 @@ ORG_NAMES.forEach((name) => {
 		name,
 		description: faker.lorem.paragraph(3),
 		users: orgUsers.map((u) => u.id),
-		tags: orgTags,
+		tags: orgTags.map((t) => t.label),
 		contacts: []
 	}
 
@@ -145,8 +147,21 @@ ORG_NAMES.forEach((name) => {
 			middle: faker.name.middleName(),
 			last: faker.name.lastName()
 		}
+		const genders = ['male', 'female', 'other']
 		const contact: DbContact = {
 			id: v4(),
+			demographics: {
+				gender: genders[i % genders.length],
+				gender_other: '',
+				ethnicity: 'undeclared',
+				ethnicity_other: '',
+				race: 'unknown',
+				race_other: 'unknown',
+				preferred_language: 'english',
+				preferred_language_other: '',
+				preferred_contact_method: '',
+				preferred_contact_time: ''
+			},
 			org_id: orgId,
 			first_name: fakeName.first,
 			last_name: fakeName.last,
@@ -163,6 +178,7 @@ ORG_NAMES.forEach((name) => {
 
 		const engagement: DbEngagement = {
 			id: v4(),
+			title: _.truncate(randomValue(engagementBlurbs), { length: 40 }),
 			org_id: orgId,
 			contacts: [contact.id],
 			start_date: yesterday.toISOString(),

packages/api/createSeedData.ts

@@ -37,7 +37,7 @@ function generatePassword(length = 10): string {
  */
 // Example:
 // const ORG_NAMES = ['Test']
-const ORG_NAMES = []
+const ORG_NAMES = ['Test']
 
 /**
  * List of lists of users to create

packages/api/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   mongodb_3616:
-    image: mongo:3.6.16
+    image: mongo:4.0.27
     ports:
       - '27017:27017'
     volumes:

packages/api/migrateMongo.ts

@@ -2,46 +2,39 @@
  * Copyright (c) Microsoft. All rights reserved.
  * Licensed under the MIT license. See LICENSE file in the project.
  */
-
-import { create, database, config, up, down, status, init } from 'migrate-mongo'
-import { migrateMongoConfig } from './migrate-mongo-config'
-
-config.set(migrateMongoConfig)
+/* eslint-disable @essex/adjacent-await */
+import { Migrator } from './src/components/Migrator'
+require('dotenv').config()
+const appConfig = require('config')
+const { Configuration } = require('./src/components/Configuration')
 
 /**
  * Wrapper around migrate-mongo to enable typescript support and load config api components
  */
 async function migration() {
 	const [, , command, migrationNameToCreate] = process.argv
-
-	const { db } = await database.connect()
+	console.log('launching migrator', process.env.NODE_CONFIG_ENV || process.env.NODE_ENV)
+	const migrator = new Migrator(new Configuration(appConfig))
+	await migrator.connect()
 
 	switch (command) {
 		case 'init':
-			await init()
+			await migrator.init()
 			console.log('Initialized migrate-mongo')
 			break
 		case 'create':
-			if (!migrationNameToCreate) {
-				throw new Error(
-					'No migration name provided to migrate-mongo create. Please provide a valid migration name.'
-				)
-			}
-
-			const migration = await create(migrationNameToCreate)
+			const migration = await migrator.create(migrationNameToCreate)
 			console.log('Created migration', migration)
 			break
 		case 'up':
-			const migrated = await up(db)
-			migrated.forEach((fileName) => console.log('Migrated:', fileName))
+			await migrator.up()
 			break
 		case 'down':
-			const migratedDown = await down(db)
-			migratedDown.forEach((fileName) => console.log('Migrated Down:', fileName))
+			await migrator.down()
 			break
 		case 'status':
-			const migrationStatus = await status(db)
-			migrationStatus.forEach(({ fileName, appliedAt }) => console.log(fileName, ':', appliedAt))
+			const status = await migrator.status()
+			console.log(status)
 			break
 		default:
 			console.log('No command provided to migrate-mongo')

packages/api/package.json

@@ -6,15 +6,14 @@
   "scripts": {
     "clean": "essex clean dist/ deploy.zip",
     "start": "nodemon src/index.ts",
-    "start_local": "docker-compose up -d && yarn start",
-    "start:api": "yarn start",
-    "start:api:local": "yarn start_local",
-    "build": "run-s clean compile write_deploy_package",
+    "start:api": "docker-compose up -d && NODE_CONFIG_ENV='development' yarn start",
+    "build": "run-s clean compile create_mock_data write_deploy_package",
     "build:api": "yarn build",
     "archive": "./scripts/archive.sh",
     "archive:api": "yarn archive",
     "compile": "ttsc -b tsconfig.json",
     "test": "jest",
+    "assets": "yarn create_mock_data",
     "create_mock_data": "ts-node createMockData.ts",
     "create_seed_data": "ts-node createSeedData.ts",
     "deploy:integ": "az webapp deployment source config-zip --resource-group greenlight --name greenlight-api-integ --src deploy.zip",
@@ -45,6 +44,7 @@
     "jsonwebtoken": "^8.5.1",
     "lodash": "^4.17.21",
     "make-dir": "^3.1.0",
+    "migrate-mongo": "^8.2.3",
     "mongodb": "3.6.12",
     "node-fetch": "^2.6.5",
     "nodemailer": "^6.6.5",
@@ -83,7 +83,6 @@
     "copy": "^0.3.2",
     "faker": "^5.5.3",
     "jest": "^27.2.1",
-    "migrate-mongo": "^8.2.3",
     "nodemon": "^2.0.12",
     "npm-run-all": "^4.1.5",
     "ts-node": "^10.2.1",

packages/api/src/components/AppBuilder.ts

@@ -67,12 +67,10 @@ export class AppBuilder {
 		if (locale) {
 			this.appContext.components.localization.setLocale(locale)
 		}
-
 		if (authHeader) {
 			const bearerToken = this.appContext.components.authenticator.extractBearerToken(authHeader)
 			user = await this.appContext.components.authenticator.getUser(bearerToken, userId)
 		}
-
 		return {
 			...this.appContext,
 			requestCtx: {
@@ -167,7 +165,7 @@ export class AppBuilder {
 				}
 			},
 			formatError: (err) => {
-				console.error('err in formatError', err)
+				console.error('err in formatError', err.message, err.stack)
 
 				// Don't give the specific errors to the client.
 				const message = err.message?.toLocaleLowerCase?.() || ''
@@ -183,7 +181,7 @@ export class AppBuilder {
 		})
 	}
 
-	public async start(): Promise<void> {
+	public async start(): Promise<http.Server> {
 		await this.#startupPromise
 		const app = fastify()
 		const httpServer = app.server
@@ -204,6 +202,7 @@ export class AppBuilder {
 			console.log(`🚀 Server ready at http://${host}:${port}${apolloServer.graphqlPath}`)
 			console.log(`🚀 Subscriptions ready at ws://${host}:${port}${apolloServer.graphqlPath}`)
 		})
+		return httpServer
 	}
 }
 

packages/api/src/components/AppContextProvider.ts

@@ -2,6 +2,9 @@
  * Copyright (c) Microsoft. All rights reserved.
  * Licensed under the MIT license. See LICENSE file in the project.
  */
+/* eslint-disable @essex/adjacent-await */
+import path from 'path'
+import fs from 'fs'
 import { Authenticator } from './Authenticator'
 import { Configuration } from './Configuration'
 import { Notifications } from './Notifications'
@@ -18,6 +21,7 @@ import {
 } from '~db'
 import { AsyncProvider, BuiltAppContext } from '~types'
 import nodemailer from 'nodemailer'
+import { PubSub } from 'graphql-subscriptions'
 import { AuthenticateInteractor } from '~interactors/AuthenticateInteractor'
 import { CreateEngagementInteractor } from '~interactors/CreateEngagementInteractor'
 import { AssignEngagementInteractor } from '~interactors/AssignEngagementInteractor'
@@ -44,9 +48,10 @@ import { UpdateServiceInteractor } from '~interactors/UpdateServiceInteractor'
 import { CreateContactInteractor } from '~interactors/CreateContactInteractor'
 import { UpdateTagInteractor } from '~interactors/UpdateTagInteractor'
 import { CreateServiceAnswersInteractor } from '~interactors/CreateServiceAnswersInteractor'
-import { PubSub } from 'graphql-subscriptions'
 import { DeleteServiceAnswerInteractor } from '~interactors/DeleteServiceAnswerInteractor'
 import { UpdateServiceAnswerInteractor } from '~interactors/UpdateServiceAnswerInteractor'
+import { Migrator } from './Migrator'
+
 const sgTransport = require('nodemailer-sendgrid-transport')
 
 export class AppContextProvider implements AsyncProvider<BuiltAppContext> {
@@ -59,6 +64,8 @@ export class AppContextProvider implements AsyncProvider<BuiltAppContext> {
 	public async get(): Promise<BuiltAppContext> {
 		const config = this.#config
 		const conn = new DatabaseConnector(config)
+		// Automigrate for integration testing, local development, etc.
+		await performDatabaseMigrations(config)
 		await conn.connect()
 		const userCollection = new UserCollection(conn.usersCollection)
 		const userTokenCollection = new UserTokenCollection(conn.userTokensCollection)
@@ -213,3 +220,25 @@ export class AppContextProvider implements AsyncProvider<BuiltAppContext> {
 		}
 	}
 }
+
+async function performDatabaseMigrations(config: Configuration) {
+	// This should prevent accidental seed data from accidentally being inserted into Azure environments
+	// (e.g. when a dev uses an env-var override locally)
+	const isSeedTargetStable = config.dbSeedConnectionString === config.dbConnectionString
+	if (!isSeedTargetStable) {
+		console.warn('unstable seed target, skipping DB seeding')
+	} else {
+		const migrator = new Migrator(config)
+		await migrator.connect()
+		if (config.dbAutoMigrate) {
+			await migrator.up()
+		}
+
+		if (config.dbSeedMockData) {
+			const SEED_FILE_ROOT = path.join(__dirname, '../../mock_data')
+			const seedFiles = fs.readdirSync(SEED_FILE_ROOT).map((f) => path.join(SEED_FILE_ROOT, f))
+			// Seed the mock data fresh (delete old data)
+			await migrator.seed(seedFiles, true)
+		}
+	}
+}

packages/api/src/components/Configuration.ts

@@ -86,6 +86,18 @@ export class Configuration {
 		return this.c.get<string>('db.servicesCollection')
 	}
 
+	public get dbAutoMigrate(): boolean {
+		return this.c.get<boolean>('db.automigrate')
+	}
+
+	public get dbSeedMockData(): boolean {
+		return this.c.get<boolean>('db.seedMockData')
+	}
+
+	public get dbSeedConnectionString(): string {
+		return this.c.get<string>('db.seedConnectionString')
+	}
+
 	public get defaultPageOffset(): number {
 		return this.c.get<number>('constants.defaultPageOffset')
 	}

packages/api/src/components/DatabaseConnector.ts

@@ -12,7 +12,9 @@ export class DatabaseConnector {
 
 	public constructor(config: Configuration) {
 		this.#config = config
-		this.#client = new MongoClient(this.#config.dbConnectionString)
+		this.#client = new MongoClient(this.#config.dbConnectionString, {
+			useUnifiedTopology: true
+		})
 	}
 
 	public async connect(): Promise<void> {