feat(deps): update mongoose to 6 (#6832)

* chore: bump mongoose to 6.12

* fix: type mismatch on cursorquery, objectid

* fix: required and validator shape in user and form model

* fix: use try catch block for mongoose connect loader

* fix: add missing goLinkSuffix property in IForm

* fix: overwrite typing of IPaymentSchema for mongoose Id

* fix: querycursor import in submission.ts

* fix: removed undefined in AgencySchema causing schema methods to be never

* fix: schema typings in form logic and sms types

* fix: replace ConnectionOptions with ConnectOptions

* fix: remove deprecated connect options

* fix: objectid imports

* chore: remove mongodb package, upgrade connect-mongo to 4.6.0

* chore: replace mongoerror import reference

* fix: typescript errors

* chore: change nModified to modifiedCount

* fix: typeerror on form_statistics_total.server.model

* fix: typeerror on bouncemodel

* fix: form server model, only fixed affected methods

* fix: mongoerror is now mongoservererror

* fix: execPopulate no longer required to be called

* fix: verification.model

* fix: replace enforcedocument to hydrateddocument

* fix: duplicateFormFieldByIdAndIndex

* fix: util/logic.ts type inference errors

* refactor: swap bson with bson-ext

* chore: update test db configs

* refactor: remove bson reference

* refactor: gloal replacement of bson

* chore: remove execPopulate

* fix: void promise returns

* fix: remove bson reference with mongoose

* fix: replace more bson on admin-forms.form.routes.specs

* fix: test failures on admin-forms.submissions.routes.spec

* fix: test failures on src/app/models

* fix: test failures on src/app/modules/payments

* fix: test failures on src/app/modules/issues

* fix: form model always defaulting to private mode on save

* chore: remove bson in jest-db

* fix: lint and type errors in loader

* chore: remove unused typedef

* chore: remove commented out BSON code

* fix: playwright mongod init failure

* refactor: rename constants

* fix: mongoose loader

* fix: removed form model business field to be required

* fix: e2e email-submission tests

* fix: make mongoose.connect to be async

* fix: add specific db connection uri

* fix: restore missing _id in table column

* fix: workspace model support

* chore: add verbose playwight reporting

* chore: update bson-ext to bson for workspace-related code

* fix: dburi should not be random

* fix: drop form collection instead of the whole db

* chore: update hosting port from 5000 to 5001 for playwright

* fix: replace bson-ext with bson

* Revert "chore: update hosting port from 5000 to 5001 for playwright"

This reverts commit e5db025.

* fix: workspace spec, exists returning null and doc instead of boolean

* chore: remove commented code

* chore: remove confusing statement

* chore: remove unneeded comments

* chore: remove duplicated typedef

* Revert "chore: remove duplicated typedef"

This reverts commit 66f638f.

---------

Co-authored-by: foochifa <[email protected]>

__tests__/e2e/email-submission.spec.ts

@@ -48,7 +48,7 @@ test.describe('Email form submission', () => {
   })
   test.afterAll(async () => {
     // Clean up db
-    db.models = {}
+    await db.dropCollection(Form.collection.name)
     await db.close()
   })
 

__tests__/e2e/encrypt-submission.spec.ts

@@ -51,7 +51,7 @@ test.describe('Storage form submission', () => {
   })
   test.afterAll(async () => {
     // Clean up db
-    db.models = {}
+    await db.dropCollection(Form.collection.name)
     await db.close()
   })
 

__tests__/e2e/setup/globalSetup.ts

@@ -14,8 +14,9 @@ async function globalSetup(): Promise<void> {
         checkMD5: true,
       },
     })
+    await mongod.start()
 
-    const uri = await mongod.getUri(true)
+    const uri = mongod.getUri()
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     ;(global as any).__MONGOINSTANCE = mongod
     process.env.MONGO_URI = `${uri.slice(0, uri.lastIndexOf('/'))}/${
@@ -27,14 +28,6 @@ async function globalSetup(): Promise<void> {
 
   // The following is to make sure the database is clean before an test starts
   await mongoose.connect(process.env.MONGO_URI, {
-    // Avoid using deprecated URL string parser in MongoDB driver
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-    // Avoid using deprecated collection.ensureIndex internally
-    useCreateIndex: true,
-    // upgrade to mongo driver's native findOneAndUpdate function instead of
-    // findAndModify.
-    useFindAndModify: false,
     promiseLibrary: global.Promise,
   })
   await mongoose.disconnect()

__tests__/e2e/utils/database.ts

@@ -1,7 +1,5 @@
 import mongoose, { Model } from 'mongoose'
 
-const dbUri = 'mongodb://127.0.0.1:3000/formsg'
-
 // Get absolute path of file
 const spec = (path: string): any => {
   const fullPath = `${process.env.PWD}/${path}`
@@ -11,11 +9,10 @@ const spec = (path: string): any => {
 /**
  * Connects to mongo-memory-server instance.
  */
-export const makeMongooseFixtures = async (): Promise<mongoose.Connection> => {
-  const connection = await mongoose.createConnection(dbUri, {
-    reconnectTries: 5,
-    useNewUrlParser: true,
-  })
+export const makeMongooseFixtures = (): Promise<mongoose.Connection> => {
+  const dbUri = 'mongodb://127.0.0.1:3000/test' // TODO: hardcoding uri as the port and path are fixed and doesn't respect values in __tests__/e2e/setup/setupConfig.ts
+
+  const connection = mongoose.createConnection(dbUri).asPromise()
   return connection
 }
 

__tests__/setup/database.js

@@ -8,7 +8,6 @@ class MemoryDatabaseServer {
         checkMD5: true,
       },
       instance: {},
-      autoStart: false,
     })
   }
 
@@ -21,7 +20,7 @@ class MemoryDatabaseServer {
   }
 
   getConnectionString() {
-    return this.mongod.getUri(true)
+    return this.mongod.getUri()
   }
 }
 

__tests__/unit/backend/helpers/jest-db.ts

@@ -1,6 +1,5 @@
 import MemoryDatabaseServer from '__tests__/setup/database'
-import { ObjectID } from 'bson'
-import mongoose from 'mongoose'
+import mongoose, { Schema, Types } from 'mongoose'
 import { FormResponseMode } from 'shared/types'
 
 import getAgencyModel from 'src/app/models/agency.server.model'
@@ -31,14 +30,9 @@ import {
  * Connect to the in-memory database
  */
 const connect = async (): Promise<typeof mongoose> => {
-  const dbUrl = await MemoryDatabaseServer.getConnectionString()
-
-  const conn = await mongoose.connect(dbUrl, {
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-    useCreateIndex: true,
-    useFindAndModify: false,
-  })
+  await MemoryDatabaseServer.start()
+  const dbUrl = MemoryDatabaseServer.getConnectionString()
+  const conn = await mongoose.connect(dbUrl)
   return conn
 }
 
@@ -91,8 +85,8 @@ const insertUser = async ({
   mailName = 'test',
   apiToken,
 }: {
-  agencyId: ObjectID
-  userId?: ObjectID
+  agencyId: Schema.Types.ObjectId
+  userId?: Schema.Types.ObjectId
   mailName?: string
   mailDomain?: string
   apiToken?: UserApiToken
@@ -122,7 +116,7 @@ const insertFormCollectionReqs = async ({
   betaFlags,
   apiToken,
 }: {
-  userId?: ObjectID
+  userId?: Schema.Types.ObjectId
   mailName?: string
   mailDomain?: string
   shortName?: string
@@ -139,7 +133,7 @@ const insertFormCollectionReqs = async ({
 
   const user = await User.create({
     email: `${mailName}@${mailDomain}`,
-    _id: userId ?? new ObjectID(),
+    _id: userId ?? new Types.ObjectId(),
     agency: agency._id,
     flags,
     betaFlags,
@@ -157,8 +151,8 @@ const insertEmailForm = async ({
   shortName = 'govtest',
   formOptions = {},
 }: {
-  formId?: ObjectID
-  userId?: ObjectID
+  formId?: Schema.Types.ObjectId
+  userId?: Schema.Types.ObjectId
   mailName?: string
   mailDomain?: string
   shortName?: string
@@ -201,8 +195,8 @@ const insertEncryptForm = async ({
   formOptions = {},
   userBetaFlags,
 }: {
-  formId?: ObjectID
-  userId?: ObjectID
+  formId?: Schema.Types.ObjectId
+  userId?: Schema.Types.ObjectId
   mailName?: string
   mailDomain?: string
   shortName?: string
@@ -248,8 +242,8 @@ const insertFormWithMsgSrvcName = async ({
   formOptions = {},
   msgSrvcName = 'mockMsgSrvcname',
 }: {
-  formId?: ObjectID
-  userId?: ObjectID
+  formId?: Schema.Types.ObjectId
+  userId?: Schema.Types.ObjectId
   mailName?: string
   mailDomain?: string
   shortName?: string
@@ -294,7 +288,7 @@ const insertFormSubmission = async ({
   version = '1',
   encryptedContent = 'encryptedContent',
 }: {
-  formId?: ObjectID
+  formId?: Schema.Types.ObjectId
   submissionType?: string
   version?: string
   encryptedContent?: string
@@ -320,8 +314,8 @@ const insertFormFeedback = async ({
   rating = '5',
   comment = 'FormSG rocks!',
 }: {
-  formId?: ObjectID
-  submissionId?: ObjectID
+  formId?: Schema.Types.ObjectId
+  submissionId?: Schema.Types.ObjectId
   rating?: string
   comment?: string
 } = {}): Promise<{