feat(deps): update mongoose to v6.0.13, update mongoose dependencies …

…likewise (#3093)

* fix(deps): bump mongoose from 5.13.5 to 6.0.12

Bumps [mongoose](https://github.com/Automattic/mongoose) from 5.13.5 to 6.0.12.
- [Release notes](https://github.com/Automattic/mongoose/releases)
- [Changelog](https://github.com/Automattic/mongoose/blob/master/CHANGELOG.md)
- [Commits](Automattic/mongoose@5.13.5...6.0.12)

---
updated-dependencies:
- dependency-name: mongoose
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <[email protected]>

* fix(deps): update connect-mongo to v4.6.0

* chore(dev-deps): update @types/mongodb to v4.0.7

* fix(FormModel): update schema validators to 6.x syntax

* fix(config): use ConnectOptions type declaration from mongoose@6

* fix(agency): correct types to use mongoose@6 types

* fix(config): remove unsupported options (now default)

see https://mongoosejs.com/docs/migrating_to_6.html#no-more-deprecation-warning-options

* fix: misc type errors due to mongoose@6

* chore(deps): add mongodb dependency

* fix(UserModel): update schema to fit new syntax and MongoServerError

* fix(MyInfoModel): use Mixed type instead of undocumented Object type

* fix: add typecasts to get around excessively deep TS error

* fix(FormLogicModel): add typecast for schema Schema.Types.Mixed

* fix(agency): further update types

* fix(packages): update bson-ext to v4.0.2, and mongoose to v6.0.13

* fix: update mongodb-memory-server-core to 8.0.2

required since bson-ext won't work with old memory core versions

* fix(tests): update instantiation of test db

* fix(tests): update mongoose toEqual matchers to toMatchObject matchers

somewhere along the way mongoose changed the shape of their returned objects, causing toEqual match to fail even if they serialize to the same string.

* fix(e2e): correctly return promise of mongoose.createConnection

v6 change

* fix(FormModel): move esrvcId schema prop to be above status

Mongoose now saves objects with keys in the order the keys are specified in the schema, not in the user-defined object.

* fix(spcp.service.spec): remove unused dbhandler

* feat: remove mockingoose package

was underutilitized and broke during mongoose v6 migration, so deleting it

* fix(test): fix failing tests due to immutable create field

mongoose v6 sets the createdAt value to be immutable.

See https://mongoosejs.com/docs/migrating_to_6.html#immutable-createdat

* fix(test): remaining toEqual -> toMatchObject matcher changes

* fix(test): replace esrvcId values with more obvious values

* fix(test): freeze date in example related tests

fixes issue where expect.anything() is not matching with whatever mongoose object is returning...

* fix(test): remove flakeyness of returned examples by sorting in result

* fix: update package-lock for snyk reported vulns

* fix(deps): update mongodb-memory-server-core to 8.0.4

* fix: ignore excessively deep type-instantiation ts type error

lots of issues on mongoose repo which seems semi related but not really related, ignoring for now till we overhaul the types mongoose uses

* fix: assign table column discrim before setting field discrim

Mongoose now saves objects with keys in the order the keys are specified in the schema, meaning the table column discriminator would not have been applied at the time the table field was assigned as a discriminator to the form's form fields.

This resulted in a bug where table columns did not have a discriminated schema causing uncaught errors to be thrown when a table field was submitted, such as

"UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'selectedValidation' of undefined" as the field did not contain the necessary discriminated column schema props due to not being assigned prior to assigning the TableFieldSchema discriminator.

See https://mongoosejs.com/docs/migrating_to_6.html#schema-defined-document-key-order

* test: add success test case for submissions containing table fields

this broke in mongoose v6 upgrade before being fixed, adding a test to ensure that this does not break in the future

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

package.json

@@ -83,12 +83,12 @@
     "body-parser": "^1.19.1",
     "bootstrap": "3.4.1",
     "boxicons": "1.8.0",
-    "bson-ext": "^2.0.6",
+    "bson-ext": "^4.0.2",
     "busboy": "^0.3.1",
     "celebrate": "^15.0.0",
     "compression": "~1.7.2",
     "connect-datadog": "0.0.9",
-    "connect-mongo": "^4.4.1",
+    "connect-mongo": "^4.6.0",
     "convict": "^6.2.1",
     "convict-format-with-validator": "^6.2.0",
     "cookie-parser": "~1.4.6",
@@ -117,8 +117,9 @@
     "libphonenumber-js": "^1.9.44",
     "lodash": "^4.17.21",
     "moment-timezone": "0.5.34",
+    "mongodb": "^4.1.4",
     "mongodb-uri": "^0.9.7",
-    "mongoose": "^5.13.5",
+    "mongoose": "^6.0.13",
     "multiparty": ">=4.2.2",
     "neverthrow": "^4.3.1",
     "ng-infinite-scroll": "^1.3.0",
@@ -176,7 +177,6 @@
     "@types/jest": "^27.0.3",
     "@types/json-stringify-safe": "^5.0.0",
     "@types/lodash": "^4.14.178",
-    "@types/mongodb": "^3.6.20",
     "@types/mongodb-uri": "^0.9.1",
     "@types/node": "^14.18.0",
     "@types/nodemailer": "^6.4.4",
@@ -223,8 +223,7 @@
     "maildev": "^1.1.0",
     "mini-css-extract-plugin": "^0.5.0",
     "mockdate": "^3.0.5",
-    "mockingoose": "^2.13.2",
-    "mongodb-memory-server-core": "^6.9.6",
+    "mongodb-memory-server-core": "^8.0.4",
     "ngrok": "^4.2.2",
     "optimize-css-assets-webpack-plugin": "^5.0.8",
     "prettier": "^2.5.1",

src/app/config/config.ts

@@ -110,14 +110,6 @@ const dbConfig: DbConfig = {
     pass: '',
     // Only create indexes in dev env to avoid adverse production impact.
     autoIndex: isDev,
-    // Avoid using deprecated URL string parser in MongoDB driver
-    useNewUrlParser: true,
-    useUnifiedTopology: true,
-    // Avoid using deprecated collection.ensureIndex internally
-    useCreateIndex: true,
-    // upgrade to mongo driver's native findOneAndUpdate function instead of
-    // findAndModify.
-    useFindAndModify: false,
     promiseLibrary: global.Promise,
   },
 }
@@ -180,7 +172,7 @@ const cookieSettings: SessionOptions['cookie'] = {
 /**
  * Fetches AWS credentials
  */
-const configureAws = async () => {
+const configureAws = async (): Promise<void> => {
   if (!isDev) {
     const getCredentials = () => {
       return new Promise<void>((resolve, reject) => {

src/app/loaders/express/session.ts

@@ -2,6 +2,7 @@ import MongoStore from 'connect-mongo'
 import cookieParser from 'cookie-parser'
 import { RequestHandler } from 'express'
 import session from 'express-session'
+import { MongoClient } from 'mongodb'
 import { Connection } from 'mongoose'
 
 import config from '../../config/config'
@@ -15,7 +16,7 @@ const sessionMiddlewares = (connection: Connection): RequestHandler[] => {
     cookie: config.cookieSettings,
     name: 'connect.sid',
     store: MongoStore.create({
-      client: connection.getClient(),
+      client: connection.getClient() as MongoClient,
     }),
   })
 

src/app/loaders/mongoose.ts

@@ -28,7 +28,7 @@ export default async (): Promise<Connection> => {
       process.exit(1)
     }
 
-    const mongod = new MongoMemoryServer({
+    const mongod = await MongoMemoryServer.create({
       binary: { version: String(process.env.MONGO_BINARY_VERSION) },
       instance: {
         port: 3000,
@@ -38,7 +38,7 @@ export default async (): Promise<Connection> => {
     })
 
     // Store the uri to connect to later on
-    config.db.uri = await mongod.getConnectionString()
+    config.db.uri = mongod.getUri()
   }
 
   // Actually connect to the database

src/app/models/__tests__/admin_verification.server.model.spec.ts

@@ -1,4 +1,3 @@
-import { ObjectID } from 'bson'
 import mongoose from 'mongoose'
 
 import getAdminVerificationModel from 'src/app/models/admin_verification.server.model'
@@ -18,7 +17,7 @@ describe('AdminVerification Model', () => {
 
   describe('Schema', () => {
     const DEFAULT_PARAMS: IAdminVerification = {
-      admin: new ObjectID(),
+      admin: new mongoose.Types.ObjectId(),
       expireAt: new Date(),
       hashedContact: 'mockHashedContact',
       hashedOtp: 'mockHashedOtp',
@@ -57,13 +56,11 @@ describe('AdminVerification Model', () => {
       expect(actual._id).toBeDefined()
       expect(actual.createdAt).toBeInstanceOf(Date)
       expect(actual.updatedAt).toBeInstanceOf(Date)
-      expect(actual).toEqual(
-        expect.objectContaining({
-          ...customParams,
-          // Add defaults that has not been overridden.
-          numOtpSent: 0,
-        }),
-      )
+      expect(actual).toMatchObject({
+        ...customParams,
+        // Add defaults that has not been overridden.
+        numOtpSent: 0,
+      })
     })
 
     it('should throw validation error on missing admin', async () => {
@@ -133,7 +130,7 @@ describe('AdminVerification Model', () => {
       it('should create successfully when document does not exist', async () => {
         // Arrange
         const params: UpsertOtpParams = {
-          admin: new ObjectID(),
+          admin: new mongoose.Types.ObjectId(),
           expireAt: new Date(),
           hashedContact: 'mockHashedContact',
           hashedOtp: 'mockHashedOtp',
@@ -149,13 +146,13 @@ describe('AdminVerification Model', () => {
         const expected = { ...params, numOtpAttempts: 0, numOtpSent: 1 }
         // Should now have one document.
         await expect(AdminVerification.countDocuments()).resolves.toEqual(1)
-        expect(actual).toEqual(expect.objectContaining(expected))
+        expect(actual).toMatchObject(expected)
       })
 
       it('should update successfully when a document already exists', async () => {
         // Arrange
         // Insert mock document into collection.
-        const adminId = new ObjectID()
+        const adminId = new mongoose.Types.ObjectId()
         const oldExpireAt = new Date()
         const newExpireAt = new Date(Date.now() + 9000000)
         const oldNumOtpSent = 3
@@ -190,14 +187,14 @@ describe('AdminVerification Model', () => {
         }
         // Should still only have one document.
         await expect(AdminVerification.countDocuments()).resolves.toEqual(1)
-        expect(actual).toEqual(expect.objectContaining(expected))
+        expect(actual).toMatchObject(expected)
       })
 
       it('should throw error if validation fails due to invalid upsert parameters', async () => {
         // Arrange
         const invalidParams: UpsertOtpParams = {
           // Invalid admin parameter.
-          admin: undefined,
+          admin: null,
           expireAt: new Date(),
           hashedContact: 'mockHashedContact',
           hashedOtp: 'mockHashedOtp',
@@ -219,7 +216,7 @@ describe('AdminVerification Model', () => {
       it('should increment successfully', async () => {
         // Arrange
         // Insert mock document into collection.
-        const adminId = new ObjectID()
+        const adminId = new mongoose.Types.ObjectId()
         const initialOtpAttempts = 5
         const adminVerificationParams = {
           admin: adminId,
@@ -240,19 +237,17 @@ describe('AdminVerification Model', () => {
         // Assert
         // Exactly the same as initial params, but with numOtpAttempts
         // incremented by 1.
-        await expect(actualPromise).resolves.toEqual(
-          expect.objectContaining({
-            ...adminVerificationParams,
-            numOtpAttempts: initialOtpAttempts + 1,
-          }),
-        )
+        await expect(actualPromise).resolves.toMatchObject({
+          ...adminVerificationParams,
+          numOtpAttempts: initialOtpAttempts + 1,
+        })
       })
 
       it('should return null if document cannot be retrieved', async () => {
         // Arrange
         // Should have no documents yet.
         await expect(AdminVerification.countDocuments()).resolves.toEqual(0)
-        const freshAdminId = new ObjectID()
+        const freshAdminId = new mongoose.Types.ObjectId()
 
         // Act
         const actualPromise =

src/app/models/__tests__/encrypt-submission.server.model.spec.ts

@@ -53,7 +53,7 @@ describe('Encrypt Submission Model', () => {
             .tz('Asia/Singapore')
             .format('Do MMM YYYY, h:mm:ss a'),
         }
-        expect(result).toEqual(expected)
+        expect(result).toMatchObject(expected)
       })
 
       it('should return null when submission is of SubmissionType.Email', async () => {
@@ -137,7 +137,7 @@ describe('Encrypt Submission Model', () => {
             }))
             .reverse(),
         }
-        expect(actual).toEqual(expected)
+        expect(actual).toMatchObject(expected)
       })
 
       it('should return offset metadata with correct count when page number is provided', async () => {
@@ -179,7 +179,7 @@ describe('Encrypt Submission Model', () => {
             },
           ],
         }
-        expect(actual).toEqual(expected)
+        expect(actual).toMatchObject(expected)
       })
 
       it('should return offset metadata with correct count when page size is provided', async () => {
@@ -221,7 +221,7 @@ describe('Encrypt Submission Model', () => {
             },
           ],
         }
-        expect(actual).toEqual(expected)
+        expect(actual).toMatchObject(expected)
       })
 
       it('should return empty metadata array when given page has no metadata', async () => {
@@ -254,7 +254,7 @@ describe('Encrypt Submission Model', () => {
           // show
           metadata: [],
         }
-        expect(actual).toEqual(expected)
+        expect(actual).toMatchObject(expected)
       })
 
       it('should return empty metadata array when formId has no metadata', async () => {
@@ -272,7 +272,7 @@ describe('Encrypt Submission Model', () => {
           // show
           metadata: [],
         }
-        expect(actual).toEqual(expected)
+        expect(actual).toMatchObject(expected)
       })
     })
 
@@ -284,6 +284,7 @@ describe('Encrypt Submission Model', () => {
           submissionType: SubmissionType.Encrypt,
           form: validFormId,
           encryptedContent: 'mock encrypted content abc',
+          verifiedContent: 'mock verified content',
           version: 3,
         })
         const expectedSubmission = pick(
@@ -309,7 +310,7 @@ describe('Encrypt Submission Model', () => {
           retrievedSubmissions.push(submission)
         }
         // Cursor stream should contain only that single submission.
-        expect(retrievedSubmissions).toEqual([expectedSubmission])
+        expect(retrievedSubmissions).toMatchObject([expectedSubmission])
       })
 
       it('should return cursor even if no submissions are found', async () => {
@@ -329,7 +330,7 @@ describe('Encrypt Submission Model', () => {
           retrievedSubmissions.push(submission)
         }
         // Cursor stream should return nothing.
-        expect(retrievedSubmissions).toEqual([])
+        expect(retrievedSubmissions).toMatchObject([])
       })
     })
 
@@ -362,7 +363,7 @@ describe('Encrypt Submission Model', () => {
           'version',
         )
         expect(actual).not.toBeNull()
-        expect(actual?.toJSON()).toEqual(expected)
+        expect(actual?.toJSON()).toMatchObject(expected)
       })
 
       it('should return null when submission id does not exist', async () => {