From 2d941f3b014ae4cf3f6bd5dd7bc3e9d778fb4a41 Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Tue, 18 Aug 2020 12:46:12 +0800 Subject: [PATCH 01/27] refactor: convert MailService to a class based Typescript implementation (#76) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: migrate mail service to Typescript * refactor(MailService): convert to class based implementation * test: update tests to use class based MailService * refactor: add defaults to MailService instantiation * feat(MailService): remove mail retries from sendNodeMail * feat: remove references to retrying of email sending * test: EmailSubmissionsController tests now use new MailService params * test(MailService): add sendNodeMail tests * feat: add more guards in sendNodeMail * feat(MailService): add param guards for sendVerificationOtp * test(MailService): add tests for sendVerificationOtp * test: remove fdescribe * test(SmsService): fix non-awaited expectAsync * feat: inline various mail related controller functions into MailService + add constants folder (#94) * refactor: split `utils/constants` into new `constants` directory * feat(MailService): add sendLoginOtp function * feat(MailService): add sendSubmissionToAdmin function * feat: remove use of config.mail.mailer.from and add comments MailService will be manually constructing the full "from" string since the class already has the necessary components needed. This removes the need to have a weirdly nested `config.mail.mailer.from` in the config and only requires `config.mail.mailFrom`. * feat(defaults): add back mail config comments * feat(MailService): add sendAutoReplyEmail function * refactor(SubmissionsController): use MailService.sendAutoReplyEmails * feat(MailService): make MailService.sendNodeMail private * feat(MailService): add email validation to sendSubmissionToAdmin * test(EmailSubmission): update tests to use new MailService function * refactor: use private modifier for sendNodeMail instead of # This allows tests to spy on the base function. Can probably be changed once we use Typescript for tests too * test(MailService): add constructor tests * test(MailService): add sendLoginOtp tests * fix: correct email guard if recipient mail is array * refactor(MailService): remove redundant email validation All email validations is now performed in the base `sendNodeMail` function * test(MailService): add sendSubmissionToAdmin tests * test(MailService): add sendAutoReplyEmail tests * refactor: move otp-email template generation to util Instead of using ejs to render a HTML, the html is just hardcoded into a new utility function so the generation of html will not rely on ejs anymore. * refactor: extract verification otp mail html to utility function * feat: silence winston logger during tests * test(MailService): migrate tests to TypeScript * test(FormField): fix flakiness when running multiple mongoose tests Flakiness is caused by calling `connection.dropDatabase()`, and since jest runs tests in parallel, one test might drop databases needed in other tests. * test: rename and simplify preloadCollections function * feat: set isDev to true once node environment is not prod * feat: silence winston logger in test node environment for jest * refactor: use # modifier for sendNodeMail * feat: add isToFieldValid function to mail utility for validation * test: add additional email validation tests * feat(MailService): add function to generate submission to admin email * test(MailService): update tests for sendSubmissionToAdmin * feat(MailService): add sendAutoReplyEmails to handle mail generation * chore: add ts-mock-imports to mock test imports * test(MailService): update sendAutoReplyEmails tests * test(EmailSubmissionsController): update tests that calls MailService * feat(MailService): remove redundant param from sendSubmissionToAdmin * fix: update package-lock.json * test(EmailSubmissionsController): update tests that calls MailService * chore(MailService): add TODO for `any` typing --- docs/DEPLOYMENT_SETUP.md | 33 +- package-lock.json | 36 +- package.json | 5 +- src/app/constants/filesize.ts | 2 + src/app/constants/mail.ts | 23 + .../authentication.server.controller.js | 50 +- .../email-submissions.server.controller.js | 83 +-- .../submissions.server.controller.js | 126 +--- src/app/models/form.server.model.ts | 2 +- src/app/services/mail.service.js | 141 ----- src/app/services/mail.service.ts | 439 ++++++++++++++ src/app/services/verification.service.js | 4 +- src/app/utils/autoreply-pdf.js | 89 --- src/app/utils/constants.js | 33 - src/app/utils/mail.ts | 125 ++++ src/app/utils/sns.js | 13 +- src/config/config.ts | 18 - src/config/defaults.ts | 6 +- src/loaders/express/session.ts | 2 +- ...mail-submissions.server.controller.spec.js | 99 +-- .../verification.server.controller.spec.js | 11 +- .../backend/services/mail.service.spec.ts | 572 ++++++++++++++++++ .../unit/backend/services/sms.service.spec.js | 4 +- tsconfig.json | 2 +- 24 files changed, 1296 insertions(+), 622 deletions(-) create mode 100644 src/app/constants/filesize.ts create mode 100644 src/app/constants/mail.ts delete mode 100644 src/app/services/mail.service.js create mode 100644 src/app/services/mail.service.ts delete mode 100644 src/app/utils/autoreply-pdf.js delete mode 100644 src/app/utils/constants.js create mode 100644 src/app/utils/mail.ts create mode 100644 tests/unit/backend/services/mail.service.spec.ts diff --git a/docs/DEPLOYMENT_SETUP.md b/docs/DEPLOYMENT_SETUP.md index c023f1e16b..d2fa808ee6 100644 --- a/docs/DEPLOYMENT_SETUP.md +++ b/docs/DEPLOYMENT_SETUP.md @@ -136,24 +136,21 @@ The following env variables are set in Travis: #### Email and Nodemailer -| Variable | Description | -| :------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SES_HOST` | SMTP hostname. | -| `SES_PORT` | SMTP port number. | -| `SES_USER` | SMTP username. | -| `SES_PASS` | SMTP password. | -| `SES_MAX_MESSAGES` | Nodemailer configuration. Connection removed and new one created when this limit is reached. This helps to keep the connection up-to-date for long-running email messaging. Defaults to `100`. | -| `SES_POOL` | Connection pool to send email in parallel to the SMTP server. Defaults to `38`. | -| `SES_RATE` | Maximum email to send per second, or per `rateDelta` if supplied. | -| `SES_RATEDELTA` | Defines the time measuring period in milliseconds for rate limiting. Defaults to `1000`. | -| `MAIL_FROM` | Sender email address. Defaults to `'donotreply@mail.form.gov.sg'`. | -| `MAIL_RETRY_DURATION` | Base duration for retries, in milliseconds. Defaults to `60000`. | -| `MAIL_RETRY_COUNT` | Maximum number of resends for emails. Defaults to `2`. | -| `MAIL_MAX_RETRY_DURATION` | The maximum duration to wait before trying again, in milliseconds. Defaults to `4800000`. | -| `MAIL_SOCKET_TIMEOUT` | Milliseconds of inactivity to allow before killing a connection. This helps to keep the connection up-to-date for long-running email messaging. Defaults to `600000`. | -| `MAIL_LOGGER` | If set to true then logs to console. If value is not set or is false then nothing is logged. | -| `MAIL_DEBUG` | If set to `true`, then logs SMTP traffic, otherwise logs only transaction events. | -| `CHROMIUM_BIN` | Filepath to chromium binary. Required for email autoreply PDF generation with Puppeteer. | +| Variable | Description | +| :-------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SES_HOST` | SMTP hostname. | +| `SES_PORT` | SMTP port number. | +| `SES_USER` | SMTP username. | +| `SES_PASS` | SMTP password. | +| `SES_MAX_MESSAGES` | Nodemailer configuration. Connection removed and new one created when this limit is reached. This helps to keep the connection up-to-date for long-running email messaging. Defaults to `100`. | +| `SES_POOL` | Connection pool to send email in parallel to the SMTP server. Defaults to `38`. | +| `SES_RATE` | Maximum email to send per second, or per `rateDelta` if supplied. | +| `SES_RATEDELTA` | Defines the time measuring period in milliseconds for rate limiting. Defaults to `1000`. | +| `MAIL_FROM` | Sender email address. Defaults to `'donotreply@mail.form.gov.sg'`. | | +| `MAIL_SOCKET_TIMEOUT` | Milliseconds of inactivity to allow before killing a connection. This helps to keep the connection up-to-date for long-running email messaging. Defaults to `600000`. | +| `MAIL_LOGGER` | If set to true then logs to console. If value is not set or is false then nothing is logged. | +| `MAIL_DEBUG` | If set to `true`, then logs SMTP traffic, otherwise logs only transaction events. | +| `CHROMIUM_BIN` | Filepath to chromium binary. Required for email autoreply PDF generation with Puppeteer. | ### Additional Features diff --git a/package-lock.json b/package-lock.json index 684f0fb926..09bb88e9b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4699,6 +4699,12 @@ "integrity": "sha512-EGlKlgMhnLt/cM4DbUSafFdrkeJoC9Mvnj0PUCU7tFmTjMjNRT957kXCx0wYm3JuEq4o4ZsS5vG+NlkM2DMd2A==", "dev": true }, + "@types/ejs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/ejs/-/ejs-3.0.4.tgz", + "integrity": "sha512-ZxnwyBGO4KX/82AsFHTX82eMw0PsoBcIngEat+zx0y+3yxoNDJucAihg9nAcrc+g4Cwiv/4WcWsX4oiy0ySrRQ==", + "dev": true + }, "@types/error-stack-parser": { "version": "1.3.18", "resolved": "https://registry.npmjs.org/@types/error-stack-parser/-/error-stack-parser-1.3.18.tgz", @@ -5022,6 +5028,24 @@ "integrity": "sha512-IkVfat549ggtkZUthUzEX49562eGikhSYeVGX97SkMFn+sTZrgRewXjQ4tPKFPCykZHkX1Zfd9OoELGqKU2jJA==", "dev": true }, + "@types/puppeteer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-3.0.1.tgz", + "integrity": "sha512-t03eNKCvWJXhQ8wkc5C6GYuSqMEdKLOX0GLMGtks25YZr38wKZlKTwGM/BoAPVtdysX7Bb9tdwrDS1+NrW3RRA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/puppeteer-core": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/puppeteer-core/-/puppeteer-core-2.0.0.tgz", + "integrity": "sha512-JvoEb7KgEkUet009ZDrtpUER3hheXoHgQByuYpJZ5WWT7LWwMH+0NTqGQXGgoOKzs+G5NA1T4DZwXK79Bhnejw==", + "dev": true, + "requires": { + "@types/puppeteer": "*" + } + }, "@types/q": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz", @@ -17870,9 +17894,9 @@ "dev": true }, "moment-timezone": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.13.tgz", - "integrity": "sha1-mc5cfYJyYusPH3AgRBd/YHRde5A=", + "version": "0.5.31", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.31.tgz", + "integrity": "sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==", "requires": { "moment": ">= 2.9.0" } @@ -24483,6 +24507,12 @@ } } }, + "ts-mock-imports": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/ts-mock-imports/-/ts-mock-imports-1.3.0.tgz", + "integrity": "sha512-cCrVcRYsp84eDvPict0ZZD/D7ppQ0/JSx4ve6aEU8DjlsaWRJWV6ADMovp2sCuh6pZcduLFoIYhKTDU2LARo7Q==", + "dev": true + }, "ts-node": { "version": "8.10.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.10.2.tgz", diff --git a/package.json b/package.json index a9f17c0447..ae089deb38 100644 --- a/package.json +++ b/package.json @@ -128,7 +128,7 @@ "libphonenumber-js": "^1.7.55", "lodash": "^4.17.19", "mobile-detect": "^1.4.2", - "moment-timezone": "0.5.13", + "moment-timezone": "0.5.31", "mongoose": "^5.9.10", "multiparty": ">=4.1.3", "ng-infinite-scroll": "^1.3.0", @@ -167,6 +167,7 @@ "@types/compression": "^1.7.0", "@types/convict": "^5.2.1", "@types/cookie-parser": "^1.4.2", + "@types/ejs": "^3.0.4", "@types/express": "^4.17.6", "@types/express-session": "^1.17.0", "@types/has-ansi": "^3.0.0", @@ -177,6 +178,7 @@ "@types/node": "^14.0.13", "@types/nodemailer": "^6.4.0", "@types/nodemailer-direct-transport": "^1.0.31", + "@types/puppeteer-core": "^2.0.0", "@types/uid-generator": "^2.0.2", "@types/uuid": "^8.0.0", "@types/validator": "^13.0.0", @@ -223,6 +225,7 @@ "testcafe": "^1.8.0", "ts-jest": "^26.1.4", "ts-loader": "^7.0.5", + "ts-mock-imports": "^1.3.0", "ts-node": "^8.10.2", "ts-node-dev": "^1.0.0-pre.44", "typescript": "^3.9.7", diff --git a/src/app/constants/filesize.ts b/src/app/constants/filesize.ts new file mode 100644 index 0000000000..de246ed4c7 --- /dev/null +++ b/src/app/constants/filesize.ts @@ -0,0 +1,2 @@ +// 1 megabyte in bytes +export const MB = 1048576 diff --git a/src/app/constants/mail.ts b/src/app/constants/mail.ts new file mode 100644 index 0000000000..e10a0a8f6b --- /dev/null +++ b/src/app/constants/mail.ts @@ -0,0 +1,23 @@ +/** + * Headers to send to SES so we can parse email notifications + * + * NOTE: ALWAYS DO CASE-INSENSITIVE CHECKS FOR THE HEADERS! + * SES will automatically convert the case, so case-sensitive + * checks might fail. + * + * For example, 'X-FormSG-Form-Id' gets changed to 'X-Formsg-Form-ID'. + */ +export const EMAIL_HEADERS = { + formId: 'X-Formsg-Form-ID', + submissionId: 'X-Formsg-Submission-ID', + emailType: 'X-Formsg-Email-Type', +} + +// Types of emails we send +export const EMAIL_TYPES = { + adminResponse: 'Admin (response)', + loginOtp: 'Login OTP', + verificationOtp: 'Verification OTP', + emailConfirmation: 'Email confirmation', + adminBounce: 'Admin (bounce notification)', +} diff --git a/src/app/controllers/authentication.server.controller.js b/src/app/controllers/authentication.server.controller.js index f77a8c4900..725a575bdc 100755 --- a/src/app/controllers/authentication.server.controller.js +++ b/src/app/controllers/authentication.server.controller.js @@ -19,12 +19,10 @@ const config = require('../../config/config') const defaults = require('../../config/defaults').default const PERMISSIONS = require('../utils/permission-levels.js') const { getRequestIp } = require('../utils/request') -const { renderPromise } = require('../utils/render-promise') const logger = require('../../config/logger').createLoggerWithLabel( 'authentication', ) -const { sendNodeMail } = require('../services/mail.service') -const { EMAIL_HEADERS, EMAIL_TYPES } = require('../utils/constants') +const MailService = require('../services/mail.service').default const MAX_OTP_ATTEMPTS = 10 @@ -201,48 +199,18 @@ exports.sendOtp = async function (req, res) { // 2. Return success statement to front end let otp = res.locals.otp - let email = res.locals.email + let recipient = res.locals.email - let emailHTML try { - emailHTML = await renderPromise(res, 'templates/otp-email', { - appName: res.app.locals.title, - appUrl: config.app.appUrl, - otp: otp, + await MailService.sendLoginOtp({ + recipient, + otp, ipAddress: getRequestIp(req), }) - } catch (renderErr) { - logger.error(getRequestIp(req), req.url, req.headers, renderErr) - return res - .status(HttpStatus.INTERNAL_SERVER_ERROR) - .send( - 'Error rendering OTP. Please try again later and if the problem persists, contact us.', - ) - } - let mailOptions = { - to: email, - from: config.mail.mailer.from, - subject: 'One-Time Password (OTP) for ' + res.app.locals.title, - html: emailHTML, - headers: { - [EMAIL_HEADERS.emailType]: EMAIL_TYPES.loginOtp, - }, - } - try { - await sendNodeMail({ - mail: mailOptions, - options: { mailId: 'OTP' }, - }) - logger.info(`Login OTP sent:\temail=${email} ip=${getRequestIp(req)}`) - return res.status(HttpStatus.OK).send('OTP sent to ' + email + '!') - } catch (emailErr) { - logger.error( - 'Mail otp error', - getRequestIp(req), - req.url, - req.headers, - emailErr, - ) + logger.info(`Login OTP sent:\temail=${recipient} ip=${getRequestIp(req)}`) + return res.status(HttpStatus.OK).send(`OTP sent to ${recipient}!`) + } catch (err) { + logger.error('Mail otp error', getRequestIp(req), req.url, req.headers, err) return res .status(HttpStatus.INTERNAL_SERVER_ERROR) .send( diff --git a/src/app/controllers/email-submissions.server.controller.js b/src/app/controllers/email-submissions.server.controller.js index 666b099e08..ff206b05d9 100644 --- a/src/app/controllers/email-submissions.server.controller.js +++ b/src/app/controllers/email-submissions.server.controller.js @@ -1,6 +1,5 @@ 'use strict' -const moment = require('moment-timezone') const Busboy = require('busboy') const { Buffer } = require('buffer') const _ = require('lodash') @@ -16,7 +15,8 @@ const { FIELDS_TO_REJECT } = require('../utils/field-validation/config') const { getParsedResponses } = require('../utils/response') const { getRequestIp } = require('../utils/request') const { ConflictError } = require('../utils/custom-errors') -const { MB, EMAIL_HEADERS, EMAIL_TYPES } = require('../utils/constants') +const { EMAIL_TYPES } = require('../constants/mail') +const { MB } = require('../constants/filesize') const { attachmentsAreValid, addAttachmentToResponses, @@ -24,7 +24,6 @@ const { handleDuplicatesInAttachments, mapAttachmentsFromParsedResponses, } = require('../utils/attachment') -const { renderPromise } = require('../utils/render-promise') const config = require('../../config/config') const logger = require('../../config/logger').createLoggerWithLabel( 'email-submissions', @@ -32,7 +31,7 @@ const logger = require('../../config/logger').createLoggerWithLabel( const emailLogger = require('../../config/logger').createCloudWatchLogger( 'email', ) -const { sendNodeMail } = require('../services/mail.service') +const MailService = require('../services/mail.service').default const { sessionSecret } = config @@ -580,7 +579,6 @@ exports.saveMetadataToDb = function (req, res, next) { * Generate and send admin email * @param {Object} req - the expressjs request. Will be injected with the * objects parsed from `req.form` and `req.attachments` - * @param {Array} req.autoReplyEmails Auto-reply email fields * @param {Array} req.replyToEmails Reply-to emails * @param {Object} req.form - the form * @param {Array} req.formData Field-value tuples for admin email @@ -600,72 +598,27 @@ exports.sendAdminEmail = async function (req, res, next) { attachments, } = req - let submissionTime = moment(submission.created) - .tz('Asia/Singapore') - .format('ddd, DD MMM YYYY hh:mm:ss A') - - jsonData.unshift( - { - question: 'Reference Number', - answer: submission.id, - }, - { - question: 'Timestamp', - answer: submissionTime, - }, - ) - let html try { - html = await renderPromise(res, 'templates/submit-form-email', { - refNo: submission.id, - formTitle: form.title, - submissionTime, - formData, - jsonData, - appName: res.app.locals.title, + logger.info({ + message: 'Sending admin mail', + submissionId: submission.id, + formId: form._id, + ip: getRequestIp(req), + submissionHash: submission.responseHash, }) - } catch (err) { - logger.warn(err) - return onSubmissionEmailFailure(err, req, res, submission) - } - let mailOptions = { - to: form.emails, - from: config.mail.mailer.from, - subject: 'formsg-auto: ' + form.title + ' (Ref: ' + submission.id + ')', - html, - attachments, - headers: { - [EMAIL_HEADERS.formId]: String(form._id), - [EMAIL_HEADERS.submissionId]: submission.id, - [EMAIL_HEADERS.emailType]: EMAIL_TYPES.adminResponse, - }, - } - - // Set reply-to to all email fields that have reply to enabled - if (replyToEmails) { - let replyTo = replyToEmails.join(', ') - if (replyTo) mailOptions.replyTo = replyTo - } - - logger.info({ - message: 'Sending admin mail', - submissionId: submission.id, - formId: form._id, - ip: getRequestIp(req), - submissionHash: submission.responseHash, - }) - // Send mail - try { - await sendNodeMail({ - mail: mailOptions, - options: { - mailId: submission.id, - formId: form._id, - }, + await MailService.sendSubmissionToAdmin({ + replyToEmails, + form, + submission, + attachments, + jsonData, + formData, }) + return next() } catch (err) { + logger.warn('sendAdminEmail error', err) return onSubmissionEmailFailure(err, req, res, submission) } } diff --git a/src/app/controllers/submissions.server.controller.js b/src/app/controllers/submissions.server.controller.js index 549c78487f..0e32b616f9 100644 --- a/src/app/controllers/submissions.server.controller.js +++ b/src/app/controllers/submissions.server.controller.js @@ -1,7 +1,6 @@ 'use strict' const axios = require('axios') -const _ = require('lodash') const mongoose = require('mongoose') const errorHandler = require('./errors.server.controller') @@ -12,17 +11,10 @@ const HttpStatus = require('http-status-codes') const { getRequestIp } = require('../utils/request') const { isMalformedDate, createQueryWithDateParam } = require('../utils/date') -const { - parseAutoReplyData, - generateAutoReplyPdf, -} = require('../utils/autoreply-pdf') -const { renderPromise } = require('../utils/render-promise') -const config = require('../../config/config') const logger = require('../../config/logger').createLoggerWithLabel( 'authentication', ) -const { sendNodeMail } = require('../services/mail.service') -const { EMAIL_HEADERS, EMAIL_TYPES } = require('../utils/constants') +const MailService = require('../services/mail.service').default const GOOGLE_RECAPTCHA_URL = 'https://www.google.com/recaptcha/api/siteverify' @@ -156,128 +148,36 @@ exports.injectAutoReplyInfo = function (req, res, next) { * Long waterfall to send autoreply * @param {Object} req - Express request object * @param {Array} req.autoReplyEmails Auto-reply email fields - * @param {Array} req.replyToEmails Reply-to emails * @param {Object} req.form - the form * @param {Array>} req.autoReplyData Field-value tuples for auto-replies * @param {Object} req.submission Mongodb Submission object * @param {Object} req.attachments - submitted attachments, parsed by - * @param {Object} res - Express response object */ -const sendEmailAutoReplies = async function (req, res) { +const sendEmailAutoReplies = async function (req) { const { form, attachments, autoReplyEmails, autoReplyData, submission } = req if (autoReplyEmails.length === 0) { return Promise.resolve() } - const renderData = parseAutoReplyData( - form, - submission, - autoReplyData, - req.get('origin') || config.app.appUrl, - ) + try { - if (_.some(autoReplyEmails, ['includeFormSummary', true])) { - const pdfBuffer = await generateAutoReplyPdf({ renderData, res }) - attachments.push({ - filename: 'response.pdf', - content: pdfBuffer, - }) - } - // If one promise is rejected, carry on with the rest - return Promise.allSettled( - autoReplyEmails.map((autoReplyEmail, index) => - sendOneEmailAutoReply( - req, - res, - autoReplyEmail, - renderData, - attachments, - index, - ), - ), - ) + return MailService.sendAutoReplyEmails({ + form, + submission, + attachments, + responsesData: autoReplyData, + autoReplyMailDatas: autoReplyEmails, + }) } catch (err) { logger.error( - `Email autoreply error for formId=${form._id} submissionId=${submission.id}:\t${err}`, + `Mail autoreply error for formId=${form._id} submissionId=${ + submission.id + } ip=${getRequestIp(req)}:\t${err}`, ) // We do not deal with failed autoreplies return Promise.resolve() } } -/** - * Render and send auto-reply emails to the form submitter - * @param {Object} req Express request object - * @param {Object} res Express response object - * @param {Array} autoReplyEmails Auto-reply email fields - * @param {Object} form Form object - * @param {Object} renderData Data about the submission and answers to form questions. This is the raw - * data that is rendered on the response PDF if the PDF was needed, otherwise it is null. - * @param {String} submissionId The ObjectId of the submission - * @param {Array} attachments The attachments to send to form submitter. - * @param {String} attachment.filename Name of file - * @param {Buffer} attachment.buffer Contents of file - */ -async function sendOneEmailAutoReply( - req, - res, - autoReplyEmail, - renderData, - attachments, - index, -) { - const { form, submission } = req - const submissionId = submission.id - const defaultSubject = 'Thank you for submitting ' + form.title - const defaultSender = form.admin.agency.fullName - const defaultBody = `Dear Sir or Madam,\n\nThank you for submitting this form.\n\nRegards,\n${form.admin.agency.fullName}` - const autoReplyBody = (autoReplyEmail.body || defaultBody).split('\n') - - // Only include the form response if the flag is set - const templateData = autoReplyEmail.includeFormSummary - ? { autoReplyBody, ...renderData } - : { autoReplyBody } - let autoReplyHtml - try { - autoReplyHtml = await renderPromise( - res, - 'templates/submit-form-autoreply', - templateData, - ) - } catch (err) { - logger.warn('Render autoreply error', err) - return Promise.reject(err) - } - const senderName = autoReplyEmail.sender || defaultSender - // Sender's name appearing after ( symbol gets truncated. Escaping it solves the problem. - const escapedSenderName = senderName.replace('(', '\\(') - - const mail = { - to: autoReplyEmail.email, - from: `${escapedSenderName} <${config.mail.mailFrom}>`, - subject: autoReplyEmail.subject || defaultSubject, - // Only send attachments if the admin has the box checked for email field - attachments: autoReplyEmail.includeFormSummary ? attachments : [], - html: autoReplyHtml, - headers: { - [EMAIL_HEADERS.formId]: String(form._id), - [EMAIL_HEADERS.submissionId]: submission.id, - [EMAIL_HEADERS.emailType]: EMAIL_TYPES.emailConfirmation, - }, - } - try { - return sendNodeMail({ - mail, - options: { - retryCount: config.mail.retry.maxRetryCount, - mailId: `${submissionId}-${index}`, - }, - }) - } catch (err) { - logger.error(`Mail autoreply error:\t ip=${getRequestIp(req)}`, err) - return Promise.reject(err) - } -} - /** * Count number of form submissions for Results tab * @param {Object} req - Express request object diff --git a/src/app/models/form.server.model.ts b/src/app/models/form.server.model.ts index c174bd27c2..9b95821019 100644 --- a/src/app/models/form.server.model.ts +++ b/src/app/models/form.server.model.ts @@ -20,7 +20,7 @@ import { ResponseMode, Status, } from '../../types' -import { MB } from '../utils/constants' +import { MB } from '../constants/filesize' import getAgencyModel from './agency.server.model' import { diff --git a/src/app/services/mail.service.js b/src/app/services/mail.service.js deleted file mode 100644 index e97830713b..0000000000 --- a/src/app/services/mail.service.js +++ /dev/null @@ -1,141 +0,0 @@ -const validator = require('validator') -const HttpStatus = require('http-status-codes') -const _ = require('lodash') - -const mailLogger = require('../../config/logger').createLoggerWithLabel('mail') -const { HASH_EXPIRE_AFTER_SECONDS } = require('../../shared/util/verification') -const config = require('../../config/config') -const { EMAIL_HEADERS, EMAIL_TYPES } = require('../utils/constants') -const { - transporter, - retry: { retryDuration, maxRetryCount, maxRetryDuration }, - mailer, -} = config.mail - -/** - * Exponential backoff with full jitter - * @param {Number} retryCount - Number of email retries left - * @returns {Number} The duration to await, in milliseconds - */ -function calcEmailRetryDuration(retryCount) { - const attempt = maxRetryCount - retryCount - const newDuration = retryDuration * Math.pow(2, attempt) - const jitter = newDuration * (Math.random() * 2 - 1) - return Math.min(maxRetryDuration, newDuration + jitter) -} -/** - * Sends email to SES / Direct transport to send out - * @param {Object} email - Email object - * @param {Object} email.mail - Email - * @param {String} email.mail.to - Email address of recipient - * @param {String} email.mail.from - Email address of sender - * @param {String} email.mail.subject - Email subject - * @param {String} email.mail.html - HTML of email - * @param {Object} email.options.retryCount - Number of retries left - must be greater than zero. - */ -async function sendNodeMail(email) { - // In case of missing mail info - if (!email.mail || !email.mail.to) { - return Promise.reject('Mail undefined error') - } - - const { options } = email - const retryEmail = options && options.retryCount ? _.cloneDeep(email) : null - const emailLogString = `"Id: ${(options || {}).mailId} Email\t from:${ - email.mail.from - }\t subject:${email.mail.subject}\t formId: ${(options || {}).formId}"` - mailLogger.info(emailLogString) - mailLogger.profile(emailLogString) - try { - const response = await transporter.sendMail(email.mail) - mailLogger.info(`mailSuccess:\t${emailLogString}`) - return response - } catch (err) { - if ( - err.responseCode >= HttpStatus.BAD_REQUEST && - err.responseCode < HttpStatus.INTERNAL_SERVER_ERROR - ) { - // Retry for any emails with retryCount > 0, and 4xx errors - if ( - retryEmail !== null && - maxRetryCount >= retryEmail.options.retryCount && - retryEmail.options.retryCount > 0 - ) { - const duration = calcEmailRetryDuration(retryEmail.options.retryCount) - mailLogger.error( - `mailError ${err.responseCode} retryCount ${retryEmail.options.retryCount} duration ${duration}ms:\t${emailLogString}`, - ) - retryEmail.options.retryCount-- - return retryNodeMail(retryEmail, duration) - } else { - // No retry specified or ran out of retries, - const retryCount = _.get(retryEmail, 'options.retryCount', undefined) - mailLogger.error( - `mailError ${err.responseCode} retryCount ${retryCount}:\t${emailLogString}`, - ) - return Promise.reject(err) - } - } else if (err && err.responseCode) { - // Pass any other errors to the callback - mailLogger.error(`mailError ${err.responseCode}:\t${emailLogString}`) - return Promise.reject(err) - } else { - mailLogger.error(`mailError "${err}":\t${emailLogString}`) - return Promise.reject(err) - } - } -} - -/** - * Attempts an email retry after a given duration. - * @param {Object} retryEmail Info on email to retry. Equivalent to first argument of sendNodeMail. - * @param {number} duration Duration to wait before attempting retry. - * @return {Promise} Response info from email. - */ -const retryNodeMail = (retryEmail, duration) => { - return new Promise((resolve, reject) => { - setTimeout(() => { - sendNodeMail(retryEmail) - .then((response) => resolve(response)) - .catch((err) => reject(err)) - }, duration) - }) -} - -/** - * Sends an otp to a valid email - * @param {string} recipient - * @param {string} otp - * @throws {Error} error if mail fails, to be handled by verification service - */ -const sendVerificationOtp = async (recipient, otp) => { - if (!validator.isEmail(recipient)) - throw new Error(`${recipient} is not a valid email`) - const mailOptions = { - to: recipient, - from: mailer.from, - subject: `Your OTP for submitting a form on ${config.app.title}`, - html: ` -

You are currently submitting a form on ${config.app.title}.

-

Your OTP is ${otp}. - It will expire in ${Math.floor( - HASH_EXPIRE_AFTER_SECONDS / 60, - )} minutes. - Please use this to verify your submission.

-

If your OTP does not work, please request for a new OTP.

- `, - headers: { - [EMAIL_HEADERS.emailType]: EMAIL_TYPES.verificationOtp, - }, - } - // Error gets caught in getNewOtp - await sendNodeMail({ - mail: mailOptions, - options: { mailId: 'verify' }, - }) -} - -module.exports = { - sendVerificationOtp, - sendNodeMail, -} diff --git a/src/app/services/mail.service.ts b/src/app/services/mail.service.ts new file mode 100644 index 0000000000..bb1b782438 --- /dev/null +++ b/src/app/services/mail.service.ts @@ -0,0 +1,439 @@ +import { isEmpty } from 'lodash' +import moment from 'moment-timezone' +import Mail from 'nodemailer/lib/mailer' +import validator from 'validator' +import { Logger } from 'winston' + +import config from '../../config/config' +import { createLoggerWithLabel } from '../../config/logger' +import { HASH_EXPIRE_AFTER_SECONDS } from '../../shared/util/verification' +import { + AutoReplyOptions, + IEmailFormSchema, + IFormSchema, + IPopulatedForm, + ISubmissionSchema, +} from '../../types' +import { EMAIL_HEADERS, EMAIL_TYPES } from '../constants/mail' +import { + generateAutoreplyHtml, + generateAutoreplyPdf, + generateLoginOtpHtml, + generateSubmissionToAdminHtml, + generateVerificationOtpHtml, + isToFieldValid, +} from '../utils/mail' + +const mailLogger = createLoggerWithLabel('mail') + +type SendMailOptions = { + mailId?: string + formId?: string +} + +type SendSingleAutoreplyMailArgs = { + form: Pick + submission: Pick + autoReplyMailData: AutoReplyMailData + attachments: Mail.Attachment[] + formSummaryRenderData: AutoreplySummaryRenderData + index: number +} + +export type SendAutoReplyEmailsArgs = { + form: Pick + submission: Pick + attachments?: Mail.Attachment[] + responsesData: { question: string; answerTemplate: string[] }[] + autoReplyMailDatas: AutoReplyMailData[] +} + +type MailServiceParams = { + appName?: string + appUrl?: string + transporter?: Mail + senderMail?: string + logger?: Logger +} + +type AutoReplyMailData = { + email: string + subject?: AutoReplyOptions['autoReplySubject'] + sender?: AutoReplyOptions['autoReplySender'] + body?: AutoReplyOptions['autoReplyMessage'] + includeFormSummary?: AutoReplyOptions['includeFormSummary'] +} + +export type AutoreplySummaryRenderData = { + refNo: ISubmissionSchema['_id'] + formTitle: IFormSchema['title'] + submissionTime: string + // TODO (#42): Add proper types once the type is determined. + formData: any + formUrl: string +} + +type MailOptions = Omit & { + to: string | string[] +} + +export class MailService { + /** + * The application name to be shown in some sent emails' fields such as mail + * subject or mail body. + */ + #appName: string + /** + * The application URL to be shown in some sent emails' fields such as mail + * subject or mail body. + */ + #appUrl: string + /** + * The transporter to be used to send mail. + */ + #transporter: Mail + /** + * The email string to denote the "from" field of the email. + */ + #senderMail: string + /** + * The full string that can be shown in the mail's "from" field created from + * the given `appName` and `senderMail` arguments. + * + * E.g. `FormSG ` + */ + #senderFromString: string + /** + * Logger to log any errors encounted while sending mail. + */ + #logger: Logger + + constructor({ + appName = config.app.title, + appUrl = config.app.appUrl, + transporter = config.mail.transporter, + senderMail = config.mail.mailFrom, + logger = mailLogger, + }: MailServiceParams = {}) { + this.#logger = logger + + // Email validation + if (!validator.isEmail(senderMail)) { + const invalidMailError = new Error( + `MailService constructor: senderMail: ${senderMail} is not a valid email`, + ) + this.#logger.error(invalidMailError) + throw invalidMailError + } + + this.#appName = appName + this.#appUrl = appUrl + this.#senderMail = senderMail + this.#senderFromString = `${appName} <${senderMail}>` + this.#transporter = transporter + } + + /** + * Private function to send email using SES / Direct transport. + * @param mail Mail data to send with + * @param sendOptions Extra options to better identify mail, such as form or mail id. + */ + #sendNodeMail = async (mail: MailOptions, sendOptions?: SendMailOptions) => { + const emailLogString = `mailId: ${sendOptions?.mailId}\t Email from:${mail?.from}\t subject:${mail?.subject}\t formId: ${sendOptions?.formId}` + + // Guard against missing mail info. + if (!mail || isEmpty(mail.to)) { + this.#logger.error(`mailError: undefined mail. ${emailLogString}`) + return Promise.reject(new Error('Mail undefined error')) + } + + // Guard against invalid emails. + if (!isToFieldValid(mail.to)) { + this.#logger.error( + `mailError: ${mail.to} is not a valid email. ${emailLogString}`, + ) + return Promise.reject(new Error('Invalid email error')) + } + + this.#logger.info(emailLogString) + this.#logger.profile(emailLogString) + + try { + const response = await this.#transporter.sendMail(mail) + this.#logger.info(`mailSuccess:\t${emailLogString}`) + return response + } catch (err) { + // Pass errors to the callback + this.#logger.error( + `mailError ${err.responseCode}:\t${emailLogString}`, + err, + ) + return Promise.reject(err) + } + } + + /** + * Private function to send a single autoreply mail to recipients. + * @param arg the autoreply mail arguments + * @param arg.autoReplyMailData the main mail data to populate mail params + * @param arg.attachments the attachments to add to the mail + * @param arg.form the form mongoose object to populate the email subject or to retrieve the sender from + * @param arg.submission the submission mongoose object to retrieve id from for metadata + * @param arg.index the index metadata of this mail for logging purposes + */ + #sendSingleAutoreplyMail = async ({ + autoReplyMailData, + attachments, + formSummaryRenderData, + form, + submission, + index, + }: SendSingleAutoreplyMailArgs) => { + const emailSubject = + autoReplyMailData.subject || `Thank you for submitting ${form.title}` + // Sender's name appearing after "("" symbol gets truncated. Escaping it + // solves the problem. + const emailSender = ( + autoReplyMailData.sender || form.admin.agency.fullName + ).replace('(', '\\(') + + const defaultBody = `Dear Sir or Madam,\n\nThank you for submitting this form.\n\nRegards,\n${form.admin.agency.fullName}` + const autoReplyBody = (autoReplyMailData.body || defaultBody).split('\n') + + const templateData = { + autoReplyBody, + // Only destructure formSummaryRenderData if form summary is included. + ...(autoReplyMailData.includeFormSummary && formSummaryRenderData), + } + + const mailHtml = await generateAutoreplyHtml(templateData) + + const mail: MailOptions = { + to: autoReplyMailData.email, + from: `${emailSender} <${this.#senderMail}>`, + subject: emailSubject, + // Only send attachments if the admin has the box checked for email + // fields. + attachments: autoReplyMailData.includeFormSummary ? attachments : [], + html: mailHtml, + headers: { + [EMAIL_HEADERS.formId]: String(form._id), + [EMAIL_HEADERS.submissionId]: submission.id, + [EMAIL_HEADERS.emailType]: EMAIL_TYPES.emailConfirmation, + }, + } + + return this.#sendNodeMail(mail, { + mailId: `${submission.id}-${index}`, + formId: form._id, + }) + } + + /** + * Sends a verification otp to a valid email + * @param recipient the recipient email address + * @param otp the otp to send + * @throws error if mail fails, to be handled by the caller + */ + sendVerificationOtp = async (recipient: string, otp: string) => { + // TODO(#42): Remove param guards once whole backend is TypeScript. + if (!otp) { + throw new Error('OTP is missing.') + } + + const minutesToExpiry = Math.floor(HASH_EXPIRE_AFTER_SECONDS / 60) + + const mail: MailOptions = { + to: recipient, + from: this.#senderFromString, + subject: `Your OTP for submitting a form on ${this.#appName}`, + html: generateVerificationOtpHtml({ + appName: this.#appName, + minutesToExpiry, + otp, + }), + headers: { + [EMAIL_HEADERS.emailType]: EMAIL_TYPES.verificationOtp, + }, + } + // Error gets caught in getNewOtp + return this.#sendNodeMail(mail, { mailId: 'verify' }) + } + + /** + * Sends a login otp email to a valid email + * @param recipient the recipient email address + * @param otp the OTP to send + * @throws error if mail fails, to be handled by the caller + */ + sendLoginOtp = async ({ + recipient, + otp, + ipAddress, + }: { + recipient: string + otp: string + ipAddress: string + }) => { + const mail: MailOptions = { + to: recipient, + from: this.#senderFromString, + subject: `One-Time Password (OTP) for ${this.#appName}`, + html: await generateLoginOtpHtml({ + appName: this.#appName, + appUrl: this.#appUrl, + ipAddress: ipAddress, + otp, + }), + headers: { + [EMAIL_HEADERS.emailType]: EMAIL_TYPES.loginOtp, + }, + } + + return this.#sendNodeMail(mail, { mailId: 'OTP' }) + } + + /** + * Sends a submission response email to the admin of the given form. + * @param args the parameter object + * @param args.replyToEmails emails to set replyTo, if any + * @param args.form the form document to retrieve some email data from + * @param args.submission the submission document to retrieve some email data from + * @param args.attachments attachments to append to the email, if any + * @param args.jsonData the data to use in the data collation tool to be appended to the end of the email + * @param args.formData the form data to display to in the body in table form + */ + sendSubmissionToAdmin = async ({ + replyToEmails, + form, + submission, + attachments, + jsonData, + formData, + }: { + replyToEmails?: string[] + form: Pick + submission: Pick + attachments?: Mail.Attachment[] + formData: any[] + jsonData: { + question: string + answer: string | number + }[] + }) => { + const refNo = submission.id + const formTitle = form.title + const submissionTime = moment(submission.created) + .tz('Asia/Singapore') + .format('ddd, DD MMM YYYY hh:mm:ss A') + + // Add in additional metadata to jsonData. + // Unshift is not used as it mutates the array. + const fullJsonData = [ + { + question: 'Reference Number', + answer: refNo, + }, + { + question: 'Timestamp', + answer: submissionTime, + }, + ...jsonData, + ] + + const htmlData = { + appName: this.#appName, + formTitle, + refNo, + submissionTime, + jsonData: fullJsonData, + formData, + } + + const mailHtml = await generateSubmissionToAdminHtml(htmlData) + + const mail: MailOptions = { + to: form.emails, + from: this.#senderFromString, + subject: `formsg-auto: ${formTitle} (Ref: ${refNo})`, + html: mailHtml, + attachments, + headers: { + [EMAIL_HEADERS.formId]: String(form._id), + [EMAIL_HEADERS.submissionId]: refNo, + [EMAIL_HEADERS.emailType]: EMAIL_TYPES.adminResponse, + }, + // replyTo options only allow string format. + replyTo: replyToEmails?.join(', '), + } + + return this.#sendNodeMail(mail, { + mailId: refNo, + formId: String(form._id), + }) + } + + /** + * Sends an autoreply emails to the filler of the given form. + * @param args the arguments object + * @param args.form the form document to retrieve some email data from + * @param args.submission the submission document to retrieve some email data from + * @param args.attachments attachments to append to the email, if any + * @param args.responsesData the array of response data to use in rendering + * the mail body or summary pdf + * @param args.autoReplyMailDatas array of objects that contains autoreply mail data to override with defaults + * @param args.autoReplyMailDatas[].email contains the recipient of the mail + * @param args.autoReplyMailDatas[].subject if available, sends the mail out with this subject instead of the default subject + * @param args.autoReplyMailDatas[].sender if available, shows the given string as the sender instead of the default sender + * @param args.autoReplyMailDatas[].includeFormSummary if true, adds the given attachments into the sent mail + */ + sendAutoReplyEmails = async ({ + form, + submission, + responsesData, + autoReplyMailDatas, + attachments = [], + }: SendAutoReplyEmailsArgs) => { + // Data to render both the submission details mail HTML body PDF. + const renderData: AutoreplySummaryRenderData = { + refNo: submission.id, + formTitle: form.title, + submissionTime: moment(submission.created) + .tz('Asia/Singapore') + .format('ddd, DD MMM YYYY hh:mm:ss A'), + formData: responsesData, + formUrl: `${this.#appUrl}/${form._id}`, + } + + // Create a copy of attachments for attaching of autoreply pdf if needed. + const attachmentsWithAutoreplyPdf = [...attachments] + + // Generate autoreply pdf and append into attachments if any of the mail has + // to include a form summary. + if (autoReplyMailDatas.some((data) => data.includeFormSummary)) { + const pdfBuffer = await generateAutoreplyPdf(renderData) + attachmentsWithAutoreplyPdf.push({ + filename: 'response.pdf', + content: pdfBuffer, + }) + } + + // Prepare mail sending for each autoreply mail. + return Promise.allSettled( + autoReplyMailDatas.map((mailData, index) => { + return this.#sendSingleAutoreplyMail({ + form, + submission, + attachments: mailData.includeFormSummary + ? attachmentsWithAutoreplyPdf + : attachments, + autoReplyMailData: mailData, + formSummaryRenderData: renderData, + index, + }) + }), + ) + } +} + +export default new MailService() diff --git a/src/app/services/verification.service.js b/src/app/services/verification.service.js index 7d9d1e52e7..ba036a0bf1 100644 --- a/src/app/services/verification.service.js +++ b/src/app/services/verification.service.js @@ -2,7 +2,7 @@ const mongoose = require('mongoose') const bcrypt = require('bcrypt') const _ = require('lodash') const { otpGenerator } = require('../../config/config') -const mailService = require('./mail.service') +const MailService = require('./mail.service').default const smsFactory = require('./../factories/sms.factory') const vfnUtil = require('../../shared/util/verification') const formsgSdk = require('../../config/formsg-sdk') @@ -219,7 +219,7 @@ const sendOTPForField = async (formId, field, recipient, otp) => { break case 'email': // call email - it should validate the recipient - await mailService.sendVerificationOtp(recipient, otp) + await MailService.sendVerificationOtp(recipient, otp) break default: throw new Error(`sendOTPForField: ${fieldType} is unsupported`) diff --git a/src/app/utils/autoreply-pdf.js b/src/app/utils/autoreply-pdf.js deleted file mode 100644 index 7c186620f5..0000000000 --- a/src/app/utils/autoreply-pdf.js +++ /dev/null @@ -1,89 +0,0 @@ -const puppeteer = require('puppeteer-core') -const moment = require('moment-timezone') - -const config = require('../../config/config') -const logger = require('../../config/logger').createLoggerWithLabel('autoreply') -const { renderPromise } = require('../utils/render-promise') - -/** - * The data required to template an autoreply email - * @typedef {Object} autoReplyDataObject - * @param {String} refNo Submission ID - * @param {String} formTitle Title of the form - * @param {Date} submissionTime Timestamp of of submission - * @param {Object[]} formData Array of objects representing question-answer pairs - * @param {String[]} formData[].answerTemplate Array of answers. - * Most fields contain only one answer element, but table-style questions - * and checkboxes may generate additional rows. - * @param {String} formData[].question The question on the form - * @param {String} formUrl The URL of the form that was filled in - */ - -/** - * Returns a data object for templating autoreply emails - * @param {Object} form Form object - * @param {Object} submission Submissions object - * @param {*} autoReplyData auto-reply data - * @param {String} urlOrigin Domain of request - * @returns {autoReplyDataObject} - */ -function parseAutoReplyData(form, submission, autoReplyData, urlOrigin) { - return { - refNo: submission.id, - formTitle: form.title, - submissionTime: moment(submission.created) - .tz('Asia/Singapore') - .format('ddd, DD MMM YYYY hh:mm:ss A'), - formData: autoReplyData, - formUrl: `${urlOrigin}/${form._id}`, - } -} - -/** - * Generate response.pdf for auto-reply emails - * @param {autoReplyDataObject} renderData Data object created by parseAutoReplyData function - * @param {Express.Response} res The Express response object used for templating the HTML page. - * @return {Promise} Promise with PDF Buffer object - */ -async function generateAutoReplyPdf({ renderData, res }) { - let summaryHTML - try { - summaryHTML = await renderPromise( - res, - 'templates/submit-form-summary-pdf', - renderData, - ) - } catch (err) { - logger.warn('Unable to create submit form summary:\t', err) - return Promise.reject(err) - } - try { - const browser = await puppeteer.launch({ - args: ['--no-sandbox'], - headless: true, - executablePath: config.chromiumBin, - }) - const page = await browser.newPage() - await page.setContent(summaryHTML, { - waitUntil: 'networkidle0', - }) - const pdfBuffer = await page.pdf({ - format: 'A4', - printBackground: true, - margin: { - top: '20px', - bottom: '40px', - }, - }) - await browser.close() - return pdfBuffer - } catch (err) { - logger.error(`PDF error with Puppeteer:\t, ${err}`) - return Promise.reject(err) - } -} - -module.exports = { - generateAutoReplyPdf, - parseAutoReplyData, -} diff --git a/src/app/utils/constants.js b/src/app/utils/constants.js deleted file mode 100644 index 4490416e4a..0000000000 --- a/src/app/utils/constants.js +++ /dev/null @@ -1,33 +0,0 @@ -const MB = 1048576 // 1 megabyte in bytes -// Headers to send to SES so we can parse email notifications -// NOTE: ALWAYS DO CASE-INSENSITIVE CHECKS FOR THE HEADERS! -// SES will automatically convert the case, so case-sensitive -// checks might fail. For example, 'X-FormSG-Form-Id' gets changed -// to 'X-Formsg-Form-ID'. -const EMAIL_HEADERS = { - formId: 'X-Formsg-Form-ID', - submissionId: 'X-Formsg-Submission-ID', - emailType: 'X-Formsg-Email-Type', -} -// EMAIL_HEADERS with keys and values swapped and the new keys (e.g. X-Formsg-Form-ID) -// changed to lowercase (x-formsg-form-id). -// NOTE: ALWAYS DO CASE-INSENSITIVE CHECKS FOR THE HEADERS! -const EMAIL_LOWERCASE_HEADER_TO_KEY = {} -for (const [key, value] of Object.entries(EMAIL_HEADERS)) { - EMAIL_LOWERCASE_HEADER_TO_KEY[value.toLowerCase()] = key -} -// Types of emails we send -const EMAIL_TYPES = { - adminResponse: 'Admin (response)', - loginOtp: 'Login OTP', - verificationOtp: 'Verification OTP', - emailConfirmation: 'Email confirmation', - adminBounce: 'Admin (bounce notification)', -} - -module.exports = { - MB, - EMAIL_HEADERS, - EMAIL_TYPES, - EMAIL_LOWERCASE_HEADER_TO_KEY, -} diff --git a/src/app/utils/mail.ts b/src/app/utils/mail.ts new file mode 100644 index 0000000000..b8c8208bc3 --- /dev/null +++ b/src/app/utils/mail.ts @@ -0,0 +1,125 @@ +import dedent from 'dedent-js' +import ejs from 'ejs' +import { flattenDeep } from 'lodash' +import puppeteer from 'puppeteer-core' +import validator from 'validator' + +import { IFormSchema, ISubmissionSchema } from 'src/types' + +import config from '../../config/config' + +export const generateLoginOtpHtml = (htmlData: { + otp: string + appName: string + appUrl: string + ipAddress: string +}) => { + const pathToTemplate = `${process.cwd()}/src/app/views/templates/otp-email.server.view.html` + return ejs.renderFile(pathToTemplate, htmlData) +} + +export const generateVerificationOtpHtml = ({ + otp, + appName, + minutesToExpiry, +}: { + otp: string + appName: string + minutesToExpiry: number +}) => { + return dedent` +

You are currently submitting a form on ${appName}.

+

+ Your OTP is ${otp}. It will expire in ${minutesToExpiry} minutes. + Please use this to verify your submission. +

+

If your OTP does not work, please request for a new OTP.

+
+

If you did not make this request, you may ignore this email.

+
+

The ${appName} Support Team

+ ` +} + +type SubmissionToAdminHtmlData = { + refNo: string + formTitle: string + submissionTime: string + // TODO (#42): Add proper types once the type is determined. + formData: any[] + jsonData: { + question: string + answer: string | number + }[] + appName: string +} + +export const generateSubmissionToAdminHtml = async ( + htmlData: SubmissionToAdminHtmlData, +) => { + const pathToTemplate = `${process.cwd()}/src/app/views/templates/submit-form-email.server.view.html` + return ejs.renderFile(pathToTemplate, htmlData) +} + +type AutoreplySummaryRenderData = { + refNo: ISubmissionSchema['_id'] + formTitle: IFormSchema['title'] + submissionTime: string + formData: any + formUrl: string +} + +export const generateAutoreplyPdf = async ( + renderData: AutoreplySummaryRenderData, +) => { + const pathToTemplate = `${process.cwd()}/src/app/views/templates/submit-form-summary-pdf.server.view.html` + + const summaryHtml = await ejs.renderFile(pathToTemplate, renderData) + const browser = await puppeteer.launch({ + args: ['--no-sandbox'], + headless: true, + executablePath: config.chromiumBin, + }) + const page = await browser.newPage() + await page.setContent(summaryHtml, { + waitUntil: 'networkidle0', + }) + const pdfBuffer = await page.pdf({ + format: 'A4', + printBackground: true, + margin: { + top: '20px', + bottom: '40px', + }, + }) + await browser.close() + return pdfBuffer +} + +type AutoreplyHtmlData = { + autoReplyBody: string[] +} & (AutoreplySummaryRenderData | {}) + +export const generateAutoreplyHtml = async (htmlData: AutoreplyHtmlData) => { + const pathToTemplate = `${process.cwd()}/src/app/views/templates/submit-form-autoreply.server.view.html` + return ejs.renderFile(pathToTemplate, htmlData) +} + +export const isToFieldValid = (addresses: string | string[]) => { + // Retrieve all emails from each address. + // As addresses can be strings or a string array, cast given addresses param + // into an array regardless and flatten deep. + // The individual strings may still be an comma separated string, and thus + // further splitting is necessary. + // The final result is once again flattened. + const mails = flattenDeep( + flattenDeep([addresses]).map((addrString) => + String(addrString) + .split(',') + .map((addr) => addr.trim()), + ), + ) + + // Every address must be an email to be valid. + return mails.every((addr) => validator.isEmail(addr)) +} diff --git a/src/app/utils/sns.js b/src/app/utils/sns.js index 600844d05e..89ed1e55d6 100644 --- a/src/app/utils/sns.js +++ b/src/app/utils/sns.js @@ -1,7 +1,7 @@ const axios = require('axios') const crypto = require('crypto') const get = require('lodash/get') -const { EMAIL_LOWERCASE_HEADER_TO_KEY } = require('./constants') +const { EMAIL_HEADERS } = require('../constants/mail') // Note that these need to be ordered in order to generate // the correct string to sign @@ -16,6 +16,17 @@ const snsKeys = [ { key: 'SignatureVersion', toSign: false }, ] +const EMAIL_LOWERCASE_HEADER_TO_KEY = (() => { + // EMAIL_HEADERS with keys and values swapped and the new keys (e.g. X-Formsg-Form-ID) + // changed to lowercase (x-formsg-form-id). + // NOTE: ALWAYS DO CASE-INSENSITIVE CHECKS FOR THE HEADERS! + const lowercasedHeadersToKey = {} + for (const [key, value] of Object.entries(EMAIL_HEADERS)) { + lowercasedHeadersToKey[value.toLowerCase()] = key + } + return lowercasedHeadersToKey +})() + // Hostname for AWS URLs const AWS_HOSTNAME = '.amazonaws.com' diff --git a/src/config/config.ts b/src/config/config.ts index bfa3840013..7ef05c06ca 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -40,19 +40,12 @@ type AwsConfig = { s3: aws.S3 } -type EmailRetryConfig = { - retryDuration: number - maxRetryCount: number - maxRetryDuration: number -} - type MailConfig = { mailFrom: string mailer: { from: string } transporter: Mail - retry: EmailRetryConfig } type Config = { @@ -277,21 +270,10 @@ const mailConfig: MailConfig = (function () { transporter = nodemailer.createTransport(directTransport({})) } - const emailRetryConfig: EmailRetryConfig = { - retryDuration: - Number(process.env.MAIL_RETRY_DURATION) || defaults.mail.retryDuration, - maxRetryCount: - Number(process.env.MAIL_RETRY_COUNT) || defaults.mail.maxRetryCount, - maxRetryDuration: - Number(process.env.MAIL_MAX_RETRY_DURATION) || - defaults.mail.maxRetryDuration, - } - return { mailFrom, mailer, transporter, - retry: emailRetryConfig, } })() diff --git a/src/config/defaults.ts b/src/config/defaults.ts index c4d6a2e6de..86199c4de4 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -21,12 +21,8 @@ const LOGIN_CONFIG = { otpLifeSpan: 900000, } -// Config for email sending/retrying. +// Config for email sending. const MAIL_CONFIG = { - // Number is in miliseconds. - retryDuration: 60000, // 1 min in ms. - maxRetryCount: 2, - maxRetryDuration: 4800000, // 80 min in ms. // The sender email to display on mail sent. mailFrom: 'donotreply@mail.form.gov.sg', } diff --git a/src/loaders/express/session.ts b/src/loaders/express/session.ts index f6ca81ea2d..cfff745dcc 100644 --- a/src/loaders/express/session.ts +++ b/src/loaders/express/session.ts @@ -13,7 +13,7 @@ const sessionMiddlewares = (connection: Connection) => { saveUninitialized: false, resave: false, secret: config.sessionSecret, - // TODO(#2505): Remove the typecast once `config` has correct types. + // TODO(#42): Remove the typecast once `config` has correct types. cookie: config.cookieSettings as SessionOptions['cookie'], name: 'connect.sid', store: new sessionMongoStore({ diff --git a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js index bac8eb865f..a841c1a57a 100644 --- a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js +++ b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js @@ -12,6 +12,8 @@ const dbHandler = require('../helpers/db-handler') const { validSnsBody } = require('../resources/valid-sns-body') const { getSnsBasestring } = require('../../../../dist/backend/app/utils/sns') const { ObjectID } = require('bson-ext') +const MailService = require('../../../../dist/backend/app/services/mail.service') + .default const User = dbHandler.makeModel('user.server.model', 'User') const Agency = dbHandler.makeModel('agency.server.model', 'Agency') @@ -25,8 +27,7 @@ describe('Email Submissions Controller', () => { // Declare global variables let spyRequest - let mockSendNodeMail = jasmine.createSpy() - + let sendSubmissionMailSpy // spec out controller such that calls to request are // directed through a callback to the request spy, // which will be destroyed and re-created for every test @@ -34,9 +35,6 @@ describe('Email Submissions Controller', () => { 'dist/backend/app/controllers/email-submissions.server.controller', { mongoose: Object.assign(mongoose, { '@noCallThru': true }), - '../services/mail.service': { - sendNodeMail: mockSendNodeMail, - }, }, ) const submissionsController = spec( @@ -47,9 +45,6 @@ describe('Email Submissions Controller', () => { '../../config/config': { sessionSecret: SESSION_SECRET, }, - '../services/mail.service': { - sendNodeMail: mockSendNodeMail, - }, }, ) const spcpController = spec( @@ -68,7 +63,6 @@ describe('Email Submissions Controller', () => { afterAll(async () => await dbHandler.closeDatabase()) describe('notifyParties', () => { - const config = spec('dist/backend/config/config') const originalConsoleError = console.error let fixtures @@ -95,20 +89,19 @@ describe('Email Submissions Controller', () => { .get(injectFixtures, controller.sendAdminEmail, (req, res) => res.status(200).send(), ) + + sendSubmissionMailSpy = spyOn(MailService, 'sendSubmissionToAdmin') }) afterAll(() => { console.error = originalConsoleError }) + afterEach(() => sendSubmissionMailSpy.calls.reset()) + beforeEach(() => { - mockSendNodeMail.and.callFake(({ mail }) => { - if (!mail.to || !mail.from || !mail.subject || !mail.html) { - throw new Error('mockSendNodeMail error') - } - }) fixtures = { - autoReplyEmails: [], + replyToEmails: [], attachments: [ { filename: 'file.txt', @@ -148,84 +141,26 @@ describe('Email Submissions Controller', () => { }) it('sends mail with correct parameters', (done) => { - request(app) - .get(endpointPath) - .expect(HttpStatus.OK) - .then(() => { - expect(mockSendNodeMail).toHaveBeenCalled() - const mailOptions = mockSendNodeMail.calls.mostRecent().args[0].mail - console.error('mailOptions.html', mailOptions.html) - expect(mailOptions.to).toEqual(fixtures.form.emails) - expect(mailOptions.from).toEqual(config.mail.mailer.from) - expect(mailOptions.subject).toContain(fixtures.form.title) - expect(mailOptions.subject).toContain(fixtures.submission.id) - expect(mailOptions.attachments).toEqual(fixtures.attachments) - expect(mailOptions.html).toContain(fixtures.formData[0].question) - expect(mailOptions.html).toContain( - fixtures.formData[0].answerTemplate, - ) - }) - .then(done) - .catch(done) - }) - - it('sends mail with reply-to emails', (done) => { - fixtures.replyToEmails = [ - 'reply-to-1@test.gov.sg', - 'reply-to-2@test.gov.sg', - ] - request(app) - .get(endpointPath) - .expect(HttpStatus.OK) - .then(() => { - expect(mockSendNodeMail).toHaveBeenCalled() - const mailOptions = mockSendNodeMail.calls.mostRecent().args[0].mail - expect(mailOptions.to).toEqual(fixtures.form.emails) - expect(mailOptions.from).toEqual(config.mail.mailer.from) - expect(mailOptions.replyTo).toEqual( - 'reply-to-1@test.gov.sg, reply-to-2@test.gov.sg', - ) - expect(mailOptions.subject).toContain(fixtures.form.title) - expect(mailOptions.subject).toContain(fixtures.submission.id) - expect(mailOptions.attachments).toEqual(fixtures.attachments) - expect(mailOptions.html).toContain(fixtures.formData[0].question) - expect(mailOptions.html).toContain( - fixtures.formData[0].answerTemplate, - ) - }) - .then(done) - .catch(done) - }) + // Arrange + const mockSuccessResponse = 'mockSuccessResponse' + sendSubmissionMailSpy.and.callFake(() => mockSuccessResponse) - // TODO: This merely tests admin mail being sent out if there is autoreply, - // but has yet to test if autoreply emails are being sent out. - it('ensures mail is still sent out with autoreply emails', (done) => { - fixtures.autoReplyEmails.push( - { email: 'no-reply@test.gov.sg' }, - { email: 'nobody@test.gov.sg' }, - ) request(app) .get(endpointPath) .expect(HttpStatus.OK) .then(() => { - expect(mockSendNodeMail).toHaveBeenCalled() - const mailOptions = mockSendNodeMail.calls.mostRecent().args[0].mail - expect(mailOptions.to).toEqual(fixtures.form.emails) - expect(mailOptions.from).toEqual(config.mail.mailer.from) - expect(mailOptions.replyTo).toEqual() - expect(mailOptions.subject).toContain(fixtures.form.title) - expect(mailOptions.subject).toContain(fixtures.submission.id) - expect(mailOptions.attachments).toEqual(fixtures.attachments) - expect(mailOptions.html).toContain(fixtures.formData[0].question) - expect(mailOptions.html).toContain( - fixtures.formData[0].answerTemplate, - ) + const mailOptions = sendSubmissionMailSpy.calls.mostRecent().args[0] + expect(mailOptions).toEqual(fixtures) }) .then(done) .catch(done) }) it('errors with 400 on send failure', (done) => { + // Arrange + sendSubmissionMailSpy.and.callFake(() => + Promise.reject(new Error('mockErrorResponse')), + ) // Trigger error by deleting recipient list delete fixtures.form.emails request(app) diff --git a/tests/unit/backend/controllers/verification.server.controller.spec.js b/tests/unit/backend/controllers/verification.server.controller.spec.js index e0bc63d72e..460b9203d5 100644 --- a/tests/unit/backend/controllers/verification.server.controller.spec.js +++ b/tests/unit/backend/controllers/verification.server.controller.spec.js @@ -6,6 +6,8 @@ const request = require('supertest') const constants = require('../../../../dist/backend/shared/util/verification') const dbHandler = require('../helpers/db-handler') +const MailService = require('../../../../dist/backend/app/services/mail.service') + .default const User = dbHandler.makeModel('user.server.model', 'User') const Agency = dbHandler.makeModel('agency.server.model', 'Agency') @@ -17,11 +19,11 @@ const Verification = dbHandler.makeModel( describe('Verification Controller', () => { const bcrypt = jasmine.createSpyObj('bcrypt', ['hash']) - const sendEmailOtp = jasmine.createSpy('sendEmailOtp') const sendSmsOtp = jasmine.createSpy('sendSmsOtp') const testOtp = '123456' let spyRequest = jasmine.createSpy('request') + let sendOtpSpy let req let res @@ -32,9 +34,6 @@ describe('Verification Controller', () => { otpGenerator: () => testOtp, logger: console, }, - './mail.service': { - sendVerificationOtp: sendEmailOtp, - }, './../factories/sms.factory': { sendVerificationOtp: sendSmsOtp, }, @@ -61,6 +60,8 @@ describe('Verification Controller', () => { beforeAll(async (done) => { await dbHandler.connect() + sendOtpSpy = spyOn(MailService, 'sendVerificationOtp') + await Agency.deleteMany({}) testAgency = new Agency({ shortName: 'govtest', @@ -260,7 +261,7 @@ describe('Verification Controller', () => { } res.sendStatus.and.callFake(function (status) { expect(status).toEqual(HttpStatus.CREATED) - expect(sendEmailOtp).toHaveBeenCalled() + expect(sendOtpSpy).toHaveBeenCalled() Verification.findById(transaction._id, function (err, result) { // eslint-disable-next-line no-console if (err) console.error(err) diff --git a/tests/unit/backend/services/mail.service.spec.ts b/tests/unit/backend/services/mail.service.spec.ts new file mode 100644 index 0000000000..8c118f61af --- /dev/null +++ b/tests/unit/backend/services/mail.service.spec.ts @@ -0,0 +1,572 @@ +import { cloneDeep } from 'lodash' +import moment from 'moment-timezone' +import Mail, { Attachment } from 'nodemailer/lib/mailer' +import { ImportMock } from 'ts-mock-imports' + +import { + AutoreplySummaryRenderData, + MailService, + SendAutoReplyEmailsArgs, +} from 'src/app/services/mail.service' +import * as MailUtils from 'src/app/utils/mail' +import { IPopulatedForm, ISubmissionSchema } from 'src/types' + +const MOCK_VALID_EMAIL = 'to@example.com' +const MOCK_VALID_EMAIL_2 = 'to2@example.com' +const MOCK_VALID_EMAIL_3 = 'to3@example.com' +const MOCK_SENDER_EMAIL = 'from@example.com' +const MOCK_APP_NAME = 'mockApp' +const MOCK_APP_URL = 'mockApp.example.com' +const MOCK_SENDER_STRING = `${MOCK_APP_NAME} <${MOCK_SENDER_EMAIL}>` +const MOCK_PDF = 'fake pdf' + +describe('mail.service', () => { + const sendMailSpy = jest.fn() + const mockTransporter = ({ + sendMail: sendMailSpy, + } as unknown) as Mail + + // Set up mocks for MailUtils + beforeAll(() => { + ImportMock.mockFunction(MailUtils, 'generateAutoreplyPdf', MOCK_PDF) + }) + beforeEach(() => sendMailSpy.mockReset()) + afterAll(() => ImportMock.restore()) + + const mailService = new MailService({ + transporter: mockTransporter, + senderMail: MOCK_SENDER_EMAIL, + appName: MOCK_APP_NAME, + appUrl: MOCK_APP_URL, + }) + + describe('Constructor', () => { + it('should throw error when invalid senderMail param is passed ', () => { + // Arrange + const invalidParams = { + transporter: mockTransporter, + senderMail: 'notAnEmail', + } + // Act + Assert + expect(() => new MailService(invalidParams)).toThrowError( + `MailService constructor: senderMail: ${invalidParams.senderMail} is not a valid email`, + ) + }) + + it('should create service successfully', () => { + expect(mailService).toBeDefined() + }) + }) + + describe('sendVerificationOtp', () => { + const MOCK_OTP = '123456' + + it('should send verification otp successfully', async () => { + // Arrange + // sendMail should return mocked success response + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const expectedArgument = { + to: MOCK_VALID_EMAIL, + from: MOCK_SENDER_STRING, + subject: `Your OTP for submitting a form on ${MOCK_APP_NAME}`, + html: MailUtils.generateVerificationOtpHtml({ + appName: MOCK_APP_NAME, + otp: MOCK_OTP, + minutesToExpiry: 10, + }), + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Verification OTP', + }, + } + + // Act + Assert + await expect( + mailService.sendVerificationOtp(MOCK_VALID_EMAIL, MOCK_OTP), + ).resolves.toEqual(mockedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArgument) + }) + + it('should reject with error when email is invalid', async () => { + // Arrange + const invalidEmail = 'notAnEmail' + // Act + Assert + await expect( + mailService.sendVerificationOtp(invalidEmail, MOCK_OTP), + ).rejects.toThrowError('Invalid email error') + }) + }) + + describe('sendLoginOtp', () => { + const MOCK_OTP = '123456' + const MOCK_IP = 'mock:5000' + it('should send login otp successfully', async () => { + // Arrange + // sendMail should return mocked success response + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const expectedArgument = { + to: MOCK_VALID_EMAIL, + from: MOCK_SENDER_STRING, + subject: `One-Time Password (OTP) for ${MOCK_APP_NAME}`, + html: await MailUtils.generateLoginOtpHtml({ + otp: MOCK_OTP, + appName: MOCK_APP_NAME, + appUrl: MOCK_APP_URL, + ipAddress: MOCK_IP, + }), + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Login OTP', + }, + } + + // Act + Assert + await expect( + mailService.sendLoginOtp({ + recipient: MOCK_VALID_EMAIL, + otp: MOCK_OTP, + ipAddress: MOCK_IP, + }), + ).resolves.toEqual(mockedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArgument) + }) + + it('should reject with error when email is invalid', async () => { + // Arrange + const invalidEmail = 'notAnEmail' + // Act + Assert + await expect( + mailService.sendVerificationOtp(invalidEmail, MOCK_OTP), + ).rejects.toThrowError('Invalid email error') + }) + }) + + describe('sendSubmissionToAdmin', () => { + let expectedHtml: string + + const MOCK_VALID_SUBMISSION_PARAMS = { + replyToEmails: ['test1@example.com', 'test2@example.com'], + form: { + title: 'Test form title', + _id: 'mockFormId', + emails: [MOCK_VALID_EMAIL], + }, + submission: { + id: 'mockSubmissionId', + created: new Date(), + }, + attachments: [], + jsonData: [ + { + question: 'some question', + answer: 'some answer', + }, + ], + formData: [], + } + + const FORMATTED_SUBMISSION_TIME = moment( + MOCK_VALID_SUBMISSION_PARAMS.submission.created, + ) + .tz('Asia/Singapore') + .format('ddd, DD MMM YYYY hh:mm:ss A') + + // Should include the metadata in the front. + const EXPECTED_JSON_DATA = [ + { + question: 'Reference Number', + answer: MOCK_VALID_SUBMISSION_PARAMS.submission.id, + }, + { + question: 'Timestamp', + answer: FORMATTED_SUBMISSION_TIME, + }, + ...MOCK_VALID_SUBMISSION_PARAMS.jsonData, + ] + + beforeAll(async () => { + const htmlData = { + appName: MOCK_APP_NAME, + formData: MOCK_VALID_SUBMISSION_PARAMS.formData, + formTitle: MOCK_VALID_SUBMISSION_PARAMS.form.title, + jsonData: EXPECTED_JSON_DATA, + refNo: MOCK_VALID_SUBMISSION_PARAMS.submission.id, + submissionTime: FORMATTED_SUBMISSION_TIME, + } + expectedHtml = await MailUtils.generateSubmissionToAdminHtml(htmlData) + }) + + it('should send submission mail to admin successfully if form.emails is an array with a single string', async () => { + // sendMail should return mocked success response + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const expectedArgument = { + to: MOCK_VALID_SUBMISSION_PARAMS.form.emails, + from: MOCK_SENDER_STRING, + subject: `formsg-auto: ${MOCK_VALID_SUBMISSION_PARAMS.form.title} (Ref: ${MOCK_VALID_SUBMISSION_PARAMS.submission.id})`, + html: expectedHtml, + attachments: MOCK_VALID_SUBMISSION_PARAMS.attachments, + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Admin (response)', + 'X-Formsg-Form-ID': MOCK_VALID_SUBMISSION_PARAMS.form._id, + 'X-Formsg-Submission-ID': MOCK_VALID_SUBMISSION_PARAMS.submission.id, + }, + replyTo: MOCK_VALID_SUBMISSION_PARAMS.replyToEmails.join(', '), + } + + // Act + Assert + await expect( + mailService.sendSubmissionToAdmin(MOCK_VALID_SUBMISSION_PARAMS), + ).resolves.toEqual(mockedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArgument) + }) + + it('should send submission mail to admin successfully if form.emails is an array with a single comma separated string', async () => { + // sendMail should return mocked success response + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const formEmailsCommaSeparated = [ + `${MOCK_VALID_EMAIL}, ${MOCK_VALID_EMAIL_2}`, + ] + const modifiedParams = cloneDeep(MOCK_VALID_SUBMISSION_PARAMS) + modifiedParams.form.emails = formEmailsCommaSeparated + + const expectedArgument = { + to: formEmailsCommaSeparated, + from: MOCK_SENDER_STRING, + subject: `formsg-auto: ${MOCK_VALID_SUBMISSION_PARAMS.form.title} (Ref: ${MOCK_VALID_SUBMISSION_PARAMS.submission.id})`, + html: expectedHtml, + attachments: MOCK_VALID_SUBMISSION_PARAMS.attachments, + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Admin (response)', + 'X-Formsg-Form-ID': MOCK_VALID_SUBMISSION_PARAMS.form._id, + 'X-Formsg-Submission-ID': MOCK_VALID_SUBMISSION_PARAMS.submission.id, + }, + replyTo: MOCK_VALID_SUBMISSION_PARAMS.replyToEmails.join(', '), + } + + // Act + Assert + await expect( + mailService.sendSubmissionToAdmin(modifiedParams), + ).resolves.toEqual(mockedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArgument) + }) + + it('should send submission mail to admin successfully if form.emails is an array with multiple emails', async () => { + // Arrange + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const formMultipleEmailsArray = [MOCK_VALID_EMAIL, MOCK_VALID_EMAIL_2] + const modifiedParams = cloneDeep(MOCK_VALID_SUBMISSION_PARAMS) + modifiedParams.form.emails = formMultipleEmailsArray + + const expectedArgument = { + to: formMultipleEmailsArray, + from: MOCK_SENDER_STRING, + subject: `formsg-auto: ${MOCK_VALID_SUBMISSION_PARAMS.form.title} (Ref: ${MOCK_VALID_SUBMISSION_PARAMS.submission.id})`, + html: expectedHtml, + attachments: MOCK_VALID_SUBMISSION_PARAMS.attachments, + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Admin (response)', + 'X-Formsg-Form-ID': MOCK_VALID_SUBMISSION_PARAMS.form._id, + 'X-Formsg-Submission-ID': MOCK_VALID_SUBMISSION_PARAMS.submission.id, + }, + replyTo: MOCK_VALID_SUBMISSION_PARAMS.replyToEmails.join(', '), + } + // Act + Assert + await expect( + mailService.sendSubmissionToAdmin(modifiedParams), + ).resolves.toEqual(mockedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArgument) + }) + + it('should send submission mail to admin successfully if form.emails is an array with a mixture of emails and comma separated emails strings', async () => { + // Arrange + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const formEmailsMixture = [ + `${MOCK_VALID_EMAIL}, ${MOCK_VALID_EMAIL_2}`, + MOCK_VALID_EMAIL_3, + ] + const modifiedParams = cloneDeep(MOCK_VALID_SUBMISSION_PARAMS) + modifiedParams.form.emails = formEmailsMixture + + const expectedArgument = { + to: formEmailsMixture, + from: MOCK_SENDER_STRING, + subject: `formsg-auto: ${MOCK_VALID_SUBMISSION_PARAMS.form.title} (Ref: ${MOCK_VALID_SUBMISSION_PARAMS.submission.id})`, + html: expectedHtml, + attachments: MOCK_VALID_SUBMISSION_PARAMS.attachments, + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Admin (response)', + 'X-Formsg-Form-ID': MOCK_VALID_SUBMISSION_PARAMS.form._id, + 'X-Formsg-Submission-ID': MOCK_VALID_SUBMISSION_PARAMS.submission.id, + }, + replyTo: MOCK_VALID_SUBMISSION_PARAMS.replyToEmails.join(', '), + } + // Act + Assert + await expect( + mailService.sendSubmissionToAdmin(modifiedParams), + ).resolves.toEqual(mockedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArgument) + }) + + it('should reject with error when form.emails array contains an invalid email string', async () => { + // Arrange + const invalidParams = cloneDeep(MOCK_VALID_SUBMISSION_PARAMS) + invalidParams.form.emails = ['notAnEmail', MOCK_VALID_EMAIL] + + // Act + Assert + await expect( + mailService.sendSubmissionToAdmin(invalidParams), + ).rejects.toThrowError('Invalid email error') + }) + + it('should reject with error when form.emails param is an empty array', async () => { + // Arrange + const invalidParams = cloneDeep(MOCK_VALID_SUBMISSION_PARAMS) + invalidParams.form.emails = [] + // Act + Assert + await expect( + mailService.sendSubmissionToAdmin(invalidParams), + ).rejects.toThrowError('Mail undefined error') + }) + }) + + describe('sendAutoReplyEmails', () => { + let defaultHtml: string + let defaultExpectedArg + + const MOCK_SENDER_NAME = 'John Doe' + const MOCK_AUTOREPLY_PARAMS: SendAutoReplyEmailsArgs = { + form: { + title: 'Test form title', + _id: 'mockFormId', + admin: { + agency: { + fullName: MOCK_SENDER_NAME, + }, + }, + } as IPopulatedForm, + submission: ({ + id: 'mockSubmissionId', + } as unknown) as ISubmissionSchema, + responsesData: [ + { + question: 'some question', + answerTemplate: ['some answer template'], + }, + ], + attachments: ['something'] as Attachment[], + autoReplyMailDatas: [ + { + email: MOCK_VALID_EMAIL_2, + }, + ], + } + const DEFAULT_AUTO_REPLY_BODY = `Dear Sir or Madam,\n\nThank you for submitting this form.\n\nRegards,\n${MOCK_AUTOREPLY_PARAMS.form.admin.agency.fullName}`.split( + '\n', + ) + + beforeAll(async () => { + defaultHtml = await MailUtils.generateAutoreplyHtml({ + autoReplyBody: DEFAULT_AUTO_REPLY_BODY, + }) + + defaultExpectedArg = { + to: MOCK_AUTOREPLY_PARAMS.autoReplyMailDatas[0].email, + from: `${MOCK_AUTOREPLY_PARAMS.form.admin.agency.fullName} <${MOCK_SENDER_EMAIL}>`, + subject: `Thank you for submitting ${MOCK_AUTOREPLY_PARAMS.form.title}`, + html: defaultHtml, + headers: { + // Hardcode in tests in case something changes this. + 'X-Formsg-Email-Type': 'Email confirmation', + 'X-Formsg-Form-ID': MOCK_AUTOREPLY_PARAMS.form._id, + 'X-Formsg-Submission-ID': MOCK_AUTOREPLY_PARAMS.submission.id, + }, + // Note this should be by default empty + attachments: [], + } + }) + + it('should send single autoreply mail successfully with defaults', async () => { + // Arrange + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + // Act + Assert + const expectedResponse = await Promise.allSettled([ + Promise.resolve(mockedResponse), + ]) + await expect( + mailService.sendAutoReplyEmails(MOCK_AUTOREPLY_PARAMS), + ).resolves.toEqual(expectedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(defaultExpectedArg) + }) + + it('should send array of multiple autoreply mails successfully with defaults', async () => { + const firstMockedResponse = 'mockedSuccessResponse1' + const secondMockedResponse = 'mockedSuccessResponse1' + sendMailSpy + .mockResolvedValueOnce(firstMockedResponse) + .mockResolvedValueOnce(secondMockedResponse) + + const multipleEmailParams = cloneDeep(MOCK_AUTOREPLY_PARAMS) + multipleEmailParams.autoReplyMailDatas.push({ + email: MOCK_VALID_EMAIL_3, + }) + + const secondExpectedArg = cloneDeep(defaultExpectedArg) + secondExpectedArg.to = MOCK_VALID_EMAIL_3 + + // Act + Assert + const expectedResponse = await Promise.allSettled([ + Promise.resolve(firstMockedResponse), + Promise.resolve(secondMockedResponse), + ]) + await expect( + mailService.sendAutoReplyEmails(multipleEmailParams), + ).resolves.toEqual(expectedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(2) + expect(sendMailSpy).toHaveBeenNthCalledWith(1, defaultExpectedArg) + // Second call should be for a different email + expect(sendMailSpy).toHaveBeenNthCalledWith(2, secondExpectedArg) + }) + + it('should send single autoreply mail successfully with custom autoreply subject', async () => { + // Arrange + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const customSubject = 'customSubject' + const customDataParams = cloneDeep(MOCK_AUTOREPLY_PARAMS) + customDataParams.autoReplyMailDatas[0].subject = customSubject + + const expectedArg = { ...defaultExpectedArg, subject: customSubject } + + // Act + Assert + const expectedResponse = await Promise.allSettled([ + Promise.resolve(mockedResponse), + ]) + await expect( + mailService.sendAutoReplyEmails(customDataParams), + ).resolves.toEqual(expectedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArg) + }) + + it('should send single autoreply mail successfully with custom autoreply sender', async () => { + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const customSender = 'customSender@example.com' + const customDataParams = cloneDeep(MOCK_AUTOREPLY_PARAMS) + customDataParams.autoReplyMailDatas[0].sender = customSender + + const expectedArg = { + ...defaultExpectedArg, + from: `${customSender} <${MOCK_SENDER_EMAIL}>`, + } + + // Act + Assert + const expectedResponse = await Promise.allSettled([ + Promise.resolve(mockedResponse), + ]) + await expect( + mailService.sendAutoReplyEmails(customDataParams), + ).resolves.toEqual(expectedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArg) + }) + + it('should send single autoreply mail with attachment if autoReply.includeFormSummary is true', async () => { + // Arrange + const mockedResponse = 'mockedSuccessResponse' + sendMailSpy.mockResolvedValueOnce(mockedResponse) + + const customDataParams = cloneDeep(MOCK_AUTOREPLY_PARAMS) + customDataParams.autoReplyMailDatas[0].includeFormSummary = true + + const expectedRenderData: AutoreplySummaryRenderData = { + formData: MOCK_AUTOREPLY_PARAMS.responsesData, + formTitle: MOCK_AUTOREPLY_PARAMS.form.title, + formUrl: `${MOCK_APP_URL}/${MOCK_AUTOREPLY_PARAMS.form._id}`, + refNo: MOCK_AUTOREPLY_PARAMS.submission.id, + submissionTime: moment(MOCK_AUTOREPLY_PARAMS.submission.created) + .tz('Asia/Singapore') + .format('ddd, DD MMM YYYY hh:mm:ss A'), + } + const expectedMailBody = await MailUtils.generateAutoreplyHtml({ + autoReplyBody: DEFAULT_AUTO_REPLY_BODY, + ...expectedRenderData, + }) + + const expectedArg = { + ...defaultExpectedArg, + html: expectedMailBody, + // Attachments should be concatted with mock pdf response + attachments: [ + ...MOCK_AUTOREPLY_PARAMS.attachments, + { + content: MOCK_PDF, + filename: 'response.pdf', + }, + ], + } + + // Act + Assert + const expectedResponse = await Promise.allSettled([ + Promise.resolve(mockedResponse), + ]) + await expect( + mailService.sendAutoReplyEmails(customDataParams), + ).resolves.toEqual(expectedResponse) + // Check arguments passed to sendNodeMail + expect(sendMailSpy).toHaveBeenCalledTimes(1) + expect(sendMailSpy).toHaveBeenCalledWith(expectedArg) + }) + + it('should reject with error when autoReplyData.email param is an invalid email', async () => { + // Arrange + const invalidDataParams = cloneDeep(MOCK_AUTOREPLY_PARAMS) + invalidDataParams.autoReplyMailDatas[0].email = 'notAnEmail' + + // Act + Assert + const expectedResponse = await Promise.allSettled([ + Promise.reject(Error('Invalid email error')), + ]) + await expect( + mailService.sendAutoReplyEmails(invalidDataParams), + ).resolves.toEqual(expectedResponse) + }) + }) +}) diff --git a/tests/unit/backend/services/sms.service.spec.js b/tests/unit/backend/services/sms.service.spec.js index c879bebc52..b62845fc38 100644 --- a/tests/unit/backend/services/sms.service.spec.js +++ b/tests/unit/backend/services/sms.service.spec.js @@ -13,7 +13,7 @@ const TWILIO_TEST_NUMBER = '+15005550006' const MOCK_MSG_SRVC_SID = 'mockMsgSrvcSid' -describe('SmsService', () => { +describe('sms.service', () => { const MOCK_VALID_CONFIG = { msgSrvcSid: MOCK_MSG_SRVC_SID, client: { @@ -64,7 +64,7 @@ describe('SmsService', () => { // Return null on Form method spyOn(Form, 'getOtpData').and.returnValue(null) - expectAsync( + await expectAsync( SmsService.sendVerificationOtp( /* recipient= */ TWILIO_TEST_NUMBER, /* otp= */ '111111', diff --git a/tsconfig.json b/tsconfig.json index 6cf58318ae..2c6eb4cf34 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,7 +8,7 @@ // "incremental": true, /* Enable incremental compilation */ "target": "ES6", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ - // "lib": [], /* Specify library files to be included in the compilation. */ + "lib": ["DOM","ES6","ES2020"], /* Specify library files to be included in the compilation. */ "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ From bb852aa748daf27a219ca0ea8939a0674ac146a1 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Tue, 18 Aug 2020 21:51:08 +0800 Subject: [PATCH 02/27] feat: create bounce collection and alarms (#131) --- .template-env | 1 + docker-compose.yml | 1 + docs/DEPLOYMENT_SETUP.md | 39 +- .../email-submissions.server.controller.js | 56 -- src/app/models/bounce.server.model.ts | 156 ++++++ src/app/modules/sns/sns.controller.ts | 36 ++ src/app/modules/sns/sns.routes.ts | 18 + src/app/modules/sns/sns.service.ts | 138 +++++ src/app/routes/public-forms.server.routes.js | 28 - src/app/utils/sns.js | 140 ----- src/config/config.ts | 8 + src/config/defaults.ts | 6 + src/loaders/express/index.ts | 2 + src/types/bounce.ts | 21 + src/types/index.ts | 2 + src/types/sns.ts | 64 +++ tests/.test-full-env | 2 + ...mail-submissions.server.controller.spec.js | 181 +------ tests/unit/backend/helpers/jest-logger.ts | 26 + tests/unit/backend/helpers/sns.ts | 111 ++++ .../models/bounce.server.model.spec.ts | 178 +++++++ .../modules/sns/sns.controller.spec.ts | 59 +++ .../backend/modules/sns/sns.service.spec.ts | 498 ++++++++++++++++++ .../unit/backend/resources/valid-sns-body.js | 15 - 24 files changed, 1350 insertions(+), 436 deletions(-) create mode 100644 src/app/models/bounce.server.model.ts create mode 100644 src/app/modules/sns/sns.controller.ts create mode 100644 src/app/modules/sns/sns.routes.ts create mode 100644 src/app/modules/sns/sns.service.ts delete mode 100644 src/app/utils/sns.js create mode 100644 src/types/bounce.ts create mode 100644 src/types/sns.ts create mode 100644 tests/unit/backend/helpers/jest-logger.ts create mode 100644 tests/unit/backend/helpers/sns.ts create mode 100644 tests/unit/backend/models/bounce.server.model.spec.ts create mode 100644 tests/unit/backend/modules/sns/sns.controller.spec.ts create mode 100644 tests/unit/backend/modules/sns/sns.service.spec.ts delete mode 100644 tests/unit/backend/resources/valid-sns-body.js diff --git a/.template-env b/.template-env index e0e6e3c8d4..70603da9f3 100644 --- a/.template-env +++ b/.template-env @@ -30,6 +30,7 @@ FORMSG_SDK_MODE= ## App Config # APP_NAME=FormSG # OTP_LIFE_SPAN=900000 +# BOUNCE_LIFE_SPAN=1800000 # AGGREGATE_COLLECTION= # If provided, a banner with the provided message will show up in every form. diff --git a/docker-compose.yml b/docker-compose.yml index 88db3b8faa..1a7e6064aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: - IMAGE_S3_BUCKET=local-image-bucket - LOGO_S3_BUCKET=local-logo-bucket - FORMSG_SDK_MODE=development + - BOUNCE_LIFE_SPAN=1800000 - GA_TRACKING_ID - SENTRY_CONFIG_URL - TWILIO_ACCOUNT_SID diff --git a/docs/DEPLOYMENT_SETUP.md b/docs/DEPLOYMENT_SETUP.md index d2fa808ee6..8a05ebc864 100644 --- a/docs/DEPLOYMENT_SETUP.md +++ b/docs/DEPLOYMENT_SETUP.md @@ -98,25 +98,26 @@ The following env variables are set in Travis: #### App and Database -| Variable | Description | -| :----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `APP_NAME` | Application name in window title; also used as an identifier for MyInfo. Defaults to `'FormSG'`. | -| `APP_DESC` | Defaults to `'Form Manager for Government'`. | -| `APP_URL` | Defaults to `'https://form.gov.sg'`. | -| `APP_KEYWORDS` | Defaults to `'forms, formbuilder, nodejs'`. | -| `APP_IMAGES` | Defaults to `'/public/modules/core/img/og/img_metatag.png,/public/modules/core/img/og/logo-vertical-color.png'`. | -| `APP_TWITTER_IMAGE` | ath to Twitter image. Defaults to `'/public/modules/core/img/og/logo-vertical-color.png'`. | -| `DB_HOST` | A MongoDB URI. | -| `OTP_LIFE_SPAN` | Time in milliseconds that admin login OTP is valid for. Defaults to 900000ms or 15 minutes | -| `PORT` | Server port. Defaults to `5000`. | -| `NODE_ENV` | [Express environment mode](https://expressjs.com/en/advanced/best-practice-performance.html#set-node_env-to-production). Defaults to `'development'`. This should always be set to a production environment | -| `SESSION_SECRET` | Secret for `express-session`. Defaults to `'sandcrawler-138577'`. This should always be set in a production environment. | -| `SITE_BANNER_CONTENT` | If set, displays a banner message on both private routes that `ADMIN_BANNER_CONTENT` covers **and** public form routes that `IS_GENERAL_MAINTENANCE` covers. Overrides all other banner environment variables | -| `ADMIN_BANNER_CONTENT` | If set, displays a banner message on private admin routes such as the form list page as well as form builder pages. | -| `IS_GENERAL_MAINTENANCE` | If set, displays a banner message on all forms. Overrides `IS_SP_MAINTENANCE` and `IS_CP_MAINTENANCE`. | -| `IS_SP_MAINTENANCE` | If set, displays a banner message on SingPass forms. Overrides `IS_CP_MAINTENANCE`. | -| `IS_CP_MAINTENANCE` | If set, displays a banner message on SingPass forms. | -| `SUBMISSIONS_TOP_UP` | Use this to inflate the number of submissions displayed on the landing page. Defaults to `0`. | +| Variable | Description | +| :----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `APP_NAME` | Application name in window title; also used as an identifier for MyInfo. Defaults to `'FormSG'`. | +| `APP_DESC` | Defaults to `'Form Manager for Government'`. | +| `APP_URL` | Defaults to `'https://form.gov.sg'`. | +| `APP_KEYWORDS` | Defaults to `'forms, formbuilder, nodejs'`. | +| `APP_IMAGES` | Defaults to `'/public/modules/core/img/og/img_metatag.png,/public/modules/core/img/og/logo-vertical-color.png'`. | +| `APP_TWITTER_IMAGE` | ath to Twitter image. Defaults to `'/public/modules/core/img/og/logo-vertical-color.png'`. | +| `DB_HOST` | A MongoDB URI. | +| `OTP_LIFE_SPAN` | Time in milliseconds that admin login OTP is valid for. Defaults to 900000ms or 15 minutes. | +| `BOUNCE_LIFE_SPAN` | Time in milliseconds that bounces are tracked for each form. Defaults to 1800000ms or 30 minutes. Only relevant if you have set up AWS to send bounce and delivery notifications to the /emailnotifications endpoint. | +| `PORT` | Server port. Defaults to `5000`. | +| `NODE_ENV` | [Express environment mode](https://expressjs.com/en/advanced/best-practice-performance.html#set-node_env-to-production). Defaults to `'development'`. This should always be set to a production environment | +| `SESSION_SECRET` | Secret for `express-session`. Defaults to `'sandcrawler-138577'`. This should always be set in a production environment. | +| `SITE_BANNER_CONTENT` | If set, displays a banner message on both private routes that `ADMIN_BANNER_CONTENT` covers **and** public form routes that `IS_GENERAL_MAINTENANCE` covers. Overrides all other banner environment variables | +| `ADMIN_BANNER_CONTENT` | If set, displays a banner message on private admin routes such as the form list page as well as form builder pages. | +| `IS_GENERAL_MAINTENANCE` | If set, displays a banner message on all forms. Overrides `IS_SP_MAINTENANCE` and `IS_CP_MAINTENANCE`. | +| `IS_SP_MAINTENANCE` | If set, displays a banner message on SingPass forms. Overrides `IS_CP_MAINTENANCE`. | +| `IS_CP_MAINTENANCE` | If set, displays a banner message on SingPass forms. | +| `SUBMISSIONS_TOP_UP` | Use this to inflate the number of submissions displayed on the landing page. Defaults to `0`. | #### AWS services diff --git a/src/app/controllers/email-submissions.server.controller.js b/src/app/controllers/email-submissions.server.controller.js index ff206b05d9..c5465b39c2 100644 --- a/src/app/controllers/email-submissions.server.controller.js +++ b/src/app/controllers/email-submissions.server.controller.js @@ -9,13 +9,10 @@ const mongoose = require('mongoose') const { getEmailSubmissionModel } = require('../models/submission.server.model') const emailSubmission = getEmailSubmissionModel(mongoose) const HttpStatus = require('http-status-codes') - -const { isValidSnsRequest, parseSns } = require('../utils/sns') const { FIELDS_TO_REJECT } = require('../utils/field-validation/config') const { getParsedResponses } = require('../utils/response') const { getRequestIp } = require('../utils/request') const { ConflictError } = require('../utils/custom-errors') -const { EMAIL_TYPES } = require('../constants/mail') const { MB } = require('../constants/filesize') const { attachmentsAreValid, @@ -28,9 +25,6 @@ const config = require('../../config/config') const logger = require('../../config/logger').createLoggerWithLabel( 'email-submissions', ) -const emailLogger = require('../../config/logger').createCloudWatchLogger( - 'email', -) const MailService = require('../services/mail.service').default const { sessionSecret } = config @@ -622,53 +616,3 @@ exports.sendAdminEmail = async function (req, res, next) { return onSubmissionEmailFailure(err, req, res, submission) } } - -/** - * Validates that a request came from Amazon SNS. - * @param {Object} req Express request object - * @param {Object} res - Express response object - * @param {Object} next - the next expressjs callback, invoked once attachments - */ -exports.verifySns = async (req, res, next) => { - if (await isValidSnsRequest(req)) { - return next() - } - return res.sendStatus(HttpStatus.FORBIDDEN) -} - -/** - * When email bounces, SNS calls this function to mark the - * submission as having bounced. - * - * Note that if anything errors in between, just return a 200 - * to SNS, as the error code to them doesn't really matter. - * - * @param {Object} req Express request object - * @param {Object} res Express response object - */ -exports.confirmOnNotification = function (req, res) { - const parsed = parseSns(req.body) - // Log to short-lived CloudWatch log group - emailLogger.info(parsed) - const { submissionId, notificationType, emailType } = parsed - if ( - notificationType !== 'Bounce' || - emailType !== EMAIL_TYPES.adminResponse || - !submissionId - ) { - return res.sendStatus(HttpStatus.OK) - } - // Mark submission ID as having bounced - emailSubmission.findOneAndUpdate( - { _id: submissionId }, - { - hasBounced: true, - }, - function (err) { - if (err) { - logger.warn(err) - } - }, - ) - return res.sendStatus(HttpStatus.OK) -} diff --git a/src/app/models/bounce.server.model.ts b/src/app/models/bounce.server.model.ts new file mode 100644 index 0000000000..05effe1264 --- /dev/null +++ b/src/app/models/bounce.server.model.ts @@ -0,0 +1,156 @@ +import { get } from 'lodash' +import { Model, Mongoose, Schema } from 'mongoose' +import validator from 'validator' + +import { bounceLifeSpan } from '../../config/config' +import { + IBounceNotification, + IBounceSchema, + IEmailNotification, + isBounceNotification, + isDeliveryNotification, + ISingleBounce, +} from '../../types' +import { EMAIL_HEADERS, EMAIL_TYPES } from '../constants/mail' + +import { FORM_SCHEMA_ID } from './form.server.model' + +export const BOUNCE_SCHEMA_ID = 'Bounce' + +export interface IBounceModel extends Model { + fromSnsNotification: (snsInfo: IEmailNotification) => IBounceSchema | null +} + +const BounceSchema = new Schema({ + formId: { + type: Schema.Types.ObjectId, + ref: FORM_SCHEMA_ID, + required: 'Form ID is required', + }, + hasAlarmed: { + type: Boolean, + default: false, + }, + bounces: { + type: [ + { + email: { + type: String, + trim: true, + required: true, + validate: { + validator: validator.isEmail, + message: 'Bounced email must be a valid email address', + }, + }, + hasBounced: { + type: Boolean, + default: false, + }, + _id: false, + }, + ], + }, + expireAt: { + type: Date, + default: () => new Date(Date.now() + bounceLifeSpan), + }, +}) +BounceSchema.index({ expireAt: 1 }, { expireAfterSeconds: 0 }) + +// Helper function for methods. +// Extracts custom headers which we send with all emails, such as form ID, submission ID +// and email type (admin response, email confirmation OTP etc). +const extractHeader = (body: IEmailNotification, header: string): string => { + return get(body, 'mail.headers').find( + (mailHeader) => mailHeader.name.toLowerCase() === header.toLowerCase(), + )?.value +} + +// Helper function for methods. +// Whether a bounce notification says a given email has bounced +const hasEmailBounced = ( + bounceInfo: IBounceNotification, + email: string, +): boolean => { + return get(bounceInfo, 'bounce.bouncedRecipients').some( + (emailInfo) => emailInfo.emailAddress === email, + ) +} + +// Create a new Bounce document from an SNS notification. +// More info on format of SNS notifications: +// https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html +BounceSchema.statics.fromSnsNotification = function ( + this: IBounceModel, + snsInfo: IEmailNotification, +): IBounceSchema | null { + const emailType = extractHeader(snsInfo, EMAIL_HEADERS.emailType) + const formId = extractHeader(snsInfo, EMAIL_HEADERS.formId) + // We only care about admin emails + if (emailType !== EMAIL_TYPES.adminResponse || !formId) { + return null + } + const isBounce = isBounceNotification(snsInfo) + const bounces: ISingleBounce[] = get(snsInfo, 'mail.commonHeaders.to').map( + (email) => { + if (isBounce && hasEmailBounced(snsInfo as IBounceNotification, email)) { + return { email, hasBounced: true } + } else { + return { email, hasBounced: false } + } + }, + ) + return new this({ formId, bounces }) +} + +// Updates an old bounce document with info from a new bounce document as well +// as an SNS notification. This function does 3 things: +// 1) If the old bounce document indicates that an email bounced, set hasBounced +// to true for that email. +// 2) If the new delivery notification indicates that an email was delivered +// successfully, set hasBounced to false for that email, even if the old bounce +// document indicates that that email previously bounced. +// 3) Update the old recipient list according to the newest bounce notification. +BounceSchema.methods.merge = function ( + this: IBounceSchema, + latestBounces: IBounceSchema, + snsInfo: IEmailNotification, +): void { + const isDelivery = isDeliveryNotification(snsInfo) + this.bounces.forEach((oldBounce) => { + // If we were previously notified that a given email has bounced, + // we want to retain that information + if (oldBounce.hasBounced) { + // Check if the latest recipient list contains that email + const matchedLatestBounce = latestBounces.bounces.find( + (newBounce) => newBounce.email === oldBounce.email, + ) + // Check if the latest notification indicates that this email + // actually succeeded. We can't just use latestBounces because + // a false in latestBounces doesn't guarantee that the email was + // delivered, only that the email has not bounced yet. + const hasSubsequentlySucceeded = + isDelivery && + get(snsInfo, 'delivery.recipients').includes(oldBounce.email) + if (matchedLatestBounce) { + // Set the latest bounce status based on the latest notification + matchedLatestBounce.hasBounced = !hasSubsequentlySucceeded + } + } + }) + this.bounces = latestBounces.bounces +} + +const getBounceModel = (db: Mongoose) => { + try { + return db.model(BOUNCE_SCHEMA_ID) as IBounceModel + } catch { + return db.model( + BOUNCE_SCHEMA_ID, + BounceSchema, + ) as IBounceModel + } +} + +export default getBounceModel diff --git a/src/app/modules/sns/sns.controller.ts b/src/app/modules/sns/sns.controller.ts new file mode 100644 index 0000000000..10d0b55a35 --- /dev/null +++ b/src/app/modules/sns/sns.controller.ts @@ -0,0 +1,36 @@ +import { Request, Response } from 'express' +import HttpStatus from 'http-status-codes' + +import { createLoggerWithLabel } from '../../../config/logger' +import { ISnsNotification } from '../../../types' + +import * as snsService from './sns.service' + +const logger = createLoggerWithLabel('sns-controller') +/** + * Validates that a request came from Amazon SNS, then updates the Bounce + * collection. + * @param req Express request object + * @param res - Express response object + */ +const handleSns = async ( + req: Request<{}, {}, ISnsNotification>, + res: Response, +) => { + // Since this function is for a public endpoint, catch all possible errors + // so we never fail on malformed input. The response code is meaningless since + // it is meant to go back to AWS. + try { + const isValid = await snsService.isValidSnsRequest(req.body) + if (!isValid) { + return res.sendStatus(HttpStatus.FORBIDDEN) + } + await snsService.updateBounces(req.body) + return res.sendStatus(HttpStatus.OK) + } catch (err) { + logger.warn(err) + return res.sendStatus(HttpStatus.BAD_REQUEST) + } +} + +export default handleSns diff --git a/src/app/modules/sns/sns.routes.ts b/src/app/modules/sns/sns.routes.ts new file mode 100644 index 0000000000..34c0072002 --- /dev/null +++ b/src/app/modules/sns/sns.routes.ts @@ -0,0 +1,18 @@ +import { Express } from 'express' + +import handleSns from './sns.controller' + +const mountSnsRoutes = (app: Express) => { + /** + * When email bounces, SNS calls this function to mark the + * submission as having bounced. + * + * Note that if anything errors in between, just return a 200 + * to SNS, as the error code to them doesn't really matter. + * + * @route POST /emailnotifications + */ + app.route('/emailnotifications').post(handleSns) +} + +export default mountSnsRoutes diff --git a/src/app/modules/sns/sns.service.ts b/src/app/modules/sns/sns.service.ts new file mode 100644 index 0000000000..976988d048 --- /dev/null +++ b/src/app/modules/sns/sns.service.ts @@ -0,0 +1,138 @@ +import axios from 'axios' +import crypto from 'crypto' +import { isEmpty } from 'lodash' +import mongoose from 'mongoose' + +import { createCloudWatchLogger } from '../../../config/logger' +import { + IBounceSchema, + IEmailNotification, + ISnsNotification, +} from '../../../types' +import getBounceModel from '../../models/bounce.server.model' + +const logger = createCloudWatchLogger('email') +const Bounce = getBounceModel(mongoose) + +// Note that these need to be ordered in order to generate +// the correct string to sign +const snsKeys: { key: keyof ISnsNotification; toSign: boolean }[] = [ + { key: 'Message', toSign: true }, + { key: 'MessageId', toSign: true }, + { key: 'Timestamp', toSign: true }, + { key: 'TopicArn', toSign: true }, + { key: 'Type', toSign: true }, + { key: 'Signature', toSign: false }, + { key: 'SigningCertURL', toSign: false }, + { key: 'SignatureVersion', toSign: false }, +] + +// Hostname for AWS URLs +const AWS_HOSTNAME = '.amazonaws.com' + +/** + * Checks that a request body has all the required keys for a message from SNS. + * @param {Object} body body from Express request object + */ +const hasRequiredKeys = (body: any): body is ISnsNotification => { + return !isEmpty(body) && snsKeys.every((keyObj) => body[keyObj.key]) +} + +/** + * Validates that a URL points to a certificate belonging to AWS. + * @param {String} url URL to check + */ +const isValidCertUrl = (certUrl: string): boolean => { + const parsed = new URL(certUrl) + return ( + parsed.protocol === 'https:' && + parsed.pathname.endsWith('.pem') && + parsed.hostname.endsWith(AWS_HOSTNAME) + ) +} + +/** + * Returns an ordered list of keys to include in SNS signing string. + */ +const getSnsKeysToSign = (): (keyof ISnsNotification)[] => { + return snsKeys.filter((keyObj) => keyObj.toSign).map((keyObj) => keyObj.key) +} + +/** + * Generates the string to sign. + * @param {Object} body body from Express request object + */ +const getSnsBasestring = (body: ISnsNotification): string => { + return getSnsKeysToSign().reduce((result, key) => { + return result + key + '\n' + body[key] + '\n' + }, '') +} + +/** + * Verify signature for SNS request + * @param {Object} body body from Express request object + */ +const isValidSnsSignature = async ( + body: ISnsNotification, +): Promise => { + const { data: cert } = await axios.get(body.SigningCertURL) + const verifier = crypto.createVerify('RSA-SHA1') + verifier.update(getSnsBasestring(body), 'utf8') + return verifier.verify(cert, body.Signature, 'base64') +} + +/** + * Verifies if a request object is correctly signed by Amazon SNS. More info: + * https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html + * @param {Object} body Body of Express request object + */ +export const isValidSnsRequest = async ( + body: ISnsNotification, +): Promise => { + const isValid = + hasRequiredKeys(body) && + body.SignatureVersion === '1' && // We only check for SHA1-RSA signatures + isValidCertUrl(body.SigningCertURL) && + (await isValidSnsSignature(body)) + return isValid +} + +// Writes a log message if all recipients have bounced +const logCriticalBounce = (bounceInfo: IBounceSchema, formId: string): void => { + if ( + !bounceInfo.hasAlarmed && + bounceInfo.bounces.every((emailInfo) => emailInfo.hasBounced) + ) { + logger.warn({ + type: 'CRITICAL BOUNCE', + formId, + recipients: bounceInfo.bounces.map((emailInfo) => emailInfo.email), + }) + // We don't want a flood of logs and alarms, so we use this to limit the rate of + // critical bounce logs for each form ID + bounceInfo.hasAlarmed = true + } +} + +/** + * Parses an SNS notification and updates the Bounce collection. + * @param body The request body of the notification + */ +export const updateBounces = async (body: ISnsNotification): Promise => { + const notification: IEmailNotification = JSON.parse(body.Message) + // This is the crucial log statement which allows us to debug bounce-related + // issues, as it logs all the details about deliveries and bounces + logger.info(notification) + const latestBounces = Bounce.fromSnsNotification(notification) + if (!latestBounces) return + const formId = latestBounces.formId + const oldBounces = await Bounce.findOne({ formId }) + if (oldBounces) { + oldBounces.merge(latestBounces, notification) + logCriticalBounce(oldBounces, formId) + await oldBounces.save() + } else { + logCriticalBounce(latestBounces, formId) + await latestBounces.save() + } +} diff --git a/src/app/routes/public-forms.server.routes.js b/src/app/routes/public-forms.server.routes.js index 8f90db6ec9..7359a06beb 100644 --- a/src/app/routes/public-forms.server.routes.js +++ b/src/app/routes/public-forms.server.routes.js @@ -113,34 +113,6 @@ module.exports = function (app) { forms.read(forms.REQUEST_TYPE.PUBLIC), ) - /** - * @typedef Mail - * @property {Array.} headers - an array of mail headers - */ - - /** - * @typedef BouncedMessage - * @property {Mail.model} mail - the bounced mail - */ - - /** - * When email bounces, SNS calls this function to mark the - * submission as having bounced. - * - * Note that if anything errors in between, just return a 200 - * to SNS, as the error code to them doesn't really matter. - * - * @route POST /emailnotifications - * @group forms - endpoints to serve forms - * @param {BouncedMessage.model} Message.body.required - the bounced message - * @consumes application/json - * @produces application/json - * @returns {string} 200 - notification acknowledged - */ - app - .route('/emailnotifications') - .post(emailSubmissions.verifySns, emailSubmissions.confirmOnNotification) - /** * @typedef SubmissionResponse * @property {string} message.required - a human-readable message diff --git a/src/app/utils/sns.js b/src/app/utils/sns.js deleted file mode 100644 index 89ed1e55d6..0000000000 --- a/src/app/utils/sns.js +++ /dev/null @@ -1,140 +0,0 @@ -const axios = require('axios') -const crypto = require('crypto') -const get = require('lodash/get') -const { EMAIL_HEADERS } = require('../constants/mail') - -// Note that these need to be ordered in order to generate -// the correct string to sign -const snsKeys = [ - { key: 'Message', toSign: true }, - { key: 'MessageId', toSign: true }, - { key: 'Timestamp', toSign: true }, - { key: 'TopicArn', toSign: true }, - { key: 'Type', toSign: true }, - { key: 'Signature', toSign: false }, - { key: 'SigningCertURL', toSign: false }, - { key: 'SignatureVersion', toSign: false }, -] - -const EMAIL_LOWERCASE_HEADER_TO_KEY = (() => { - // EMAIL_HEADERS with keys and values swapped and the new keys (e.g. X-Formsg-Form-ID) - // changed to lowercase (x-formsg-form-id). - // NOTE: ALWAYS DO CASE-INSENSITIVE CHECKS FOR THE HEADERS! - const lowercasedHeadersToKey = {} - for (const [key, value] of Object.entries(EMAIL_HEADERS)) { - lowercasedHeadersToKey[value.toLowerCase()] = key - } - return lowercasedHeadersToKey -})() - -// Hostname for AWS URLs -const AWS_HOSTNAME = '.amazonaws.com' - -/** - * Checks that a request body has all the required keys for a message from SNS. - * @param {Object} body body from Express request object - */ -const hasRequiredKeys = (body) => { - return snsKeys.every((keyObj) => body[keyObj.key]) -} - -/** - * Validates that a URL points to a certificate belonging to AWS. - * @param {String} url URL to check - */ -const isValidCertUrl = (certUrl) => { - const parsed = new URL(certUrl) - return ( - parsed.protocol === 'https:' && - parsed.pathname.endsWith('.pem') && - parsed.hostname.endsWith(AWS_HOSTNAME) - ) -} - -/** - * Returns an ordered list of keys to include in SNS signing string. - */ -const getSnsKeysToSign = () => { - return snsKeys.filter((keyObj) => keyObj.toSign).map((keyObj) => keyObj.key) -} - -/** - * Generates the string to sign. - * @param {Object} body body from Express request object - */ -const getSnsBasestring = (body) => { - return getSnsKeysToSign().reduce((result, key) => { - return result + key + '\n' + body[key] + '\n' - }, '') -} - -/** - * Verify signature for SNS request - * @param {Object} body body from Express request object - */ -const isValidSnsSignature = async (body) => { - const { data: cert } = await axios.get(body.SigningCertURL) - const verifier = crypto.createVerify('RSA-SHA1') - verifier.update(getSnsBasestring(body), 'utf-8') - return verifier.verify(cert, body.Signature, 'base64') -} - -/** - * Verifies if a request object is correctly signed by Amazon SNS. - * @param {Object} req Express request object - */ -const isValidSnsRequest = async (req) => { - const isValid = - hasRequiredKeys(req.body) && - req.body.SignatureVersion === '1' && // We only check for SHA1-RSA signatures - isValidCertUrl(req.body.SigningCertURL) && - (await isValidSnsSignature(req.body)) - return isValid -} - -/** - * Parses the POST body of an SNS notification for SES - * emails. Returns an object with the important keys. - * Structure of notification can be found at - * https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html - * @param {Object} body POST body of SNS request - * @returns {Object} Object containing keys parsed from POST body or empty object if parsing fails - */ -const parseSns = (body) => { - try { - // Extract relevant values - const content = JSON.parse(body.Message) - const parsed = {} - parsed.to = get(content, 'mail.commonHeaders.to') - parsed.subject = get(content, 'mail.commonHeaders.subject') - parsed.notificationType = content.notificationType - if (parsed.notificationType === 'Bounce') { - parsed.bounceType = get(content, 'bounce.bounceType') - parsed.bouncedEmails = get(content, 'bounce.bouncedRecipients').map( - (info) => info.emailAddress, - ) - } - // Custom headers which we send with all emails, such as form ID, submission ID - // and email type (admin response, email confirmation OTP etc). - // e.g. if header.name === 'X-Formsg-Form-ID', we want to set - // parsed.formId = header.value - const headers = get(content, 'mail.headers') - headers.forEach((header) => { - const customHeaderKey = - EMAIL_LOWERCASE_HEADER_TO_KEY[header.name.toLowerCase()] - if (customHeaderKey) { - parsed[customHeaderKey] = header.value - } - }) - return parsed - } catch (err) { - // Could not parse - return {} - } -} - -module.exports = { - getSnsBasestring, - isValidSnsRequest, - parseSns, -} diff --git a/src/config/config.ts b/src/config/config.ts index 7ef05c06ca..cc4ef03a74 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -63,6 +63,7 @@ type Config = { cspReportUri: string chromiumBin: string otpLifeSpan: number + bounceLifeSpan: number formsgSdkMode: PackageMode submissionsTopUp: number customCloudWatchGroup?: string @@ -98,6 +99,12 @@ const sessionSecret = process.env.SESSION_SECRET || defaults.app.sessionSecret const otpLifeSpan = parseInt(process.env.OTP_LIFE_SPAN, 10) || defaults.login.otpLifeSpan +/** + * TTL of bounce documents in milliseconds. + */ +const bounceLifeSpan = + parseInt(process.env.BOUNCE_LIFE_SPAN, 10) || defaults.bounce.bounceLifeSpan + /** * Number of submissions to top up submissions statistic by */ @@ -406,6 +413,7 @@ const config: Config = { customCloudWatchGroup, sessionSecret, otpLifeSpan, + bounceLifeSpan, formsgSdkMode, chromiumBin, cspReportUri, diff --git a/src/config/defaults.ts b/src/config/defaults.ts index 86199c4de4..d2bbc78ac5 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -47,6 +47,11 @@ const LINKS = { supportFormLink: 'https://go.gov.sg/formsg-support', } +const BOUNCE_CONFIG = { + // TTL of Bounce document in milliseconds + bounceLifeSpan: 1800000, +} + export default { app: APP_CONFIG, login: LOGIN_CONFIG, @@ -54,4 +59,5 @@ export default { aws: AWS_CONFIG, ses: SES_CONFIG, links: LINKS, + bounce: BOUNCE_CONFIG, } diff --git a/src/loaders/express/index.ts b/src/loaders/express/index.ts index c72930140a..3d04371170 100644 --- a/src/loaders/express/index.ts +++ b/src/loaders/express/index.ts @@ -7,6 +7,7 @@ import { Connection } from 'mongoose' import path from 'path' import url from 'url' +import mountSnsRoutes from '../../app/modules/sns/sns.routes' import apiRoutes from '../../app/routes' import config from '../../config/config' @@ -120,6 +121,7 @@ const loadExpressApp = async (connection: Connection) => { apiRoutes.forEach(function (routeFunction) { routeFunction(app) }) + mountSnsRoutes(app) app.use(sentryMiddlewares()) diff --git a/src/types/bounce.ts b/src/types/bounce.ts new file mode 100644 index 0000000000..75f1c5ec04 --- /dev/null +++ b/src/types/bounce.ts @@ -0,0 +1,21 @@ +import { Document } from 'mongoose' + +import { IFormSchema } from './form' +import { IEmailNotification } from './sns' + +export interface ISingleBounce { + email: string + hasBounced: boolean +} + +export interface IBounce { + formId: IFormSchema['_id'] + bounces: ISingleBounce[] + hasAlarmed: boolean + expireAt: Date + _id: Document['_id'] +} + +export interface IBounceSchema extends IBounce, Document { + merge: (latestBounces: IBounceSchema, snsInfo: IEmailNotification) => void +} diff --git a/src/types/index.ts b/src/types/index.ts index 12ad98c288..088c8d6436 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,6 @@ export * from './field' export * from './agency' +export * from './bounce' export * from './form_feedback' export * from './form_logic' export * from './form_logo' @@ -9,6 +10,7 @@ export * from './login' export * from './myinfo_hash' export * from './response' export * from './sms_count' +export * from './sns' export * from './submission' export * from './token' export * from './user' diff --git a/src/types/sns.ts b/src/types/sns.ts new file mode 100644 index 0000000000..a3873fbdb0 --- /dev/null +++ b/src/types/sns.ts @@ -0,0 +1,64 @@ +/** + * More info on SNS verification: + * https://docs.aws.amazon.com/sns/latest/dg/sns-verify-signature-of-message.html + */ +export interface ISnsNotification { + Message: string + MessageId: string + Timestamp: string + TopicArn: string + Type: string + Signature: string + SigningCertURL: string + SignatureVersion: string +} + +/** + * The structure of SNS notifications for SES can be found here: + * https://docs.aws.amazon.com/ses/latest/DeveloperGuide/notification-contents.html + * This interface only contains the more important fields. + */ +export interface IEmailNotification { + notificationType: 'Bounce' | 'Complaint' | 'Delivery' + mail: { + source: string + destination: string[] + headers: { + name: string + value: string + }[] + commonHeaders: { + from: string + to: string[] + subject: string + } + } +} + +export interface IBounceNotification extends IEmailNotification { + notificationType: 'Bounce' + bounce: { + bounceType: string + bounceSubType: string + bouncedRecipients: { + emailAddress: string + }[] + } +} + +export interface IDeliveryNotification extends IEmailNotification { + notificationType: 'Delivery' + delivery: { + recipients: string[] + } +} + +// If an email notification is for bounces +export const isBounceNotification = ( + body: IEmailNotification, +): body is IBounceNotification => body.notificationType === 'Bounce' + +// If an email notification is for successful delivery +export const isDeliveryNotification = ( + body: IEmailNotification, +): body is IDeliveryNotification => body.notificationType === 'Delivery' diff --git a/tests/.test-full-env b/tests/.test-full-env index 2ca8b688fa..4d46aa8fdb 100644 --- a/tests/.test-full-env +++ b/tests/.test-full-env @@ -69,3 +69,5 @@ MONGO_BINARY_VERSION=3.6.12 AWS_ACCESS_KEY_ID=fakeAccessKeyId AWS_SECRET_ACCESS_KEY=fakeSecretAccessKey + +BOUNCE_LIFE_SPAN=1800000 diff --git a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js index a841c1a57a..f3679df6d0 100644 --- a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js +++ b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js @@ -1,16 +1,12 @@ const HttpStatus = require('http-status-codes') -const { cloneDeep, times } = require('lodash') -const axios = require('axios') -const MockAdapter = require('axios-mock-adapter') -const crypto = require('crypto') +const { times } = require('lodash') const ejs = require('ejs') const express = require('express') const request = require('supertest') const mongoose = require('mongoose') const dbHandler = require('../helpers/db-handler') -const { validSnsBody } = require('../resources/valid-sns-body') -const { getSnsBasestring } = require('../../../../dist/backend/app/utils/sns') + const { ObjectID } = require('bson-ext') const MailService = require('../../../../dist/backend/app/services/mail.service') .default @@ -19,8 +15,6 @@ const User = dbHandler.makeModel('user.server.model', 'User') const Agency = dbHandler.makeModel('agency.server.model', 'Agency') const Form = dbHandler.makeModel('form.server.model', 'Form') const EmailForm = mongoose.model('email') -const Submission = dbHandler.makeModel('submission.server.model', 'Submission') -const emailSubmission = mongoose.model('emailSubmission') describe('Email Submissions Controller', () => { const SESSION_SECRET = 'secret' @@ -28,6 +22,7 @@ describe('Email Submissions Controller', () => { // Declare global variables let spyRequest let sendSubmissionMailSpy + // spec out controller such that calls to request are // directed through a callback to the request spy, // which will be destroyed and re-created for every test @@ -171,176 +166,6 @@ describe('Email Submissions Controller', () => { }) }) - describe('verifySNS', () => { - let req, res, next, privateKey - let mockAxios - beforeAll(() => { - mockAxios = new MockAdapter(axios) - const keys = crypto.generateKeyPairSync('rsa', { - modulusLength: 2048, - publicKeyEncoding: { - type: 'pkcs1', - format: 'pem', - }, - privateKeyEncoding: { - type: 'pkcs8', - format: 'pem', - }, - }) - privateKey = keys.privateKey - mockAxios.onGet(validSnsBody.SigningCertURL).reply(200, keys.publicKey) - }) - beforeEach(() => { - req = { body: cloneDeep(validSnsBody) } - res = jasmine.createSpyObj('res', ['sendStatus']) - next = jasmine.createSpy() - }) - afterAll(() => { - mockAxios.restore() - }) - const verifyFailure = async () => { - await controller.verifySns(req, res, next) - expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.FORBIDDEN) - expect(next).not.toHaveBeenCalled() - } - const verifySuccess = async () => { - await controller.verifySns(req, res, next) - expect(res.sendStatus).not.toHaveBeenCalled() - expect(next).toHaveBeenCalled() - } - it('should reject requests without valid structure', async () => { - delete req.body.Type - await verifyFailure() - }) - it('should reject requests with invalid certificate URL', async () => { - req.body.SigningCertURL = 'http://www.example.com' - await verifyFailure() - }) - it('should reject requests with invalid signature version', async () => { - req.body.SignatureVersion = 'wrongSignatureVersion' - await verifyFailure() - }) - it('should reject requests with invalid signature', async () => { - await verifyFailure() - }) - it('should accept valid requests', async () => { - const signer = crypto.createSign('RSA-SHA1') - signer.write(getSnsBasestring(req.body)) - req.body.Signature = signer.sign(privateKey, 'base64') - await verifySuccess() - }) - }) - - describe('confirmOnNotification', () => { - let submission - let content - const bodyParser = require('body-parser') - const app = express() - const endpointPath = '/emailnotifications' - const message = (json) => ({ Message: JSON.stringify(json) }) - - beforeAll(() => { - app - .route(endpointPath) - .post(bodyParser.json(), controller.confirmOnNotification, (req, res) => - res.sendStatus(HttpStatus.OK), - ) - }) - - beforeEach(async () => { - // Clear mock db and insert test form before each test - await dbHandler.clearDatabase() - const { form } = await dbHandler.preloadCollections() - - submission = await emailSubmission.create({ - form: form._id, - responseHash: 'hash', - responseSalt: 'salt', - }) - - content = { - notificationType: 'Bounce', - mail: { - headers: [ - { - name: 'X-Formsg-Form-ID', - value: form._id, - }, - { - name: 'X-Formsg-Submission-ID', - value: submission._id, - }, - { - name: 'X-Formsg-Email-Type', - value: 'Admin (response)', - }, - ], - commonHeaders: { subject: `Title (Ref: ${submission._id})` }, - }, - bounce: { - bounceType: 'Transient', - bouncedRecipients: [{ emailAddress: 'fake@email.gov.sg' }], - }, - } - }) - - afterAll(async () => await dbHandler.clearDatabase()) - - const expectNotifySubmissionHasBounced = (json, hasBounced, done) => { - request(app) - .post(endpointPath) - .send(message(json)) - .expect(HttpStatus.OK) - .then(() => Submission.findOne({ _id: submission._id })) - .then((s) => { - expect(s.hasBounced).toEqual(hasBounced) - }) - .then(done) - .catch(done) - } - - it('fails silently on bad payload', (done) => { - request(app).post(endpointPath).expect(HttpStatus.OK).end(done) - }) - - it('exits early on irrelevant payload', (done) => { - expectNotifySubmissionHasBounced( - { notificationType: 'Success' }, - false, - done, - ) - }) - - it('exits early by malformed bounce missing mail key', (done) => { - delete content.mail - expectNotifySubmissionHasBounced(content, false, done) - }) - - it('exits early by malformed bounce missing headers', (done) => { - delete content.mail.headers - expectNotifySubmissionHasBounced(content, false, done) - }) - - it('exits early by malformed bounce missing bounce key', (done) => { - delete content.bounce - expectNotifySubmissionHasBounced(content, false, done) - }) - - it('exits early by malformed bounce missing submission ID', (done) => { - content.mail.headers.splice(1, 1) - expectNotifySubmissionHasBounced(content, false, done) - }) - - it('exits early by malformed bounce missing email type', (done) => { - content.mail.headers.splice(2, 1) - expectNotifySubmissionHasBounced(content, false, done) - }) - - it('marks submission has bounced', (done) => { - expectNotifySubmissionHasBounced(content, true, done) - }) - }) - describe('receiveEmailSubmissionUsingBusBoy', () => { const endpointPath = '/v2/submissions/email' const sendSubmissionBack = (req, res) => { diff --git a/tests/unit/backend/helpers/jest-logger.ts b/tests/unit/backend/helpers/jest-logger.ts new file mode 100644 index 0000000000..359bd80214 --- /dev/null +++ b/tests/unit/backend/helpers/jest-logger.ts @@ -0,0 +1,26 @@ +import winston, { Logger } from 'winston' + +interface MockLogger extends Logger { + log: jest.Mock + warn: jest.Mock + info: jest.Mock + profile: jest.Mock +} + +const getMockLogger = (): MockLogger => { + const logger = winston.createLogger() + logger.log = jest.fn() + logger.warn = jest.fn() + logger.info = jest.fn() + logger.profile = jest.fn() + return logger as MockLogger +} + +export const resetMockLogger = (logger: MockLogger) => { + logger.log.mockReset() + logger.warn.mockReset() + logger.info.mockReset() + logger.profile.mockReset() +} + +export default getMockLogger diff --git a/tests/unit/backend/helpers/sns.ts b/tests/unit/backend/helpers/sns.ts new file mode 100644 index 0000000000..4b69b3a71a --- /dev/null +++ b/tests/unit/backend/helpers/sns.ts @@ -0,0 +1,111 @@ +import { ObjectId } from 'bson' +import { cloneDeep, merge, pick } from 'lodash' + +import { + IBounce, + IBounceNotification, + IBounceSchema, + IDeliveryNotification, + IEmailNotification, + ISnsNotification, +} from 'src/types' + +export const MOCK_SNS_BODY: ISnsNotification = { + Type: 'type', + MessageId: 'message-id', + TopicArn: 'topic-arn', + Message: 'message', + Timestamp: 'timestamp', + SignatureVersion: '1', + Signature: 'signature', + SigningCertURL: 'https://fakeawsurl.amazonaws.com/cert.pem', +} + +const makeEmailNotification = ( + notificationType: 'Bounce' | 'Delivery', + formId: ObjectId, + submissionId: ObjectId, + recipientList: string[], +): IEmailNotification => { + return { + notificationType, + mail: { + source: 'donotreply@form.gov.sg', + destination: recipientList, + headers: [ + { + name: 'X-Formsg-Form-ID', + value: String(formId), + }, + { + name: 'X-Formsg-Submission-ID', + value: String(submissionId), + }, + { + name: 'X-Formsg-Email-Type', + value: 'Admin (response)', + }, + ], + commonHeaders: { + subject: `Title (Ref: ${submissionId})`, + to: recipientList, + from: 'donotreply@form.gov.sg', + }, + }, + } +} + +export const makeBounceNotification = ( + formId: ObjectId = new ObjectId(), + submissionId: ObjectId = new ObjectId(), + recipientList: string[] = [], + bouncedList: string[] = [], + bounceType: 'Transient' | 'Permanent' = 'Permanent', +): ISnsNotification => { + const Message = merge( + makeEmailNotification('Bounce', formId, submissionId, recipientList), + { + bounce: { + bounceType, + bouncedRecipients: bouncedList.map((emailAddress) => ({ + emailAddress, + })), + }, + }, + ) as IBounceNotification + const body = cloneDeep(MOCK_SNS_BODY) + body.Message = JSON.stringify(Message) + return body +} + +export const makeDeliveryNotification = ( + formId: ObjectId = new ObjectId(), + submissionId: ObjectId = new ObjectId(), + recipientList: string[] = [], + deliveredList: string[] = [], +): ISnsNotification => { + const Message = merge( + makeEmailNotification('Delivery', formId, submissionId, recipientList), + { + delivery: { + recipients: deliveredList, + }, + }, + ) as IDeliveryNotification + const body = cloneDeep(MOCK_SNS_BODY) + body.Message = JSON.stringify(Message) + return body +} + +// Omit mongoose values from Bounce document +export const extractBounceObject = ( + bounce: IBounceSchema, +): Omit => { + const extracted = pick(bounce.toObject(), [ + 'formId', + 'hasAlarmed', + 'expireAt', + 'bounces', + ]) + return extracted +} diff --git a/tests/unit/backend/models/bounce.server.model.spec.ts b/tests/unit/backend/models/bounce.server.model.spec.ts new file mode 100644 index 0000000000..3836698f02 --- /dev/null +++ b/tests/unit/backend/models/bounce.server.model.spec.ts @@ -0,0 +1,178 @@ +import { ObjectId } from 'bson' +import { omit, pick } from 'lodash' +import mongoose from 'mongoose' + +import getBounceModel from 'src/app/models/bounce.server.model' + +import dbHandler from '../helpers/jest-db' +import { + extractBounceObject, + makeBounceNotification, + makeDeliveryNotification, +} from '../helpers/sns' + +const Bounce = getBounceModel(mongoose) + +const MOCK_EMAIL = 'email@email.com' + +describe('Bounce Model', () => { + beforeAll(async () => await dbHandler.connect()) + afterEach(async () => await dbHandler.clearDatabase()) + afterAll(async () => await dbHandler.closeDatabase()) + + describe('schema', () => { + test('should create and save successfully with defaults', async () => { + const formId = new ObjectId() + const bounces = [{ email: MOCK_EMAIL }] + const savedBounce = await new Bounce({ formId, bounces }).save() + const savedBounceObject = extractBounceObject(savedBounce) + expect(savedBounce._id).toBeDefined() + expect(savedBounce.expireAt).toBeInstanceOf(Date) + expect(omit(savedBounceObject, 'expireAt')).toEqual({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: false }], + hasAlarmed: false, + }) + }) + + test('should create and save successfully with non-defaults', async () => { + const params = { + formId: new ObjectId(), + bounces: [{ email: MOCK_EMAIL, hasBounced: true }], + expireAt: new Date(Date.now()), + hasAlarmed: true, + } + const savedBounce = await new Bounce(params).save() + const savedBounceObject = extractBounceObject(savedBounce) + expect(savedBounceObject).toEqual(params) + }) + + test('should reject invalid emails', async () => { + const bounce = new Bounce({ + formId: new ObjectId(), + bounces: [{ email: 'this is an ex-parrot' }], + }) + await expect(bounce.save()).rejects.toThrow() + }) + + test('should reject empty form IDs', async () => { + const bounce = new Bounce() + await expect(bounce.save()).rejects.toThrowError('Form ID is required') + }) + }) + + describe('methods', () => { + describe('merge', () => { + test('should update old bounce with latest bounce info', async () => { + const formId = new ObjectId() + const oldBounce = new Bounce({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: false }], + }) + const latestBounce = new Bounce({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: true }], + }) + const snsInfo = JSON.parse(makeBounceNotification().Message) + oldBounce.merge(latestBounce, snsInfo) + expect(pick(oldBounce.toObject(), ['formId', 'bounces'])).toEqual({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: true }], + }) + }) + + test('should set hasBounced to false if email is delivered later', async () => { + const formId = new ObjectId() + const oldBounce = new Bounce({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: true }], + }) + const latestBounce = new Bounce({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: false }], + }) + const notification = makeDeliveryNotification( + formId, + new ObjectId(), + [MOCK_EMAIL], + [MOCK_EMAIL], + ) + const snsInfo = JSON.parse(notification.Message) + oldBounce.merge(latestBounce, snsInfo) + expect(pick(oldBounce.toObject(), ['formId', 'bounces'])).toEqual({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: false }], + }) + }) + + test('should update email list as necessary', async () => { + const newEmail = 'newemail@email.com' + const formId = new ObjectId() + const oldBounce = new Bounce({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: true }], + }) + const latestBounce = new Bounce({ + formId, + bounces: [{ email: newEmail, hasBounced: false }], + }) + const notification = makeDeliveryNotification( + formId, + new ObjectId(), + [MOCK_EMAIL], + [MOCK_EMAIL], + ) + const snsInfo = JSON.parse(notification.Message) + oldBounce.merge(latestBounce, snsInfo) + expect(pick(oldBounce.toObject(), ['formId', 'bounces'])).toEqual({ + formId, + bounces: [{ email: newEmail, hasBounced: false }], + }) + }) + }) + }) + + describe('statics', () => { + describe('fromSnsNotification', () => { + test('should create documents from delivery notifications correctly', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification = JSON.parse( + makeDeliveryNotification( + formId, + submissionId, + [MOCK_EMAIL], + [MOCK_EMAIL], + ).Message, + ) + const actual = Bounce.fromSnsNotification(notification) + expect(omit(extractBounceObject(actual), 'expireAt')).toEqual({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: false }], + hasAlarmed: false, + }) + expect(actual.expireAt).toBeInstanceOf(Date) + }) + + test('should create documents from bounce notifications correctly', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification = JSON.parse( + makeBounceNotification( + formId, + submissionId, + [MOCK_EMAIL], + [MOCK_EMAIL], + ).Message, + ) + const actual = Bounce.fromSnsNotification(notification) + expect(omit(extractBounceObject(actual), 'expireAt')).toEqual({ + formId, + bounces: [{ email: MOCK_EMAIL, hasBounced: true }], + hasAlarmed: false, + }) + expect(actual.expireAt).toBeInstanceOf(Date) + }) + }) + }) +}) diff --git a/tests/unit/backend/modules/sns/sns.controller.spec.ts b/tests/unit/backend/modules/sns/sns.controller.spec.ts new file mode 100644 index 0000000000..3477fc56ef --- /dev/null +++ b/tests/unit/backend/modules/sns/sns.controller.spec.ts @@ -0,0 +1,59 @@ +import HttpStatus from 'http-status-codes' +import { mocked } from 'ts-jest/utils' + +import handleSns from 'src/app/modules/sns/sns.controller' +import * as snsService from 'src/app/modules/sns/sns.service' + +jest.mock('src/app/modules/sns/sns.service') +const mockSnsService = mocked(snsService, true) + +describe('handleSns', () => { + let req, res + beforeEach(() => { + req = { body: 'somebody' } + res = { sendStatus: jest.fn() } + }) + afterEach(() => { + mockSnsService.updateBounces.mockReset() + mockSnsService.isValidSnsRequest.mockReset() + }) + test('does not call updateBounces for invalid requests', async () => { + mockSnsService.isValidSnsRequest.mockImplementation(() => + Promise.resolve(false), + ) + await handleSns(req, res) + expect(mockSnsService.isValidSnsRequest).toHaveBeenCalledWith(req.body) + expect(mockSnsService.updateBounces).not.toHaveBeenCalled() + expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.FORBIDDEN) + }) + test('calls updateBounces for valid requests', async () => { + mockSnsService.isValidSnsRequest.mockImplementation(() => + Promise.resolve(true), + ) + await handleSns(req, res) + expect(mockSnsService.isValidSnsRequest).toHaveBeenCalledWith(req.body) + expect(mockSnsService.updateBounces).toHaveBeenCalledWith(req.body) + expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.OK) + }) + test('catches errors thrown in isValidSnsRequest', async () => { + mockSnsService.isValidSnsRequest.mockImplementation(() => { + throw new Error() + }) + await handleSns(req, res) + expect(mockSnsService.isValidSnsRequest).toHaveBeenCalledWith(req.body) + expect(mockSnsService.updateBounces).not.toHaveBeenCalled() + expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST) + }) + test('catches errors thrown in updateBounces', async () => { + mockSnsService.isValidSnsRequest.mockImplementation(() => + Promise.resolve(true), + ) + mockSnsService.updateBounces.mockImplementation(() => { + throw new Error() + }) + await handleSns(req, res) + expect(mockSnsService.isValidSnsRequest).toHaveBeenCalledWith(req.body) + expect(mockSnsService.updateBounces).toHaveBeenCalledWith(req.body) + expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST) + }) +}) diff --git a/tests/unit/backend/modules/sns/sns.service.spec.ts b/tests/unit/backend/modules/sns/sns.service.spec.ts new file mode 100644 index 0000000000..4d679b5613 --- /dev/null +++ b/tests/unit/backend/modules/sns/sns.service.spec.ts @@ -0,0 +1,498 @@ +import axios from 'axios' +import { ObjectId } from 'bson' +import crypto from 'crypto' +import dedent from 'dedent' +import { cloneDeep, omit } from 'lodash' +import { mocked } from 'ts-jest/utils' + +import * as loggerModule from 'src/config/logger' +import { ISnsNotification } from 'src/types' + +import dbHandler from '../../helpers/db-handler' +import getMockLogger, { resetMockLogger } from '../../helpers/jest-logger' +import { + extractBounceObject, + makeBounceNotification, + makeDeliveryNotification, + MOCK_SNS_BODY, +} from '../../helpers/sns' + +const Bounce = dbHandler.makeModel('bounce.server.model', 'Bounce') + +jest.mock('axios') +const mockAxios = mocked(axios, true) +jest.mock('src/config/logger') +const mockLoggerModule = mocked(loggerModule, true) +const mockLogger = getMockLogger() +mockLoggerModule.createCloudWatchLogger.mockImplementation(() => mockLogger) +mockLoggerModule.createLoggerWithLabel.mockImplementation(() => getMockLogger()) + +// Import the service last so that mocks get imported correctly +// eslint-disable-next-line import/first +import { + isValidSnsRequest, + updateBounces, +} from 'src/app/modules/sns/sns.service' + +describe('isValidSnsRequest', () => { + let keys, body: ISnsNotification + beforeAll(() => { + keys = crypto.generateKeyPairSync('rsa', { + modulusLength: 2048, + publicKeyEncoding: { + type: 'pkcs1', + format: 'pem', + }, + privateKeyEncoding: { + type: 'pkcs8', + format: 'pem', + }, + }) + }) + beforeEach(() => { + body = cloneDeep(MOCK_SNS_BODY) + mockAxios.get.mockResolvedValue({ + data: keys.publicKey, + }) + }) + test('should gracefully reject empty input', () => { + return expect(isValidSnsRequest(undefined)).resolves.toBe(false) + }) + test('should reject requests without valid structure', () => { + delete body.Type + return expect(isValidSnsRequest(body)).resolves.toBe(false) + }) + test('should reject requests with invalid certificate URL', () => { + body.SigningCertURL = 'http://www.example.com' + return expect(isValidSnsRequest(body)).resolves.toBe(false) + }) + test('should reject requests with invalid signature version', () => { + body.SignatureVersion = 'wrongSignatureVersion' + return expect(isValidSnsRequest(body)).resolves.toBe(false) + }) + test('should reject requests with invalid signature', () => { + return expect(isValidSnsRequest(body)).resolves.toBe(false) + }) + test('should accept valid requests', () => { + const signer = crypto.createSign('RSA-SHA1') + const baseString = + dedent`Message + ${body.Message} + MessageId + ${body.MessageId} + Timestamp + ${body.Timestamp} + TopicArn + ${body.TopicArn} + Type + ${body.Type} + ` + '\n' + signer.write(baseString) + body.Signature = signer.sign(keys.privateKey, 'base64') + return expect(isValidSnsRequest(body)).resolves.toBe(true) + }) +}) + +describe('updateBounces', () => { + const recipientList = [ + 'email1@example.com', + 'email2@example.com', + 'email3@example.com', + ] + beforeAll(async () => await dbHandler.connect()) + afterEach(async () => { + await dbHandler.clearDatabase() + resetMockLogger(mockLogger) + }) + afterAll(async () => await dbHandler.closeDatabase()) + + test('should save a single delivery notification correctly', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification = makeDeliveryNotification( + formId, + submissionId, + recipientList, + recipientList, + ) + await updateBounces(notification) + const actualBounceDoc = await Bounce.findOne({ formId }) + const actualBounce = extractBounceObject(actualBounceDoc) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: false, + })) + expect(mockLogger.info).toHaveBeenCalledWith( + JSON.parse(notification.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save a single non-critical bounce notification correctly', async () => { + const bounces = { + [recipientList[0]]: true, + [recipientList[1]]: false, + [recipientList[2]]: false, + } + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // Only first email bounced + ) + await updateBounces(notification) + const actualBounceDoc = await Bounce.findOne({ formId }) + const actualBounce = extractBounceObject(actualBounceDoc) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: bounces[email], + })) + expect(mockLogger.info).toHaveBeenCalledWith( + JSON.parse(notification.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save a single critical bounce notification correctly', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList, + ) + await updateBounces(notification) + const actualBounceDoc = await Bounce.findOne({ formId }) + const actualBounce = extractBounceObject(actualBounceDoc) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: true, + })) + expect(mockLogger.info).toHaveBeenCalledWith( + JSON.parse(notification.Message), + ) + expect(mockLogger.warn.mock.calls[0][0]).toMatchObject({ + type: 'CRITICAL BOUNCE', + }) + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: true, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save consecutive delivery notifications correctly', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification1 = makeDeliveryNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // First email delivered + ) + const notification2 = makeDeliveryNotification( + formId, + submissionId, + recipientList, + recipientList.slice(1), // Second two emails delivered + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: false, + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save consecutive non-critical bounce notifications correctly', async () => { + const bounces = { + [recipientList[0]]: true, + [recipientList[1]]: true, + [recipientList[2]]: false, + } + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification1 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // First email bounced + ) + const notification2 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(1, 2), // Second email bounced + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: bounces[email], + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save consecutive critical bounce notifications correctly', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification1 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // First email bounced + ) + const notification2 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(1, 3), // Second and third email bounced + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: true, + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + expect(mockLogger.warn.mock.calls[0][0]).toMatchObject({ + type: 'CRITICAL BOUNCE', + }) + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: true, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save delivery, then bounce notifications correctly', async () => { + const bounces = { + [recipientList[0]]: false, + [recipientList[1]]: true, + [recipientList[2]]: true, + } + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification1 = makeDeliveryNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // First email delivered + ) + const notification2 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(1, 3), // Second and third email bounced + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: bounces[email], + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should save bounce, then delivery notifications correctly', async () => { + const bounces = { + [recipientList[0]]: true, + [recipientList[1]]: false, + [recipientList[2]]: false, + } + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification1 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // First email bounced + ) + const notification2 = makeDeliveryNotification( + formId, + submissionId, + recipientList, + recipientList.slice(1, 3), // Second and third email delivered + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: bounces[email], + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should set hasBounced to false on subsequent success', async () => { + const formId = new ObjectId() + const submissionId = new ObjectId() + const notification1 = makeBounceNotification( + formId, + submissionId, + recipientList, + recipientList.slice(0, 1), // First email bounced + ) + const notification2 = makeDeliveryNotification( + formId, + submissionId, + recipientList, + recipientList, // All emails delivered + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: false, + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + expect(mockLogger.warn).not.toHaveBeenCalled() + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: false, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) + + test('should not log critical bounces if hasAlarmed is true', async () => { + const formId = new ObjectId() + const submissionId1 = new ObjectId() + const submissionId2 = new ObjectId() + const notification1 = makeBounceNotification( + formId, + submissionId1, + recipientList, + recipientList, + ) + const notification2 = makeBounceNotification( + formId, + submissionId2, + recipientList, + recipientList, + ) + await updateBounces(notification1) + await updateBounces(notification2) + const actualBounceCursor = await Bounce.find({ formId }) + const actualBounce = extractBounceObject(actualBounceCursor[0]) + const expectedBounces = recipientList.map((email) => ({ + email, + hasBounced: true, + })) + // There should only be one document after 2 notifications + expect(actualBounceCursor.length).toBe(1) + expect(mockLogger.info.mock.calls[0][0]).toEqual( + JSON.parse(notification1.Message), + ) + expect(mockLogger.info.mock.calls[1][0]).toEqual( + JSON.parse(notification2.Message), + ) + // Expect only 1 call to logger.warn + expect(mockLogger.warn.mock.calls.length).toBe(1) + expect(omit(actualBounce, 'expireAt')).toEqual({ + formId, + hasAlarmed: true, + bounces: expectedBounces, + }) + expect(actualBounce.expireAt).toBeInstanceOf(Date) + }) +}) diff --git a/tests/unit/backend/resources/valid-sns-body.js b/tests/unit/backend/resources/valid-sns-body.js deleted file mode 100644 index d08bfbbec3..0000000000 --- a/tests/unit/backend/resources/valid-sns-body.js +++ /dev/null @@ -1,15 +0,0 @@ -const validSnsBody = { - Type: 'type', - MessageId: 'message-id', - TopicArn: 'topic-arn', - Message: 'message', - Timestamp: 'timestamp', - SignatureVersion: '1', - Signature: 'signature', - SigningCertURL: 'https://fakeawsurl.amazonaws.com/cert.pem', - UnsubscribeURL: 'unsubscribe-url', -} - -module.exports = { - validSnsBody, -} From df0ef58ff12c7bcd123ef0dfcd347e4e34cc48f5 Mon Sep 17 00:00:00 2001 From: arshadali172 Date: Wed, 19 Aug 2020 13:26:39 +0800 Subject: [PATCH 03/27] refactor: convert date util to typescript (#161) * Edit util function to reduce side effects and limit access to req * Typify date util * Fix typing to allow string instead of date type * Add tests for date util Co-authored-by: Arshad Ali --- .../encrypt-submissions.server.controller.js | 7 +- .../submissions.server.controller.js | 7 +- src/app/utils/date.js | 26 ------- src/app/utils/date.ts | 21 ++++++ tests/unit/backend/utils/date.spec.ts | 70 +++++++++++++++++++ 5 files changed, 103 insertions(+), 28 deletions(-) delete mode 100644 src/app/utils/date.js create mode 100644 src/app/utils/date.ts create mode 100644 tests/unit/backend/utils/date.spec.ts diff --git a/src/app/controllers/encrypt-submissions.server.controller.js b/src/app/controllers/encrypt-submissions.server.controller.js index 095d28cbdf..8adf8c91ea 100644 --- a/src/app/controllers/encrypt-submissions.server.controller.js +++ b/src/app/controllers/encrypt-submissions.server.controller.js @@ -348,7 +348,12 @@ exports.streamEncryptedResponses = async function (req, res) { submissionType: 'encryptSubmission', } - query = createQueryWithDateParam(query, req) + const augmentedQuery = createQueryWithDateParam( + req.query.startDate, + req.query.endDate, + ) + + query = { ...query, ...augmentedQuery } Submission.find(query, { encryptedContent: 1, diff --git a/src/app/controllers/submissions.server.controller.js b/src/app/controllers/submissions.server.controller.js index 0e32b616f9..0de3f8ac7d 100644 --- a/src/app/controllers/submissions.server.controller.js +++ b/src/app/controllers/submissions.server.controller.js @@ -197,7 +197,12 @@ exports.count = function (req, res) { }) } - query = createQueryWithDateParam(query, req) + const augmentedQuery = createQueryWithDateParam( + req.query.startDate, + req.query.endDate, + ) + + query = { ...query, ...augmentedQuery } Submission.countDocuments(query, function (err, count) { if (err) { diff --git a/src/app/utils/date.js b/src/app/utils/date.js deleted file mode 100644 index 6a5de6ee7d..0000000000 --- a/src/app/utils/date.js +++ /dev/null @@ -1,26 +0,0 @@ -const moment = require('moment-timezone') - -const isMalformedDate = (date) => - date && !moment(date, 'YYYY-MM-DD', true).isValid() - -const createQueryWithDateParam = (query, req) => { - const augmentedQuery = { ...query } - if (req.query.startDate && req.query.endDate) { - augmentedQuery.created = { - $gte: moment - .tz(req.query.startDate, 'Asia/Singapore') - .startOf('day') - .toDate(), - $lte: moment - .tz(req.query.endDate, 'Asia/Singapore') - .endOf('day') - .toDate(), - } - } - return augmentedQuery -} - -module.exports = { - isMalformedDate, - createQueryWithDateParam, -} diff --git a/src/app/utils/date.ts b/src/app/utils/date.ts new file mode 100644 index 0000000000..46410f4211 --- /dev/null +++ b/src/app/utils/date.ts @@ -0,0 +1,21 @@ +import moment from 'moment-timezone' + +export const isMalformedDate = (date?: string): Boolean => { + return Boolean(date) && !moment(date, 'YYYY-MM-DD', true).isValid() +} + +export const createQueryWithDateParam = ( + startDate?: string, + endDate?: string, +): { created?: { $gte: Date; $lte: Date } } => { + if (startDate && endDate) { + return { + created: { + $gte: moment.tz(startDate, 'Asia/Singapore').startOf('day').toDate(), + $lte: moment.tz(endDate, 'Asia/Singapore').endOf('day').toDate(), + }, + } + } else { + return {} + } +} diff --git a/tests/unit/backend/utils/date.spec.ts b/tests/unit/backend/utils/date.spec.ts new file mode 100644 index 0000000000..5484d312cf --- /dev/null +++ b/tests/unit/backend/utils/date.spec.ts @@ -0,0 +1,70 @@ +import moment from 'moment-timezone' + +import { createQueryWithDateParam, isMalformedDate } from 'src/app/utils/date' + +describe('Date Util', () => { + describe('isMalformedDate', () => { + it('should return false if no date provided', async () => { + // Act + const result = isMalformedDate() + // Assert + expect(result).toEqual(false) + }) + + it('should return false if valid date provided', async () => { + // Act + const result = isMalformedDate('2019-09-01') + // Assert + expect(result).toEqual(false) + }) + + it('should return true if impossible date provided', async () => { + // Act + const result = isMalformedDate('2019-22-01') + // Assert + expect(result).toEqual(true) + }) + + it('should return true if invalid date format provided', async () => { + // Act + const result = isMalformedDate('000000') + // Assert + expect(result).toEqual(true) + }) + }) + + describe('createQueryWithDateParam', () => { + it('should return empty object if startDate and endDate not provided', () => { + // Act + const result = createQueryWithDateParam() + // Assert + expect(result).toEqual({}) + }) + + it('should return empty object if only 1 date provided', () => { + // Act + const startDate = '2020-08-17' + const result = createQueryWithDateParam(startDate) + // Assert + expect(result).toEqual({}) + }) + + it('should return object with created field if startDate and endDate provided', () => { + // Act + const startDate = '2020-08-17' + const endDate = '2020-09-18' + const result = createQueryWithDateParam(startDate, endDate) + // Assert + const expected = { + startDateStartOfDay: moment('2020-08-16T16:00:00.000Z').toDate(), + endDateEndOfDay: moment('2020-09-18T15:59:59.999Z').toDate(), + } + expect(result).toEqual({ + created: { + $gte: expected.startDateStartOfDay, + $lte: expected.endDateEndOfDay, + }, + }) + }) + }) +}) From a86c2e5051616dbcfb288a1604ce9eb6a8663cc3 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Wed, 19 Aug 2020 13:29:25 +0800 Subject: [PATCH 04/27] refactor: delete render promise util (#168) --- src/app/utils/render-promise.js | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 src/app/utils/render-promise.js diff --git a/src/app/utils/render-promise.js b/src/app/utils/render-promise.js deleted file mode 100644 index 74e501724d..0000000000 --- a/src/app/utils/render-promise.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Calls app.render in Promise form. Useful for rendering HTML email templates. - * @param {Object} res Express response object - * @param {string} templatePath Path to template relative to src/app/views - * @param {Object} renderData Parameters to render in the template - */ -const renderPromise = (res, templatePath, renderData) => { - return new Promise((resolve, reject) => { - res.app.render(templatePath, renderData, (err, html) => { - if (err) { - return reject(err) - } - return resolve(html) - }) - }) -} - -module.exports = { renderPromise } From a93a2cdc82db764168f8da2855ada9b3b8287104 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Wed, 19 Aug 2020 13:30:31 +0800 Subject: [PATCH 05/27] refactor: migrate encryption util to typescript (#167) --- src/app/utils/{encryption.js => encryption.ts} | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) rename src/app/utils/{encryption.js => encryption.ts} (69%) diff --git a/src/app/utils/encryption.js b/src/app/utils/encryption.ts similarity index 69% rename from src/app/utils/encryption.js rename to src/app/utils/encryption.ts index bfd7f9995b..e0eff43798 100644 --- a/src/app/utils/encryption.js +++ b/src/app/utils/encryption.ts @@ -1,6 +1,7 @@ -const { decode: decodeBase64 } = require('@stablelib/base64') +import { decode as decodeBase64 } from '@stablelib/base64' -const checkIsEncryptedEncoding = (encryptedStr) => { +export const checkIsEncryptedEncoding = (encryptedStr: string): boolean => { + // TODO (#42): Remove this type check once whole backend is in TypeScript. if (typeof encryptedStr !== 'string') { throw new Error('encryptedStr is not of type `string`') } @@ -20,7 +21,3 @@ const checkIsEncryptedEncoding = (encryptedStr) => { return false } } - -module.exports = { - checkIsEncryptedEncoding, -} From bd67e640f1eb3f1b5db7e10c1f3d3b600042b1b1 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Wed, 19 Aug 2020 16:43:24 +0800 Subject: [PATCH 06/27] feat: increase breaker window time and add minimum volume threshold (#165) --- src/app/services/myinfo.service.js | 6 +++-- .../backend/services/myinfo.service.spec.ts | 26 ++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/app/services/myinfo.service.js b/src/app/services/myinfo.service.js index 8137b3fe38..0fe4553ce2 100644 --- a/src/app/services/myinfo.service.js +++ b/src/app/services/myinfo.service.js @@ -12,8 +12,10 @@ class MyInfoService { this.myInfoClientBreaker = new CircuitBreaker( (params) => MyInfoGovClient.getPersonBasic(params), { - errorThresholdPercentage: 80, - timeout: 5000, + errorThresholdPercentage: 80, // % of errors before breaker trips + timeout: 5000, // max time before individual request fails, ms + rollingCountTimeout: 30000, // width of statistical window, ms + volumeThreshold: 5, // min number of requests within statistical window before breaker trips }, ) } diff --git a/tests/unit/backend/services/myinfo.service.spec.ts b/tests/unit/backend/services/myinfo.service.spec.ts index 36be9ee019..d5081cc877 100644 --- a/tests/unit/backend/services/myinfo.service.spec.ts +++ b/tests/unit/backend/services/myinfo.service.spec.ts @@ -167,27 +167,29 @@ describe('MyInfoService', () => { ).rejects.toThrowError(mockError) }) - it('should throw circuit breaker error after first fetch failure', async () => { + it('should throw circuit breaker error after 5 fetch failures', async () => { // Arrange const mockError = new Error('Mock MyInfo server failure') - MyInfoGovClient.getPersonBasic.mockImplementationOnce(() => - Promise.reject(mockError), - ) // Act + Assert - // Call fetch twice - // First fetch should be correctly rejected - await expect( - myInfoService.fetchMyInfoPersonData(mockFetchPersonDataParams), - ).rejects.toThrowError(mockError) + // Call fetch 5 times + // All should be correctly rejected + for (let i = 0; i < 5; i++) { + MyInfoGovClient.getPersonBasic.mockImplementationOnce(() => + Promise.reject(mockError), + ) + await expect( + myInfoService.fetchMyInfoPersonData(mockFetchPersonDataParams), + ).rejects.toThrowError(mockError) + } - // Second fetch should be circuit broken error, without needing to check + // 6th fetch should be circuit broken error, without needing to check // MyInfoGovClient.getPersonBasic. await expect( myInfoService.fetchMyInfoPersonData(mockFetchPersonDataParams), ).rejects.toThrowError(new Error('Breaker is open')) - // Total number of calls to getPersonBasic should be only 1 - expect(MyInfoGovClient.getPersonBasic).toBeCalledTimes(1) + // Total number of calls to getPersonBasic should be only 5 + expect(MyInfoGovClient.getPersonBasic).toBeCalledTimes(5) }) }) From 0922395387eb0bda002c79d0595e25aa1cf498a1 Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Thu, 20 Aug 2020 14:30:47 +0800 Subject: [PATCH 07/27] fix: enable forceDelivery on twilio message sending (#178) --- src/app/services/sms.service.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/app/services/sms.service.ts b/src/app/services/sms.service.ts index 58828e35fd..4bc53c958b 100644 --- a/src/app/services/sms.service.ts +++ b/src/app/services/sms.service.ts @@ -146,7 +146,12 @@ const send = async ( const { client, msgSrvcSid } = twilioConfig return client.messages - .create({ to: recipient, body: message, from: msgSrvcSid }) + .create({ + to: recipient, + body: message, + from: msgSrvcSid, + forceDelivery: true, + }) .then(({ status, sid, errorCode, errorMessage }) => { // Sent but with error code. // Throw error to be caught in catch block. From 2d3fb53594e25b9444ed7d3ef16bfb1b7f138dd2 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Thu, 20 Aug 2020 15:18:34 +0800 Subject: [PATCH 08/27] feat: mailto link for secret key (#150) * style: restyle safekeeping guide and update copy * feat: add mailto link to email secret key * feat: add GA tracking for clicks * style: change colour of urls * style: fix weird spacing before email --- .../core/services/gtag.client.service.js | 11 +++++++ .../create-form-modal.client.controller.js | 28 +++++++++++++++++ .../modules/forms/admin/css/pop-up-modal.css | 27 ++++++++++++++++ .../admin/views/create-form.client.modal.html | 31 ++++++++++++++----- 4 files changed, 90 insertions(+), 7 deletions(-) diff --git a/src/public/modules/core/services/gtag.client.service.js b/src/public/modules/core/services/gtag.client.service.js index 28674492e5..73fa8602b4 100644 --- a/src/public/modules/core/services/gtag.client.service.js +++ b/src/public/modules/core/services/gtag.client.service.js @@ -468,5 +468,16 @@ function GTag($rootScope, $window) { }) } + /** + * Logs clicking on mailto link to share form secret key with collaborators. + */ + gtagService.clickSecretKeyMailto = (formTitle) => { + _gtagEvents('storage', { + event_category: 'Storage Mode Form', + event_action: 'Secret key mailto clicked', + event_label: formTitle + }) + } + return gtagService } diff --git a/src/public/modules/forms/admin/controllers/create-form-modal.client.controller.js b/src/public/modules/forms/admin/controllers/create-form-modal.client.controller.js index b658bb2326..d2a088d632 100644 --- a/src/public/modules/forms/admin/controllers/create-form-modal.client.controller.js +++ b/src/public/modules/forms/admin/controllers/create-form-modal.client.controller.js @@ -2,6 +2,7 @@ const { triggerFileDownload } = require('../../helpers/util') const { templates } = require('../constants/covid19') const Form = require('../../viewmodels/Form.class') +const dedent = require('dedent-js') /** * Determine the form title when duplicating a form @@ -33,6 +34,7 @@ angular '$state', '$timeout', '$uibModal', + '$window', 'Toastr', 'responseModeEnum', 'createFormModalOptions', @@ -50,6 +52,7 @@ function CreateFormModalController( $state, $timeout, $uibModal, + $window, Toastr, responseModeEnum, createFormModalOptions, @@ -221,6 +224,7 @@ function CreateFormModalController( const { publicKey, secretKey } = FormSgSdk.crypto.generate() vm.publicKey = publicKey vm.secretKey = secretKey + vm.mailToUri = generateMailToUri(vm.formData.title, secretKey) vm.formStatus = 3 }) } else if ( @@ -297,6 +301,30 @@ function CreateFormModalController( } } + const generateMailToUri = (title, secretKey) => { + return 'mailto:?subject=' + + $window.encodeURIComponent(`Shared Secret Key for ${title}`) + + '&body=' + + $window.encodeURIComponent( + dedent` + Dear collaborator, + + I am sharing my form's secret key with you for safekeeping and backup. This is an important key that is needed to access all form responses. + + Form title: ${title} + Secret key: ${secretKey} + + All you need to do is keep this email as a record, and please do not share this key with anyone else. + + Thank you for helping to safekeep my form! + ` + ) + } + + vm.handleMailToClick = () => { + GTag.clickSecretKeyMailto(vm.formData.title) + } + // Whether user has copied secret key vm.isCopied = false vm.copied = () => { diff --git a/src/public/modules/forms/admin/css/pop-up-modal.css b/src/public/modules/forms/admin/css/pop-up-modal.css index eeba863f33..2d0cdbbaf8 100644 --- a/src/public/modules/forms/admin/css/pop-up-modal.css +++ b/src/public/modules/forms/admin/css/pop-up-modal.css @@ -174,3 +174,30 @@ display: block; margin: 0 auto; } + +#create-form-modal .secret-key-safekeeping.alert-custom { + display: flex; + padding: 14px 16px; + line-height: 1.39; +} + +#create-form-modal .secret-key-safekeeping i { + padding-top: 4px; + padding-right: 16px; +} + +#create-form-modal .secret-key-safekeeping ul { + list-style-type: none; + padding-left: 0; + padding-top: 0.5em; +} + +#create-form-modal .secret-key-safekeeping li::before { + /* Add a space here because otherwise there's a weird spacing before the first word */ + content: '- '; + margin-right: 0.5em; +} + +#create-form-modal .secret-key-safekeeping a { + color: #2f60ce; +} diff --git a/src/public/modules/forms/admin/views/create-form.client.modal.html b/src/public/modules/forms/admin/views/create-form.client.modal.html index 1c917f3292..30687c54dd 100644 --- a/src/public/modules/forms/admin/views/create-form.client.modal.html +++ b/src/public/modules/forms/admin/views/create-form.client.modal.html @@ -110,13 +110,30 @@
-
- Secret key safekeeping: Email your secret key to colleagues for safekeeping, or organize - multiple keys in a spreadsheet. +
+ +
+ + Ways to safekeep + (guide) +
    +
  • + Email + your secret key to collaborators for safekeeping +
    • +
  • Organise multiple keys in a spreadsheet
    • +
+ +
From dd1461afbb95dbd4db2aa6233bd3928fdb7896a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2020 15:23:07 +0800 Subject: [PATCH 09/27] chore(deps-dev): bump env-cmd from 9.0.3 to 10.1.0 (#133) Bumps [env-cmd](https://github.com/toddbluhm/env-cmd) from 9.0.3 to 10.1.0. - [Release notes](https://github.com/toddbluhm/env-cmd/releases) - [Changelog](https://github.com/toddbluhm/env-cmd/blob/master/CHANGELOG.md) - [Commits](https://github.com/toddbluhm/env-cmd/compare/9.0.3...10.1.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 59 +++++++++++++++++++++++++++++++++++++++++++---- package.json | 2 +- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index af5f8e1a73..bae4b6b598 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10630,13 +10630,62 @@ "dev": true }, "env-cmd": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-9.0.3.tgz", - "integrity": "sha512-DXNeSkLMlYOmq9At+GyvqpdIDuy3gRvz2Z77kN4cAhRbAGVmeiYaqdWqgHSTJ9wCck6ZD0rtbhHVcN7cc2j7rw==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-10.1.0.tgz", + "integrity": "sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==", "dev": true, "requires": { - "commander": "^2.0.0", - "cross-spawn": "^6.0.0" + "commander": "^4.0.0", + "cross-spawn": "^7.0.0" + }, + "dependencies": { + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } } }, "env-variable": { diff --git a/package.json b/package.json index 05814287d2..4c819d5438 100644 --- a/package.json +++ b/package.json @@ -190,7 +190,7 @@ "copy-webpack-plugin": "^6.0.2", "core-js": "^3.6.4", "css-loader": "^2.1.1", - "env-cmd": "^9.0.3", + "env-cmd": "^10.1.0", "eslint": "^6.8.0", "eslint-config-prettier": "^6.11.0", "eslint-plugin-angular": "^4.0.1", From 4a7447a63be367f1398500ab94c74a17ac1454b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2020 15:57:44 +0800 Subject: [PATCH 10/27] chore(deps-dev): bump jasmine from 3.5.0 to 3.6.1 (#158) Bumps [jasmine](https://github.com/jasmine/jasmine-npm) from 3.5.0 to 3.6.1. - [Release notes](https://github.com/jasmine/jasmine-npm/releases) - [Commits](https://github.com/jasmine/jasmine-npm/compare/v3.5.0...v3.6.1) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 168 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 2 +- 2 files changed, 164 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index bae4b6b598..b5ec28390e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14316,13 +14316,171 @@ } }, "jasmine": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.5.0.tgz", - "integrity": "sha512-DYypSryORqzsGoMazemIHUfMkXM7I7easFaxAvNM3Mr6Xz3Fy36TupTrAOxZWN8MVKEU5xECv22J4tUQf3uBzQ==", + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-3.6.1.tgz", + "integrity": "sha512-Jqp8P6ZWkTVFGmJwBK46p+kJNrZCdqkQ4GL+PGuBXZwK1fM4ST9BizkYgIwCFqYYqnTizAy6+XG2Ej5dFrej9Q==", "dev": true, "requires": { - "glob": "^7.1.4", - "jasmine-core": "~3.5.0" + "fast-glob": "^2.2.6", + "jasmine-core": "~3.6.0" + }, + "dependencies": { + "@nodelib/fs.stat": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz", + "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", + "dev": true + }, + "braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "requires": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "fast-glob": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.7.tgz", + "integrity": "sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw==", + "dev": true, + "requires": { + "@mrmlnc/readdir-enhanced": "^2.2.1", + "@nodelib/fs.stat": "^1.1.2", + "glob-parent": "^3.1.0", + "is-glob": "^4.0.0", + "merge2": "^1.2.3", + "micromatch": "^3.1.10" + } + }, + "fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", + "dev": true, + "requires": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "dependencies": { + "extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "requires": { + "is-extendable": "^0.1.0" + } + } + } + }, + "glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "dev": true, + "requires": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + }, + "dependencies": { + "is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", + "dev": true, + "requires": { + "is-extglob": "^2.1.0" + } + } + } + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + }, + "dependencies": { + "kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "requires": { + "is-buffer": "^1.1.5" + } + } + } + }, + "jasmine-core": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-3.6.0.tgz", + "integrity": "sha512-8uQYa7zJN8hq9z+g8z1bqCfdC8eoDAeVnM5sfqs7KHv9/ifoJ500m018fpFc7RDaO6SWCLCXwo/wPSNcdYTgcw==", + "dev": true + }, + "micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "requires": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + } + }, + "to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "dev": true, + "requires": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + } + } } }, "jasmine-core": { diff --git a/package.json b/package.json index 4c819d5438..c3ab060192 100644 --- a/package.json +++ b/package.json @@ -203,7 +203,7 @@ "html-loader": "~0.5.5", "htmlhint": "^0.14.1", "husky": "^4.2.5", - "jasmine": "^3.1.0", + "jasmine": "^3.6.1", "jasmine-core": "^3.1.0", "jasmine-sinon": "^0.4.0", "jasmine-spec-reporter": "^5.0.2", From 4910a61a2af77b26fbbf53654d596d50fe6683ef Mon Sep 17 00:00:00 2001 From: Yuan Ruo Date: Thu, 20 Aug 2020 19:27:08 +0800 Subject: [PATCH 11/27] fix: run npm audit fix to resolve security issues with minimist dependency in the selectize package (#181) --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b5ec28390e..ffb3db5e9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10436,9 +10436,9 @@ "dev": true }, "elliptic": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz", - "integrity": "sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw==", + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", "dev": true, "requires": { "bn.js": "^4.4.0", From f25aa1fbb49512d3e1e24babe40b99741abf9faa Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Fri, 21 Aug 2020 23:43:20 +0800 Subject: [PATCH 12/27] feat: support &`;'" in form title (#156) --- src/app/models/form.server.model.ts | 2 +- .../admin/componentViews/form-title-input.client.view.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/models/form.server.model.ts b/src/app/models/form.server.model.ts index 9b95821019..83300d8241 100644 --- a/src/app/models/form.server.model.ts +++ b/src/app/models/form.server.model.ts @@ -145,7 +145,7 @@ const compileFormModel = (db: Mongoose): IFormModel => { minlength: [4, 'Form name must be at least 4 characters'], maxlength: [200, 'Form name can have a maximum of 200 characters'], match: [ - /^[a-zA-Z0-9_\-./() ]*$/, + /^[a-zA-Z0-9_\-./() &`;'"]*$/, 'Form name cannot contain special characters', ], }, diff --git a/src/public/modules/forms/admin/componentViews/form-title-input.client.view.html b/src/public/modules/forms/admin/componentViews/form-title-input.client.view.html index 130f6538c8..4edbce3b36 100644 --- a/src/public/modules/forms/admin/componentViews/form-title-input.client.view.html +++ b/src/public/modules/forms/admin/componentViews/form-title-input.client.view.html @@ -13,7 +13,7 @@ ng-minlength="4" ng-required="true" ng-maxlength="200" - ng-pattern="/^[a-zA-Z0-9_\-.\/() ]*$/" + ng-pattern="/^[a-zA-Z0-9_\-.\/() &`;'"]*$/" autocomplete="off" ng-keyup="$event.keyCode === 13 && vm.formController.title.$valid && vm.saveForm()" ng-class="vm.formController.title.$invalid && vm.formController.title.$dirty ? 'input-error' : ''" From b15c141d94aa7b684ac86f3edd4f177f5ebc19d8 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Sat, 22 Aug 2020 00:05:03 +0800 Subject: [PATCH 13/27] refactor(verification): convert to module and typescriptify (#172) --- package-lock.json | 26 +- package.json | 3 +- .../verification/verification.controller.ts} | 77 +- .../verification/verification.factory.ts} | 28 +- .../verification/verification.routes.ts} | 13 +- .../verification/verification.service.ts} | 189 ++-- src/app/routes/index.js | 1 - src/loaders/express/index.ts | 2 + src/shared/util/verification.ts | 4 +- ...mail-submissions.server.controller.spec.js | 378 ++++++++ .../verification.server.controller.spec.js | 863 ------------------ tests/unit/backend/helpers/jest-express.ts | 16 + .../modules/sns/sns.controller.spec.ts | 12 +- .../backend/modules/sns/sns.service.spec.ts | 15 +- .../verification.controller.spec.ts | 288 ++++++ .../verification/verification.service.spec.ts | 488 ++++++++++ 16 files changed, 1393 insertions(+), 1010 deletions(-) rename src/app/{controllers/verification.server.controller.js => modules/verification/verification.controller.ts} (74%) rename src/app/{factories/verified-fields.factory.js => modules/verification/verification.factory.ts} (57%) rename src/app/{routes/verification.server.routes.js => modules/verification/verification.routes.ts} (86%) rename src/app/{services/verification.service.js => modules/verification/verification.service.ts} (61%) delete mode 100644 tests/unit/backend/controllers/verification.server.controller.spec.js create mode 100644 tests/unit/backend/helpers/jest-express.ts create mode 100644 tests/unit/backend/modules/verification/verification.controller.spec.ts create mode 100644 tests/unit/backend/modules/verification/verification.service.spec.ts diff --git a/package-lock.json b/package-lock.json index ffb3db5e9b..de73070aad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3386,9 +3386,9 @@ } }, "@hapi/address": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@hapi/address/-/address-4.0.1.tgz", - "integrity": "sha512-0oEP5UiyV4f3d6cBL8F3Z5S7iWSX39Knnl0lY8i+6gfmmIBj44JCBNtcMgwyS+5v7j3VYavNay0NFHDS+UGQcw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@hapi/address/-/address-4.1.0.tgz", + "integrity": "sha512-SkszZf13HVgGmChdHo/PxchnSaCJ6cetVqLzyciudzZRT0jcOouIF/Q93mgjw8cce+D+4F4C1Z/WrfFN+O3VHQ==", "requires": { "@hapi/hoek": "^9.0.0" } @@ -4620,6 +4620,11 @@ "@babel/types": "^7.3.0" } }, + "@types/bcrypt": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/bcrypt/-/bcrypt-3.0.0.tgz", + "integrity": "sha512-nohgNyv+1ViVcubKBh0+XiNJ3dO8nYu///9aJ4cgSqv70gBL+94SNy/iC2NLzKPT2Zt/QavrOkBVbZRLZmw6NQ==" + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -4788,6 +4793,11 @@ "@types/node": "*" } }, + "@types/hapi__joi": { + "version": "17.1.4", + "resolved": "https://registry.npmjs.org/@types/hapi__joi/-/hapi__joi-17.1.4.tgz", + "integrity": "sha512-gqY3TeTyZvnyNhM02HgyCIoGIWsTFMnuzMfnD8evTsr1KIfueGJaz+QC77j+dFvhZ5cJArUNjDRHUjPxNohzGA==" + }, "@types/has-ansi": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/has-ansi/-/has-ansi-3.0.0.tgz", @@ -8033,12 +8043,14 @@ "dev": true }, "celebrate": { - "version": "12.1.1", - "resolved": "https://registry.npmjs.org/celebrate/-/celebrate-12.1.1.tgz", - "integrity": "sha512-d6A0KRX3sbGCbB/id6D4FAqGHwRfFb1B2e77pberUFF1awsF3P9RsLeeRYwbsrgA6Zj7J01SA07NgUyR6BeJVQ==", + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/celebrate/-/celebrate-12.2.0.tgz", + "integrity": "sha512-dkcQaUL4zrPOua/NwTM74jf/NY3wv9Fyb1mkC2ru75KRHowSIDe/tJtIG9yRyPyFCfkr1odif8zNQq23eTwEYg==", "requires": { "@hapi/joi": "17.x.x", - "escape-html": "1.0.3" + "@types/hapi__joi": "17.x.x", + "escape-html": "1.0.3", + "lodash": "4.17.x" } }, "chai": { diff --git a/package.json b/package.json index c3ab060192..2369093670 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "@opengovsg/ng-file-upload": "^12.2.14", "@opengovsg/spcp-auth-client": "^1.3.0", "@stablelib/base64": "^1.0.0", + "@types/bcrypt": "^3.0.0", "JSONStream": "^1.3.5", "ajv": "^5.2.3", "angular": "~1.8.0", @@ -94,7 +95,7 @@ "boxicons": "1.8.0", "bson-ext": "^2.0.3", "busboy": "^0.3.1", - "celebrate": "^12.1.0", + "celebrate": "^12.2.0", "compression": "~1.7.2", "connect": "^3.6.6", "connect-mongo": "^3.2.0", diff --git a/src/app/controllers/verification.server.controller.js b/src/app/modules/verification/verification.controller.ts similarity index 74% rename from src/app/controllers/verification.server.controller.js rename to src/app/modules/verification/verification.controller.ts index 6172b41341..2eb989c717 100644 --- a/src/app/controllers/verification.server.controller.js +++ b/src/app/modules/verification/verification.controller.ts @@ -1,18 +1,25 @@ -const HttpStatus = require('http-status-codes') -const verificationService = require('../services/verification.service') -const logger = require('../../config/logger').createLoggerWithLabel( - 'verification', -) -const { VfnErrors } = require('../../shared/util/verification') +import { RequestHandler, Response } from 'express' +import HttpStatus from 'http-status-codes' + +import { createLoggerWithLabel } from '../../../config/logger' +import { VfnErrors } from '../../../shared/util/verification' + +import * as verificationService from './verification.service' + +const logger = createLoggerWithLabel('verification') /** * When a form is loaded publicly, a transaction is created, and populated with the field ids of fields that are verifiable. * If no fields are verifiable, then it did not create a transaction and returns an empty object. - * @param {Express.Request} req - * @param {Express.Response} res + * @param req + * @param res * @returns 201 - transaction is created * @returns 200 - transaction was not created as no fields were verifiable for the form */ -const createTransaction = async (req, res) => { +export const createTransaction: RequestHandler< + {}, + {}, + { formId: string } +> = async (req, res) => { try { const { formId } = req.body const transaction = await verificationService.createTransaction(formId) @@ -26,10 +33,12 @@ const createTransaction = async (req, res) => { } /** * Returns a transaction's id and expiry time if it exists - * @param {Express.Request} req - * @param {Express.Response} res + * @param req + * @param res */ -const getTransactionMetadata = async (req, res) => { +export const getTransactionMetadata: RequestHandler<{ + transactionId: string +}> = async (req, res) => { try { const { transactionId } = req.params const transaction = await verificationService.getTransactionMetadata( @@ -44,10 +53,14 @@ const getTransactionMetadata = async (req, res) => { /** * When user changes the input value in the verifiable field, * we reset the field in the transaction, removing the previously saved signature. - * @param {Express.Request} req - * @param {Express.Response} res + * @param req + * @param res */ -const resetFieldInTransaction = async (req, res) => { +export const resetFieldInTransaction: RequestHandler< + { transactionId: string }, + {}, + { fieldId: string } +> = async (req, res) => { try { const { transactionId } = req.params const { fieldId } = req.body @@ -62,10 +75,14 @@ const resetFieldInTransaction = async (req, res) => { /** * When user requests to verify a field, an otp is generated. * The current answer is signed, and the signature is also saved in the transaction, with the field id as the key. - * @param {Express.Request} req - * @param {Express.Response} res + * @param req + * @param res */ -const getNewOtp = async (req, res) => { +export const getNewOtp: RequestHandler< + { transactionId: string }, + {}, + { answer: string; fieldId: string } +> = async (req, res) => { try { const { transactionId } = req.params const { answer, fieldId } = req.body @@ -81,10 +98,14 @@ const getNewOtp = async (req, res) => { * When user submits their otp for the field, the otp is validated. * If it is correct, we return the signature that was saved. * This signature will be appended to the response when the form is submitted. - * @param {Express.Request} req - * @param {Express.Response} res + * @param req + * @param res */ -const verifyOtp = async (req, res) => { +export const verifyOtp: RequestHandler< + { transactionId: string }, + {}, + { otp: string; fieldId: string } +> = async (req, res) => { try { const { transactionId } = req.params const { fieldId, otp } = req.body @@ -98,10 +119,10 @@ const verifyOtp = async (req, res) => { } /** * Returns relevant http status code for different verification failures - * @param {Error} error - * @param {Express.Response} res + * @param error + * @param res */ -const handleError = (error, res) => { +const handleError = (error: Error, res: Response) => { let status = HttpStatus.INTERNAL_SERVER_ERROR let message = error.message switch (error.name) { @@ -124,11 +145,3 @@ const handleError = (error, res) => { } return res.status(status).json(message) } - -module.exports = { - createTransaction, - getTransactionMetadata, - resetFieldInTransaction, - getNewOtp, - verifyOtp, -} diff --git a/src/app/factories/verified-fields.factory.js b/src/app/modules/verification/verification.factory.ts similarity index 57% rename from src/app/factories/verified-fields.factory.js rename to src/app/modules/verification/verification.factory.ts index 05557c8e24..98f0ba3034 100644 --- a/src/app/factories/verified-fields.factory.js +++ b/src/app/modules/verification/verification.factory.ts @@ -1,8 +1,24 @@ -const featureManager = require('../../config/feature-manager').default -const verification = require('../../app/controllers/verification.server.controller') -const HttpStatus = require('http-status-codes') +import { RequestHandler } from 'express' +import HttpStatus from 'http-status-codes' -const verifiedFieldsFactory = ({ isEnabled }) => { +import featureManager from '../../../config/feature-manager' +import { FeatureNames } from '../../../config/feature-manager/types' + +import * as verification from './verification.controller' + +interface IVerifiedFieldsFactory { + createTransaction: RequestHandler + getTransactionMetadata: RequestHandler + resetFieldInTransaction: RequestHandler + getNewOtp: RequestHandler + verifyOtp: RequestHandler +} + +const verifiedFieldsFactory = ({ + isEnabled, +}: { + isEnabled: boolean +}): IVerifiedFieldsFactory => { if (isEnabled) { return { createTransaction: verification.createTransaction, @@ -28,4 +44,6 @@ const verifiedFieldsFactory = ({ isEnabled }) => { } } -module.exports = verifiedFieldsFactory(featureManager.get('verified-fields')) +export default verifiedFieldsFactory( + featureManager.get(FeatureNames.VerifiedFields), +) diff --git a/src/app/routes/verification.server.routes.js b/src/app/modules/verification/verification.routes.ts similarity index 86% rename from src/app/routes/verification.server.routes.js rename to src/app/modules/verification/verification.routes.ts index 8d111d6e7c..f1925e3f8b 100644 --- a/src/app/routes/verification.server.routes.js +++ b/src/app/modules/verification/verification.routes.ts @@ -1,12 +1,9 @@ -'use strict' +import { celebrate, Joi } from 'celebrate' +import { Express } from 'express' -/** - * Module dependencies. - */ -const { celebrate, Joi } = require('celebrate') -const verifiedFieldsFactory = require('../factories/verified-fields.factory') +import verifiedFieldsFactory from './verification.factory' -module.exports = function (app) { +const mountVfnRoutes = (app: Express): void => { const formatOfId = Joi.string().length(24).hex().required() app.route('/transaction').post( celebrate({ @@ -64,3 +61,5 @@ module.exports = function (app) { verifiedFieldsFactory.verifyOtp, ) } + +export default mountVfnRoutes diff --git a/src/app/services/verification.service.js b/src/app/modules/verification/verification.service.ts similarity index 61% rename from src/app/services/verification.service.js rename to src/app/modules/verification/verification.service.ts index ba036a0bf1..3688f0b393 100644 --- a/src/app/services/verification.service.js +++ b/src/app/modules/verification/verification.service.ts @@ -1,51 +1,66 @@ -const mongoose = require('mongoose') -const bcrypt = require('bcrypt') -const _ = require('lodash') -const { otpGenerator } = require('../../config/config') -const MailService = require('./mail.service').default -const smsFactory = require('./../factories/sms.factory') -const vfnUtil = require('../../shared/util/verification') -const formsgSdk = require('../../config/formsg-sdk') +import bcrypt from 'bcrypt' +import _ from 'lodash' +import mongoose from 'mongoose' -const getFormModel = require('../models/form.server.model').default -const getVerificationModel = require('../models/verification.server.model') - .default +import { otpGenerator } from '../../../config/config' +import formsgSdk from '../../../config/formsg-sdk' +import * as vfnUtil from '../../../shared/util/verification' +import { + IEmailFieldSchema, + IFieldSchema, + IFormSchema, + IMobileFieldSchema, + IVerificationFieldSchema, + IVerificationSchema, +} from '../../../types' +import smsFactory from '../../factories/sms.factory' +import getFormModel from '../../models/form.server.model' +import getVerificationModel from '../../models/verification.server.model' +import MailService from '../../services/mail.service' const Form = getFormModel(mongoose) const Verification = getVerificationModel(mongoose) - const { - VERIFIED_FIELDTYPES, - SALT_ROUNDS, HASH_EXPIRE_AFTER_SECONDS, - WAIT_FOR_OTP_SECONDS, NUM_OTP_RETRIES, + SALT_ROUNDS, + VERIFIED_FIELDTYPES, VfnErrors, + WAIT_FOR_OTP_SECONDS, } = vfnUtil +interface ITransaction { + transactionId: IVerificationSchema['_id'] + expireAt: IVerificationSchema['expireAt'] +} + /** * Creates a transaction for a form that has verifiable fields - * @param {string} formId - * @returns {object} + * @param formId */ -const createTransaction = async (formId) => { +export const createTransaction = async ( + formId: string, +): Promise => { const form = await Form.findById(formId) const fields = initializeVerifiableFields(form) if (!_.isEmpty(fields)) { - const doc = await Verification.create({ formId, fields }) + const verification = new Verification({ formId, fields }) + const doc = await verification.save() return { transactionId: doc._id, expireAt: doc.expireAt } } return null } +type TransactionMetadata = ReturnType< + typeof Verification.findTransactionMetadata +> /** * Retrieves a transaction's metadata by id - * @param {string} transactionId - * @returns {transaction._id} - * @returns {transaction.formId} - * @returns {transaction.expireAt} + * @param transactionId */ -const getTransactionMetadata = async (transactionId) => { +export const getTransactionMetadata = async ( + transactionId: string, +): Promise => { const transaction = await Verification.findTransactionMetadata(transactionId) if (transaction === null) { throwError(VfnErrors.TransactionNotFound) @@ -55,9 +70,11 @@ const getTransactionMetadata = async (transactionId) => { /** * Retrieves an entire transaction - * @param {string} transactionId + * @param transactionId */ -const getTransaction = async (transactionId) => { +export const getTransaction = async ( + transactionId: string, +): Promise => { const transaction = await Verification.findById(transactionId) if (transaction === null) { throwError(VfnErrors.TransactionNotFound) @@ -67,10 +84,13 @@ const getTransaction = async (transactionId) => { /** * Sets signedData, hashedOtp, hashCreatedAt to null for that field in that transaction - * @param {Mongoose.Document} transaction - * @param {string} fieldId + * @param transaction + * @param fieldId */ -const resetFieldInTransaction = async (transaction, fieldId) => { +export const resetFieldInTransaction = async ( + transaction: IVerificationSchema, + fieldId: string, +): Promise => { const { _id: transactionId } = transaction const { n } = await Verification.updateOne( { _id: transactionId, 'fields._id': fieldId }, @@ -90,11 +110,15 @@ const resetFieldInTransaction = async (transaction, fieldId) => { /** * Generates hashed otp and signed data for the given transaction, fieldId, and answer - * @param {Mongoose.Document} transaction - * @param {string} fieldId - * @param {string} answer + * @param transaction + * @param fieldId + * @param answer */ -const getNewOtp = async (transaction, fieldId, answer) => { +export const getNewOtp = async ( + transaction: IVerificationSchema, + fieldId: string, + answer: string, +): Promise => { if (isTransactionExpired(transaction.expireAt)) { throwError(VfnErrors.TransactionNotFound) } @@ -140,11 +164,15 @@ const getNewOtp = async (transaction, fieldId, answer) => { /** * Compares the given otp. If correct, returns signedData, else returns an error - * @param {Mongoose.Document} transaction - * @param {string} fieldId - * @param {string} inputOtp + * @param transaction + * @param fieldId + * @param inputOtp */ -const verifyOtp = async (transaction, fieldId, inputOtp) => { +export const verifyOtp = async ( + transaction: IVerificationSchema, + fieldId: string, + inputOtp: string, +): Promise => { if (isTransactionExpired(transaction.expireAt)) { throwError(VfnErrors.TransactionNotFound) } @@ -175,10 +203,11 @@ const verifyOtp = async (transaction, fieldId, inputOtp) => { /** * Gets verifiable fields from form and initializes the values to be stored in a transaction - * @param {Mongoose.Document} form - * @returns Array + * @param form */ -const initializeVerifiableFields = (form) => { +const initializeVerifiableFields = ( + form: IFormSchema, +): Pick[] => { return _.get(form, 'form_fields', []) .filter(isFieldVerifiable) .map(({ _id, fieldType }) => { @@ -189,28 +218,39 @@ const initializeVerifiableFields = (form) => { }) } +/** + * Whether a field is of a type that can be verified. + * @param field + */ +const isPossiblyVerifiable = ( + field: IFieldSchema, +): field is IEmailFieldSchema | IMobileFieldSchema => { + return VERIFIED_FIELDTYPES.includes(field.fieldType) +} + /** * Evaluates whether a field is verifiable - * @param {object} field - * @param {string} field.fieldType - * @param {boolean} field.isVerifiable + * @param field */ -const isFieldVerifiable = (field) => { - return ( - VERIFIED_FIELDTYPES.includes(field.fieldType) && field.isVerifiable === true - ) +const isFieldVerifiable = (field: IFieldSchema): boolean => { + return isPossiblyVerifiable(field) && field.isVerifiable === true } /** * Send otp to recipient * - * @param {string} formId - * @param {object} field - * @param {string} field.fieldType - * @param {string} recipient - * @param {string} otp + * @param formId + * @param field + * @param field.fieldType + * @param recipient + * @param otp */ -const sendOTPForField = async (formId, field, recipient, otp) => { +const sendOTPForField = async ( + formId: string, + field: IVerificationFieldSchema, + recipient: string, + otp: string, +): Promise => { const { fieldType } = field switch (fieldType) { case 'mobile': @@ -227,20 +267,20 @@ const sendOTPForField = async (formId, field, recipient, otp) => { } /** - * Checks if expireAt is in the past -- ie transaction has expired - * @param {Date} expireAt + * Checks if expireAt is in the past -- ie transaction has expired + * @param expireAt * @returns boolean */ -const isTransactionExpired = (expireAt) => { +const isTransactionExpired = (expireAt: Date): boolean => { const currentDate = new Date() return expireAt < currentDate } /** - * Checks if HASH_EXPIRE_AFTER_SECONDS has elapsed since the hash was created - ie hash has expired - * @param {Date} hashCreatedAt + * Checks if HASH_EXPIRE_AFTER_SECONDS has elapsed since the hash was created - ie hash has expired + * @param hashCreatedAt */ -const isHashedOtpExpired = (hashCreatedAt) => { +const isHashedOtpExpired = (hashCreatedAt: Date): boolean => { const currentDate = new Date() const expireAt = vfnUtil.getExpiryDate( HASH_EXPIRE_AFTER_SECONDS, @@ -251,10 +291,9 @@ const isHashedOtpExpired = (hashCreatedAt) => { /** * Checks how many seconds remain before a new otp can be generated - * @param {Date} hashCreatedAt - * @returns {Number} + * @param hashCreatedAt */ -const waitToResendOtpSeconds = (hashCreatedAt) => { +const waitToResendOtpSeconds = (hashCreatedAt: Date): number => { if (!hashCreatedAt) { // Hash has not been created return 0 @@ -268,30 +307,24 @@ const waitToResendOtpSeconds = (hashCreatedAt) => { /** * Finds a field by id in a transaction - * @param {Mongoose.Document} transaction - * @param {string} fieldId + * @param transaction + * @param fieldId * @returns verification field */ -const getFieldFromTransaction = (transaction, fieldId) => { +const getFieldFromTransaction = ( + transaction: IVerificationSchema, + fieldId: string, +): IVerificationFieldSchema | undefined => { return transaction.fields.find((field) => field._id === fieldId) } /** - * Helper method to throw an error - * @param {string} message - * @param {string} name + * Helper method to throw an error + * @param message + * @param name */ -const throwError = (message, name) => { +const throwError = (message: string, name?: string): never => { let error = new Error(message) error.name = name || message throw error } - -module.exports = { - createTransaction, - getTransactionMetadata, - getTransaction, - resetFieldInTransaction, - getNewOtp, - verifyOtp, -} diff --git a/src/app/routes/index.js b/src/app/routes/index.js index 6134512e4a..3dacdafc3a 100644 --- a/src/app/routes/index.js +++ b/src/app/routes/index.js @@ -6,5 +6,4 @@ module.exports = [ require('./frontend.server.routes.js'), require('./public-forms.server.routes.js'), require('./spcp.server.routes.js'), - require('./verification.server.routes.js'), ] diff --git a/src/loaders/express/index.ts b/src/loaders/express/index.ts index 3d04371170..fa45b85c56 100644 --- a/src/loaders/express/index.ts +++ b/src/loaders/express/index.ts @@ -8,6 +8,7 @@ import path from 'path' import url from 'url' import mountSnsRoutes from '../../app/modules/sns/sns.routes' +import mountVfnRoutes from '../../app/modules/verification/verification.routes' import apiRoutes from '../../app/routes' import config from '../../config/config' @@ -122,6 +123,7 @@ const loadExpressApp = async (connection: Connection) => { routeFunction(app) }) mountSnsRoutes(app) + mountVfnRoutes(app) app.use(sentryMiddlewares()) diff --git a/src/shared/util/verification.ts b/src/shared/util/verification.ts index c60cefd796..382e8d3a30 100644 --- a/src/shared/util/verification.ts +++ b/src/shared/util/verification.ts @@ -1,4 +1,6 @@ -export const VERIFIED_FIELDTYPES = ['email', 'mobile'] +import { BasicFieldType } from '../../types' + +export const VERIFIED_FIELDTYPES = [BasicFieldType.Email, BasicFieldType.Mobile] export const SALT_ROUNDS = 10 export const TRANSACTION_EXPIRE_AFTER_SECONDS = 14400 // 4 hours export const HASH_EXPIRE_AFTER_SECONDS = 600 // 10 minutes diff --git a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js index f3679df6d0..cb13ef5f25 100644 --- a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js +++ b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js @@ -14,7 +14,12 @@ const MailService = require('../../../../dist/backend/app/services/mail.service' const User = dbHandler.makeModel('user.server.model', 'User') const Agency = dbHandler.makeModel('agency.server.model', 'Agency') const Form = dbHandler.makeModel('form.server.model', 'Form') +const Verification = dbHandler.makeModel( + 'verification.server.model', + 'Verification', +) const EmailForm = mongoose.model('email') +const vfnConstants = require('../../../../dist/backend/shared/util/verification') describe('Email Submissions Controller', () => { const SESSION_SECRET = 'secret' @@ -30,6 +35,7 @@ describe('Email Submissions Controller', () => { 'dist/backend/app/controllers/email-submissions.server.controller', { mongoose: Object.assign(mongoose, { '@noCallThru': true }), + request: (url, callback) => spyRequest(url, callback), }, ) const submissionsController = spec( @@ -2727,4 +2733,376 @@ describe('Email Submissions Controller', () => { }) }) }) + + describe('Verified fields', () => { + let fixtures + let testAgency, testUser, testForm + + const expireAt = new Date() + expireAt.setTime( + expireAt.getTime() + vfnConstants.TRANSACTION_EXPIRE_AFTER_SECONDS * 1000, + ) // Expires 4 hours later + const hasExpired = new Date() + hasExpired.setTime( + hasExpired.getTime() - + vfnConstants.TRANSACTION_EXPIRE_AFTER_SECONDS * 2000, + ) // Expired 2 days ago + + const endpointPath = '/submissions' + const injectFixtures = (req, res, next) => { + Object.assign(req, fixtures) + next() + } + const sendSubmissionBack = (req, res) => { + res.status(200).send({ + body: req.body, + }) + } + + const app = express() + + const sendAndExpect = (status, expectedResponse = null) => { + let send = request(app).post(endpointPath).expect(status) + if (expectedResponse) { + send = send.expect(expectedResponse) + } + return send + } + + const createTransactionForForm = (form) => { + return (expireAt, verifiableFields) => { + let t = { + formId: String(form._id), + expireAt, + } + t.fields = verifiableFields.map((field, i) => { + const { + fieldType, + hashCreatedAt, + hashedOtp, + signedData, + hashRetries, + } = field + return { + _id: form.form_fields[i]._id, + fieldType, + hashCreatedAt: hashCreatedAt === undefined ? null : hashCreatedAt, + hashedOtp: hashedOtp === undefined ? null : hashedOtp, + signedData: signedData === undefined ? null : signedData, + hashRetries: hashRetries === undefined ? 0 : hashRetries, + } + }) + return Verification.create(t) + } + } + + beforeAll((done) => { + app + .route(endpointPath) + .post( + injectFixtures, + controller.validateEmailSubmission, + sendSubmissionBack, + ) + testAgency = new Agency({ + shortName: 'govtest', + fullName: 'Government Testing Agency', + emailDomain: 'test.gov.sg', + logo: '/invalid-path/test.jpg', + }) + testAgency + .save() + .then(() => { + return User.deleteMany({}) + }) + .then(() => { + testUser = new User({ + email: 'test@test.gov.sg', + agency: testAgency._id, + }) + return testUser.save() + }) + .then(done) + }) + + // Submission + describe('No verified fields in form', () => { + beforeEach((done) => { + testForm = new Form({ + title: 'Test Form', + emails: 'test@test.gov.sg', + admin: testUser._id, + form_fields: [{ title: 'Email', fieldType: 'email' }], + }) + testForm + .save({ validateBeforeSave: false }) + .then(() => { + fixtures = { + form: testForm, + body: { + responses: [], + }, + } + }) + .then(done) + }) + it('should allow submission if transaction does not exist for forms that do not contain any fields that have to be verified', (done) => { + // No transaction created for testForm + const field = testForm.form_fields[0] + const response = { + _id: String(field._id), + fieldType: field.fieldType, + question: field.title, + answer: 'test@abc.com', + } + fixtures.body.responses.push(response) + sendAndExpect(HttpStatus.OK, { + body: { + parsedResponses: [Object.assign(response, { isVisible: true })], + }, + }).end(done) + }) + }) + describe('Verified fields', () => { + let createTransaction + beforeAll((done) => { + testForm = new Form({ + title: 'Test Form', + emails: 'test@test.gov.sg', + admin: testUser._id, + form_fields: [ + { title: 'Email', fieldType: 'email', isVerifiable: true }, + ], + }) + testForm + .save({ validateBeforeSave: false }) + .then(() => { + createTransaction = createTransactionForForm(testForm) + }) + .then(done) + }) + beforeEach(() => { + fixtures = { + form: testForm, + body: { + responses: [], + }, + } + }) + + describe('No transaction', () => { + it('should prevent submission if transaction does not exist for a form containing fields that have to be verified', (done) => { + const field = testForm.form_fields[0] + const response = { + _id: String(field._id), + fieldType: field.fieldType, + question: field.title, + answer: 'test@abc.com', + } + fixtures.body.responses.push(response) + + sendAndExpect(HttpStatus.BAD_REQUEST).end(done) + }) + }) + + describe('Has transaction', () => { + it('should prevent submission if transaction has expired for a form containing fields that have to be verified', (done) => { + createTransaction(hasExpired, [ + { + fieldType: 'email', + hashCreatedAt: new Date(), + hashedOtp: 'someHashValue', + signedData: 'someData', + }, + ]).then(() => { + const field = testForm.form_fields[0] + const response = { + _id: String(field._id), + fieldType: field.fieldType, + question: field.title, + answer: 'test@abc.com', + signature: 'someData', + } + fixtures.body.responses.push(response) + + sendAndExpect(HttpStatus.BAD_REQUEST).end(done) + }) + }) + + it('should prevent submission if any of the transaction fields are not verified', (done) => { + createTransaction(expireAt, [ + { + fieldType: 'email', + hashCreatedAt: null, + hashedOtp: null, + signedData: null, + }, + ]).then(() => { + const field = testForm.form_fields[0] + const response = { + _id: String(field._id), + fieldType: field.fieldType, + question: field.title, + answer: 'test@abc.com', + } + fixtures.body.responses.push(response) + + sendAndExpect(HttpStatus.BAD_REQUEST).end(done) + }) + }) + + it('should allow submission if all of the transaction fields are verified', (done) => { + const formsg = require('@opengovsg/formsg-sdk')({ + mode: 'test', + verificationOptions: { + secretKey: process.env.VERIFICATION_SECRET_KEY, + }, + }) + const transactionId = mongoose.Types.ObjectId( + '5e71ef8b19c1ed04b54cd5f9', + ) + + const field = testForm.form_fields[0] + const formId = testForm._id + let response = { + _id: String(field._id), + fieldType: field.fieldType, + question: field.title, + answer: 'test@abc.com', + } + const signature = formsg.verification.generateSignature({ + transactionId: String(transactionId), + formId: String(formId), + fieldId: response._id, + answer: response.answer, + }) + response.signature = signature + + createTransaction(expireAt, [ + { + fieldType: 'email', + hashCreatedAt: new Date(), + hashedOtp: 'someHashValue', + signedData: signature, + }, + ]).then(() => { + fixtures.body.responses.push(response) + sendAndExpect(HttpStatus.OK).end(done) + }) + }) + }) + }) + + describe('Hidden and optional fields', () => { + const expireAt = new Date() + expireAt.setTime(expireAt.getTime() + 86400) + + beforeEach(() => { + fixtures = { + form: {}, + body: { + responses: [], + }, + } + }) + + const test = ({ + fieldValue, + fieldIsRequired, + fieldIsHidden, + expectedStatus, + done, + }) => { + const field = { + _id: '5e719d5b62a2c4aa5d9789e2', + title: 'Email', + fieldType: 'email', + isVerifiable: true, + required: fieldIsRequired, + } + const yesNoField = { + _id: '5e719d5b62a2c4aa5d9789e3', + title: 'Show email if this field is yes', + fieldType: 'yes_no', + } + let form = new Form({ + title: 'Test Form', + emails: 'test@test.gov.sg', + admin: testUser._id, + form_fields: [field, yesNoField], + form_logics: [ + { + show: [field._id], + conditions: [ + { + ifValueType: 'single-select', + _id: '58169', + field: yesNoField._id, + state: 'is equals to', + value: 'Yes', + }, + ], + _id: '5db00a15af2ffb29487d4eb1', + logicType: 'showFields', + }, + ], + }) + form.save({ validateBeforeSave: false }).then(() => { + const response = { + _id: String(field._id), + fieldType: field.fieldType, + question: field.title, + answer: fieldValue, + } + const yesNoResponse = { + _id: yesNoField._id, + question: yesNoField.title, + fieldType: yesNoField.fieldType, + answer: fieldIsHidden ? 'No' : 'Yes', + } + fixtures.form = form + fixtures.body.responses.push(yesNoResponse) + fixtures.body.responses.push(response) + sendAndExpect(expectedStatus).end(done) + }) + } + it('should verify fields that are optional and filled in', (done) => { + test({ + fieldValue: 'test@abc.com', + fieldIsRequired: false, + fieldIsHidden: false, + expectedStatus: HttpStatus.BAD_REQUEST, + done, + }) + }) + it('should not verify fields that are optional and not filled in', (done) => { + test({ + fieldValue: '', + fieldIsRequired: false, + fieldIsHidden: false, + expectedStatus: HttpStatus.OK, + done, + }) + }) + it('should verify fields that are required and not hidden by logic', (done) => { + test({ + fieldValue: 'test@abc.com', + fieldIsRequired: true, + fieldIsHidden: false, + expectedStatus: HttpStatus.BAD_REQUEST, + done, + }) + }) + + it('should not verify fields that are required and hidden by logic', (done) => { + test({ + fieldValue: '', + fieldIsRequired: true, + fieldIsHidden: true, + expectedStatus: HttpStatus.OK, + done, + }) + }) + }) + }) }) diff --git a/tests/unit/backend/controllers/verification.server.controller.spec.js b/tests/unit/backend/controllers/verification.server.controller.spec.js deleted file mode 100644 index 460b9203d5..0000000000 --- a/tests/unit/backend/controllers/verification.server.controller.spec.js +++ /dev/null @@ -1,863 +0,0 @@ -const HttpStatus = require('http-status-codes') -const { hash } = require('bcrypt') -const mongoose = require('mongoose') -const express = require('express') -const request = require('supertest') - -const constants = require('../../../../dist/backend/shared/util/verification') -const dbHandler = require('../helpers/db-handler') -const MailService = require('../../../../dist/backend/app/services/mail.service') - .default - -const User = dbHandler.makeModel('user.server.model', 'User') -const Agency = dbHandler.makeModel('agency.server.model', 'Agency') -const Form = dbHandler.makeModel('form.server.model', 'Form') -const Verification = dbHandler.makeModel( - 'verification.server.model', - 'Verification', -) - -describe('Verification Controller', () => { - const bcrypt = jasmine.createSpyObj('bcrypt', ['hash']) - const sendSmsOtp = jasmine.createSpy('sendSmsOtp') - const testOtp = '123456' - - let spyRequest = jasmine.createSpy('request') - let sendOtpSpy - let req - let res - - const service = spec('dist/backend/app/services/verification.service', { - mongoose: Object.assign(mongoose, { '@noCallThru': true }), - bcrypt, - '../../config/config': { - otpGenerator: () => testOtp, - logger: console, - }, - './../factories/sms.factory': { - sendVerificationOtp: sendSmsOtp, - }, - }) - const controller = spec( - 'dist/backend/app/controllers/verification.server.controller', - { - '../services/verification.service': service, - '../../config/config': { - logger: console, - }, - }, - ) - const submissionController = spec( - 'dist/backend/app/controllers/email-submissions.server.controller', - { - mongoose: Object.assign(mongoose, { '@noCallThru': true }), - request: (url, callback) => spyRequest(url, callback), - }, - ) - - let testAgency, testUser, testForm - - beforeAll(async (done) => { - await dbHandler.connect() - - sendOtpSpy = spyOn(MailService, 'sendVerificationOtp') - - await Agency.deleteMany({}) - testAgency = new Agency({ - shortName: 'govtest', - fullName: 'Government Testing Agency', - emailDomain: 'test.gov.sg', - logo: '/invalid-path/test.jpg', - }) - testAgency - .save() - .then(() => { - return User.deleteMany({}) - }) - .then(() => { - testUser = new User({ - email: 'test@test.gov.sg', - agency: testAgency._id, - }) - return testUser.save() - }) - .then(done) - }) - beforeEach(() => { - req = { - query: {}, - params: {}, - body: {}, - session: {}, - headers: {}, - ip: '127.0.0.1', - } - - res = jasmine.createSpyObj('res', ['status', 'send', 'json', 'sendStatus']) - res.locals = {} - - spyRequest = jasmine.createSpy('request') - }) - - afterAll(async () => await dbHandler.closeDatabase()) - - const expectStatus = function (expectedStatus) { - return function (status) { - expect(status).toEqual(expectedStatus) - return this - } - } - - const createTransactionForForm = (form) => { - return (expireAt, verifiableFields) => { - let t = { - formId: String(form._id), - expireAt, - } - t.fields = verifiableFields.map((field, i) => { - const { - fieldType, - hashCreatedAt, - hashedOtp, - signedData, - hashRetries, - } = field - return { - _id: form.form_fields[i]._id, - fieldType, - hashCreatedAt: hashCreatedAt === undefined ? null : hashCreatedAt, - hashedOtp: hashedOtp === undefined ? null : hashedOtp, - signedData: signedData === undefined ? null : signedData, - hashRetries: hashRetries === undefined ? 0 : hashRetries, - } - }) - return Verification.create(t) - } - } - - const expireAt = new Date() - expireAt.setTime( - expireAt.getTime() + constants.TRANSACTION_EXPIRE_AFTER_SECONDS * 1000, - ) // Expires 4 hours later - const hasExpired = new Date() - hasExpired.setTime( - hasExpired.getTime() - constants.TRANSACTION_EXPIRE_AFTER_SECONDS * 2000, - ) // Expired 2 days ago - - describe('No verified fields in form', () => { - beforeEach((done) => { - testForm = new Form({ - title: 'Test Form', - emails: 'test@test.gov.sg', - admin: testUser._id, - form_fields: [{ fieldType: 'email' }, { fieldType: 'mobile' }], // Not verifiable - }) - testForm.save({ validateBeforeSave: false }).then(done) - }) - - it('should not create a transaction for forms that do not contain any fields that have to be verified', (done) => { - const spyOnCreate = spyOn(Verification, 'create').and.callThrough() - req.body.formId = testForm._id - res.sendStatus.and.callFake((status) => { - expect(status).toEqual(HttpStatus.OK) - expect(spyOnCreate).not.toHaveBeenCalled() - done() - }) - - controller.createTransaction(req, res) - }) - }) - - describe('Verified fields', () => { - let createTransaction - beforeAll((done) => { - testForm = new Form({ - title: 'Test Form', - emails: 'test@test.gov.sg', - admin: testUser._id, - form_fields: [ - { fieldType: 'email', isVerifiable: true }, - { fieldType: 'mobile', isVerifiable: true }, - ], // Verifiable - }) - testForm - .save({ validateBeforeSave: false }) - .then(() => { - createTransaction = createTransactionForForm(testForm) - }) - .then(done) - }) - - it('should create a transaction for forms that contain fields that have to be verified', (done) => { - const spyOnCreate = spyOn(Verification, 'create').and.callThrough() - req.body.formId = testForm._id - res.status.and.callFake(expectStatus(HttpStatus.CREATED)) - res.json.and.callFake(function (json) { - expect(spyOnCreate).toHaveBeenCalled() - Verification.findById(json.transactionId, function (err, result) { - // eslint-disable-next-line no-console - if (err) console.error(err) - expect(json.transactionId).toEqual(result._id) - expect(json.expireAt).toEqual(result.expireAt) - const testFields = testForm.form_fields.filter((f) => f.isVerifiable) - expect(testFields.length).toEqual(result.fields.length) - testFields.forEach((field, i) => { - expect(result.fields[i]._id).toEqual(String(field._id)) - expect(result.fields[i].fieldType).toEqual(field.fieldType) - }) - done() - }) - return this - }) - controller.createTransaction(req, res) - }) - - it('should get transaction', (done) => { - createTransaction(expireAt, [ - { fieldType: 'email' }, - { fieldType: 'mobile' }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - res.status.and.callFake(expectStatus(HttpStatus.OK)) - res.json.and.callFake(function (json) { - expect(json.fields).toEqual(undefined) // fields should not be returned in metadata - Verification.findById(json._id, function (err, result) { - // eslint-disable-next-line no-console - if (err) console.error(err) - expect(json._id).toEqual(result._id) - expect(json.formId).toEqual(result.formId) - expect(json.expireAt).toEqual(result.expireAt) - const testFields = testForm.form_fields.filter( - (f) => f.isVerifiable, - ) - expect(testFields.length).toEqual(result.fields.length) - testFields.forEach((field, i) => { - expect(result.fields[i]._id).toEqual(String(field._id)) - expect(result.fields[i].fieldType).toEqual(field.fieldType) - expect(result.fields[i].hashCreatedAt).toEqual(null) - expect(result.fields[i].hashedOtp).toEqual(null) - expect(result.fields[i].signedData).toEqual(null) - }) - - done() - }) - }) - controller.getTransactionMetadata(req, res) - }) - }) - - it('should create an otp for the email field to be verified and send it out', (done) => { - const hashValue = 'hashValue' - bcrypt.hash.and.returnValue('hashValue') - - createTransaction(expireAt, [ - { fieldType: 'email' }, - { fieldType: 'mobile' }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - answer: 'test@abc.com', - } - res.sendStatus.and.callFake(function (status) { - expect(status).toEqual(HttpStatus.CREATED) - expect(sendOtpSpy).toHaveBeenCalled() - Verification.findById(transaction._id, function (err, result) { - // eslint-disable-next-line no-console - if (err) console.error(err) - const transactionField = result.fields[0] - const testField = testForm.form_fields[0] - expect(transactionField._id).toEqual(String(testField._id)) - expect(transactionField.fieldType).toEqual(testField.fieldType) - // The hashes should have been created - expect(transactionField.hashCreatedAt instanceof Date).toEqual(true) - expect(transactionField.hashedOtp).toEqual(hashValue) - expect(transactionField.signedData).not.toEqual(null) - done() - }) - return this - }) - controller.getNewOtp(req, res) - }) - }) - - it('should create an otp for the mobile sms field to be verified and send it out', (done) => { - const hashValue = 'hashValue' - bcrypt.hash.and.returnValue('hashValue') - createTransaction(expireAt, [ - { fieldType: 'email' }, - { fieldType: 'mobile' }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[1]._id), - answer: '+6583334444', - } - res.sendStatus.and.callFake(function (status) { - expect(status).toEqual(HttpStatus.CREATED) - expect(sendSmsOtp).toHaveBeenCalled() - Verification.findById(transaction._id, function (err, result) { - // eslint-disable-next-line no-console - if (err) console.error(err) - const transactionField = result.fields[1] - const testField = testForm.form_fields[1] - expect(transactionField._id).toEqual(String(testField._id)) - expect(transactionField.fieldType).toEqual(testField.fieldType) - // The hashes should have been created - expect(transactionField.hashCreatedAt instanceof Date).toEqual(true) - expect(transactionField.hashedOtp).toEqual(hashValue) - expect(transactionField.signedData).not.toEqual(null) - done() - }) - return this - }) - controller.getNewOtp(req, res) - }) - }) - - it('should limit the rate of otp requests', (done) => { - const hashCreatedAt = new Date() - const hashedOtp = 'someHash' - const signedData = 'someData' - - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt, - hashedOtp, - signedData, - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - answer: 'test@abc.com', - } - // WAIT_FOR_OTP - res.status.and.callFake(expectStatus(HttpStatus.ACCEPTED)) - res.json.and.callFake(function () { - Verification.findById(transaction._id, function (err, result) { - // eslint-disable-next-line no-console - if (err) console.error(err) - const transactionField = result.fields[0] - const testField = testForm.form_fields[0] - expect(transactionField._id).toEqual(String(testField._id)) - expect(transactionField.fieldType).toEqual(testField.fieldType) - // The hashes should remain the same, no change because a new otp was not created - expect(transactionField.hashCreatedAt).toEqual(hashCreatedAt) - expect(transactionField.hashedOtp).toEqual(hashedOtp) - expect(transactionField.signedData).toEqual(signedData) - done() - }) - }) - controller.getNewOtp(req, res) - }) - }) - - it('should return an error if transaction has expired when getting otp', (done) => { - createTransaction(hasExpired, [ - { - fieldType: 'email', - hashCreatedAt: null, - hashedOtp: null, - signedData: null, - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - answer: 'test@abc.com', - } - // TRANSACTION_NOT_FOUND - res.status.and.callFake(expectStatus(HttpStatus.NOT_FOUND)) - res.json.and.callFake(function () { - done() - }) - controller.getNewOtp(req, res) - }) - }) - it('should return an error if transaction has expired when verifying otp', (done) => { - createTransaction(hasExpired, [ - { - fieldType: 'email', - hashCreatedAt: null, - hashedOtp: null, - signedData: null, - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - otp: '000000', - } - // TRANSACTION_NOT_FOUND - res.status.and.callFake(expectStatus(HttpStatus.NOT_FOUND)) - res.json.and.callFake(function () { - done() - }) - controller.verifyOtp(req, res) - }) - }) - it('should return an error if hash has expired when verifying otp', (done) => { - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt: hasExpired, - hashedOtp: 'someHashValue', - signedData: 'someData', - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - otp: '000000', - } - // RESEND_OTP - res.status.and.callFake(expectStatus(HttpStatus.UNPROCESSABLE_ENTITY)) - res.json.and.callFake(function () { - done() - }) - controller.verifyOtp(req, res) - }) - }) - - it('should limit the rate of otp retries', (done) => { - const hashCreatedAt = new Date() - const hashedOtp = 'someHashValue' - const signedData = 'someData' - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt, - hashedOtp, - signedData, - hashRetries: 4, - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - otp: '000000', - } - res.status.and.callFake(expectStatus(HttpStatus.UNPROCESSABLE_ENTITY)) - res.json.and.callFake(function (json) { - expect(json).toEqual('RESEND_OTP') - done() - }) - controller.verifyOtp(req, res) - }) - }) - - it('should return an error if otp is incorrect when verifying otp', (done) => { - const hashCreatedAt = new Date() - hashCreatedAt.setTime( - expireAt.getTime() - constants.TRANSACTION_EXPIRE_AFTER_SECONDS * 500, - ) - hash(testOtp, 10, function (err, hashedOtp) { - // eslint-disable-next-line no-console - if (err) console.error(err) - - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt, - hashedOtp, - signedData: 'someData', - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - otp: '000000', - } - // INVALID_OTP - res.status.and.callFake(expectStatus(HttpStatus.UNPROCESSABLE_ENTITY)) - res.json.and.callFake(function () { - done() - }) - controller.verifyOtp(req, res) - }) - }) - }) - - it('should return signed data if otp is correct when verifying otp', (done) => { - const hashCreatedAt = new Date() - hashCreatedAt.setTime( - expireAt.getTime() - constants.TRANSACTION_EXPIRE_AFTER_SECONDS * 500, - ) - const signedData = 'someData' - hash(testOtp, 10, function (err, hashedOtp) { - // eslint-disable-next-line no-console - if (err) console.error(err) - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt, - hashedOtp, - signedData, - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - otp: testOtp, - } - res.status.and.callFake(expectStatus(HttpStatus.OK)) - res.json.and.callFake(function (json) { - expect(json).toEqual(signedData) - done() - }) - controller.verifyOtp(req, res) - }) - }) - }) - - it("should reset the otp if field's value has changed", (done) => { - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt: new Date(), - hashedOtp: 'someHashValue', - signedData: 'someData', - }, - ]).then((transaction) => { - req.params.transactionId = transaction.id - req.body = { - fieldId: String(testForm.form_fields[0]._id), - } - res.sendStatus.and.callFake(function (status) { - expect(status).toEqual(HttpStatus.OK) - Verification.findById(transaction._id, function (err, result) { - // eslint-disable-next-line no-console - if (err) console.error(err) - const transactionField = result.fields[0] - const testField = testForm.form_fields[0] - expect(transactionField._id).toEqual(String(testField._id)) - expect(transactionField.fieldType).toEqual(testField.fieldType) - // The hashes should have been reset back to null - expect(transactionField.hashCreatedAt).toEqual(null) - expect(transactionField.hashedOtp).toEqual(null) - expect(transactionField.signedData).toEqual(null) - done() - }) - }) - controller.resetFieldInTransaction(req, res) - }) - }) - }) - - describe('submissions', () => { - let fixtures - - const endpointPath = '/submissions' - const injectFixtures = (req, res, next) => { - Object.assign(req, fixtures) - next() - } - const sendSubmissionBack = (req, res) => { - res.status(200).send({ - body: req.body, - }) - } - - const app = express() - - const sendAndExpect = (status, expectedResponse = null) => { - let send = request(app).post(endpointPath).expect(status) - if (expectedResponse) { - send = send.expect(expectedResponse) - } - return send - } - - beforeAll(() => { - app - .route(endpointPath) - .post( - injectFixtures, - submissionController.validateEmailSubmission, - sendSubmissionBack, - ) - }) - - // Submission - describe('No verified fields in form', () => { - beforeEach((done) => { - testForm = new Form({ - title: 'Test Form', - emails: 'test@test.gov.sg', - admin: testUser._id, - form_fields: [{ title: 'Email', fieldType: 'email' }], - }) - testForm - .save({ validateBeforeSave: false }) - .then(() => { - fixtures = { - form: testForm, - body: { - responses: [], - }, - } - }) - .then(done) - }) - it('should allow submission if transaction does not exist for forms that do not contain any fields that have to be verified', (done) => { - // No transaction created for testForm - const field = testForm.form_fields[0] - const response = { - _id: String(field._id), - fieldType: field.fieldType, - question: field.title, - answer: 'test@abc.com', - } - fixtures.body.responses.push(response) - sendAndExpect(HttpStatus.OK, { - body: { - parsedResponses: [Object.assign(response, { isVisible: true })], - }, - }).end(done) - }) - }) - describe('Verified fields', () => { - let createTransaction - beforeAll((done) => { - testForm = new Form({ - title: 'Test Form', - emails: 'test@test.gov.sg', - admin: testUser._id, - form_fields: [ - { title: 'Email', fieldType: 'email', isVerifiable: true }, - ], - }) - testForm - .save({ validateBeforeSave: false }) - .then(() => { - createTransaction = createTransactionForForm(testForm) - }) - .then(done) - }) - beforeEach(() => { - fixtures = { - form: testForm, - body: { - responses: [], - }, - } - }) - - describe('No transaction', () => { - it('should prevent submission if transaction does not exist for a form containing fields that have to be verified', (done) => { - const field = testForm.form_fields[0] - const response = { - _id: String(field._id), - fieldType: field.fieldType, - question: field.title, - answer: 'test@abc.com', - } - fixtures.body.responses.push(response) - - sendAndExpect(HttpStatus.BAD_REQUEST).end(done) - }) - }) - - describe('Has transaction', () => { - it('should prevent submission if transaction has expired for a form containing fields that have to be verified', (done) => { - createTransaction(hasExpired, [ - { - fieldType: 'email', - hashCreatedAt: new Date(), - hashedOtp: 'someHashValue', - signedData: 'someData', - }, - ]).then(() => { - const field = testForm.form_fields[0] - const response = { - _id: String(field._id), - fieldType: field.fieldType, - question: field.title, - answer: 'test@abc.com', - signature: 'someData', - } - fixtures.body.responses.push(response) - - sendAndExpect(HttpStatus.BAD_REQUEST).end(done) - }) - }) - - it('should prevent submission if any of the transaction fields are not verified', (done) => { - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt: null, - hashedOtp: null, - signedData: null, - }, - ]).then(() => { - const field = testForm.form_fields[0] - const response = { - _id: String(field._id), - fieldType: field.fieldType, - question: field.title, - answer: 'test@abc.com', - } - fixtures.body.responses.push(response) - - sendAndExpect(HttpStatus.BAD_REQUEST).end(done) - }) - }) - - it('should allow submission if all of the transaction fields are verified', (done) => { - const formsg = require('@opengovsg/formsg-sdk')({ - mode: 'test', - verificationOptions: { - secretKey: process.env.VERIFICATION_SECRET_KEY, - }, - }) - const transactionId = mongoose.Types.ObjectId( - '5e71ef8b19c1ed04b54cd5f9', - ) - - const field = testForm.form_fields[0] - const formId = testForm._id - let response = { - _id: String(field._id), - fieldType: field.fieldType, - question: field.title, - answer: 'test@abc.com', - } - const signature = formsg.verification.generateSignature({ - transactionId: String(transactionId), - formId: String(formId), - fieldId: response._id, - answer: response.answer, - }) - response.signature = signature - - createTransaction(expireAt, [ - { - fieldType: 'email', - hashCreatedAt: new Date(), - hashedOtp: 'someHashValue', - signedData: signature, - }, - ]).then(() => { - fixtures.body.responses.push(response) - sendAndExpect(HttpStatus.OK).end(done) - }) - }) - }) - }) - - describe('Hidden and optional fields', () => { - const expireAt = new Date() - expireAt.setTime(expireAt.getTime() + 86400) - - beforeEach(() => { - fixtures = { - form: {}, - body: { - responses: [], - }, - } - }) - - const test = ({ - fieldValue, - fieldIsRequired, - fieldIsHidden, - expectedStatus, - done, - }) => { - const field = { - _id: '5e719d5b62a2c4aa5d9789e2', - title: 'Email', - fieldType: 'email', - isVerifiable: true, - required: fieldIsRequired, - } - const yesNoField = { - _id: '5e719d5b62a2c4aa5d9789e3', - title: 'Show email if this field is yes', - fieldType: 'yes_no', - } - let form = new Form({ - title: 'Test Form', - emails: 'test@test.gov.sg', - admin: testUser._id, - form_fields: [field, yesNoField], - form_logics: [ - { - show: [field._id], - conditions: [ - { - ifValueType: 'single-select', - _id: '58169', - field: yesNoField._id, - state: 'is equals to', - value: 'Yes', - }, - ], - _id: '5db00a15af2ffb29487d4eb1', - logicType: 'showFields', - }, - ], - }) - form.save({ validateBeforeSave: false }).then(() => { - const response = { - _id: String(field._id), - fieldType: field.fieldType, - question: field.title, - answer: fieldValue, - } - const yesNoResponse = { - _id: yesNoField._id, - question: yesNoField.title, - fieldType: yesNoField.fieldType, - answer: fieldIsHidden ? 'No' : 'Yes', - } - fixtures.form = form - fixtures.body.responses.push(yesNoResponse) - fixtures.body.responses.push(response) - sendAndExpect(expectedStatus).end(done) - }) - } - it('should verify fields that are optional and filled in', (done) => { - test({ - fieldValue: 'test@abc.com', - fieldIsRequired: false, - fieldIsHidden: false, - expectedStatus: HttpStatus.BAD_REQUEST, - done, - }) - }) - it('should not verify fields that are optional and not filled in', (done) => { - test({ - fieldValue: '', - fieldIsRequired: false, - fieldIsHidden: false, - expectedStatus: HttpStatus.OK, - done, - }) - }) - it('should verify fields that are required and not hidden by logic', (done) => { - test({ - fieldValue: 'test@abc.com', - fieldIsRequired: true, - fieldIsHidden: false, - expectedStatus: HttpStatus.BAD_REQUEST, - done, - }) - }) - - it('should not verify fields that are required and hidden by logic', (done) => { - test({ - fieldValue: '', - fieldIsRequired: true, - fieldIsHidden: true, - expectedStatus: HttpStatus.OK, - done, - }) - }) - }) - }) -}) diff --git a/tests/unit/backend/helpers/jest-express.ts b/tests/unit/backend/helpers/jest-express.ts new file mode 100644 index 0000000000..f61f9fc168 --- /dev/null +++ b/tests/unit/backend/helpers/jest-express.ts @@ -0,0 +1,16 @@ +import { Response } from 'express' + +interface MockResponse extends Response { + sendStatus: jest.Mock + send: jest.Mock + status: jest.Mock + json: jest.Mock +} + +export const mockResponse = () => + ({ + sendStatus: jest.fn(), + send: jest.fn(), + status: jest.fn().mockReturnThis(), + json: jest.fn(), + } as MockResponse) diff --git a/tests/unit/backend/modules/sns/sns.controller.spec.ts b/tests/unit/backend/modules/sns/sns.controller.spec.ts index 3477fc56ef..50b529b1c2 100644 --- a/tests/unit/backend/modules/sns/sns.controller.spec.ts +++ b/tests/unit/backend/modules/sns/sns.controller.spec.ts @@ -18,18 +18,14 @@ describe('handleSns', () => { mockSnsService.isValidSnsRequest.mockReset() }) test('does not call updateBounces for invalid requests', async () => { - mockSnsService.isValidSnsRequest.mockImplementation(() => - Promise.resolve(false), - ) + mockSnsService.isValidSnsRequest.mockReturnValueOnce(Promise.resolve(false)) await handleSns(req, res) expect(mockSnsService.isValidSnsRequest).toHaveBeenCalledWith(req.body) expect(mockSnsService.updateBounces).not.toHaveBeenCalled() expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.FORBIDDEN) }) test('calls updateBounces for valid requests', async () => { - mockSnsService.isValidSnsRequest.mockImplementation(() => - Promise.resolve(true), - ) + mockSnsService.isValidSnsRequest.mockReturnValueOnce(Promise.resolve(true)) await handleSns(req, res) expect(mockSnsService.isValidSnsRequest).toHaveBeenCalledWith(req.body) expect(mockSnsService.updateBounces).toHaveBeenCalledWith(req.body) @@ -45,9 +41,7 @@ describe('handleSns', () => { expect(res.sendStatus).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST) }) test('catches errors thrown in updateBounces', async () => { - mockSnsService.isValidSnsRequest.mockImplementation(() => - Promise.resolve(true), - ) + mockSnsService.isValidSnsRequest.mockReturnValueOnce(Promise.resolve(true)) mockSnsService.updateBounces.mockImplementation(() => { throw new Error() }) diff --git a/tests/unit/backend/modules/sns/sns.service.spec.ts b/tests/unit/backend/modules/sns/sns.service.spec.ts index 4d679b5613..09b7b58bf1 100644 --- a/tests/unit/backend/modules/sns/sns.service.spec.ts +++ b/tests/unit/backend/modules/sns/sns.service.spec.ts @@ -3,12 +3,13 @@ import { ObjectId } from 'bson' import crypto from 'crypto' import dedent from 'dedent' import { cloneDeep, omit } from 'lodash' +import mongoose from 'mongoose' import { mocked } from 'ts-jest/utils' import * as loggerModule from 'src/config/logger' import { ISnsNotification } from 'src/types' -import dbHandler from '../../helpers/db-handler' +import dbHandler from '../../helpers/jest-db' import getMockLogger, { resetMockLogger } from '../../helpers/jest-logger' import { extractBounceObject, @@ -17,23 +18,25 @@ import { MOCK_SNS_BODY, } from '../../helpers/sns' -const Bounce = dbHandler.makeModel('bounce.server.model', 'Bounce') - jest.mock('axios') const mockAxios = mocked(axios, true) jest.mock('src/config/logger') const mockLoggerModule = mocked(loggerModule, true) const mockLogger = getMockLogger() -mockLoggerModule.createCloudWatchLogger.mockImplementation(() => mockLogger) -mockLoggerModule.createLoggerWithLabel.mockImplementation(() => getMockLogger()) +mockLoggerModule.createCloudWatchLogger.mockReturnValue(mockLogger) +mockLoggerModule.createLoggerWithLabel.mockReturnValue(getMockLogger()) -// Import the service last so that mocks get imported correctly +// Import modules which depend on config last so that mocks get imported correctly +// eslint-disable-next-line import/first +import getBounceModel from 'src/app/models/bounce.server.model' // eslint-disable-next-line import/first import { isValidSnsRequest, updateBounces, } from 'src/app/modules/sns/sns.service' +const Bounce = getBounceModel(mongoose) + describe('isValidSnsRequest', () => { let keys, body: ISnsNotification beforeAll(() => { diff --git a/tests/unit/backend/modules/verification/verification.controller.spec.ts b/tests/unit/backend/modules/verification/verification.controller.spec.ts new file mode 100644 index 0000000000..2f26a5da88 --- /dev/null +++ b/tests/unit/backend/modules/verification/verification.controller.spec.ts @@ -0,0 +1,288 @@ +import { ObjectId } from 'bson' +import HttpStatus from 'http-status-codes' +import mongoose from 'mongoose' +import { mocked } from 'ts-jest/utils' + +import getVerificationModel from 'src/app/models/verification.server.model' +import { + createTransaction, + getNewOtp, + getTransactionMetadata, + resetFieldInTransaction, + verifyOtp, +} from 'src/app/modules/verification/verification.controller' +import * as vfnService from 'src/app/modules/verification/verification.service' + +import { mockResponse } from '../../helpers/jest-express' + +jest.mock('src/app/modules/verification/verification.service') +const mockVfnService = mocked(vfnService, true) +const noop = () => {} +const MOCK_FORM_ID = 'formId' +const MOCK_TRANSACTION_ID = 'transactionId' +const MOCK_FIELD_ID = 'fieldId' +const MOCK_ANSWER = 'answer' +const MOCK_OTP = 'otp' +const MOCK_DATA = 'data' +const mockRes = mockResponse() +const Verification = getVerificationModel(mongoose) +const notFoundError = 'TRANSACTION_NOT_FOUND' +const waitOtpError = 'WAIT_FOR_OTP' +const sendOtpError = 'SEND_OTP_FAILED' +const invalidOtpError = 'INVALID_OTP' + +describe('Verification controller', () => { + describe('createTransaction', () => { + let mockReq + afterEach(() => jest.clearAllMocks()) + + beforeAll(() => { + mockReq = { body: { formId: MOCK_FORM_ID } } + }) + + test('correctly returns transaction', async () => { + const returnValue = { + transactionId: 'Bereft of life, it rests in peace', + expireAt: new Date(), + } + mockVfnService.createTransaction.mockReturnValueOnce( + Promise.resolve(returnValue), + ) + await createTransaction(mockReq, mockRes, noop) + expect(mockVfnService.createTransaction).toHaveBeenCalledWith( + MOCK_FORM_ID, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.CREATED) + expect(mockRes.json).toHaveBeenCalledWith(returnValue) + }) + + test('correctly returns 200 when transaction is not found', async () => { + mockVfnService.createTransaction.mockReturnValueOnce(null) + await createTransaction(mockReq, mockRes, noop) + expect(mockVfnService.createTransaction).toHaveBeenCalledWith( + MOCK_FORM_ID, + ) + expect(mockRes.sendStatus).toHaveBeenCalledWith(HttpStatus.OK) + }) + }) + + describe('getTransactionMetadata', () => { + let mockReq + afterEach(() => jest.clearAllMocks()) + + beforeAll(() => { + mockReq = { params: { transactionId: MOCK_TRANSACTION_ID } } + }) + + test('correctly returns metadata', async () => { + // Coerce type + const transaction = ('test' as unknown) as ReturnType< + typeof mockVfnService.getTransactionMetadata + > + mockVfnService.getTransactionMetadata.mockReturnValueOnce( + Promise.resolve(transaction), + ) + await getTransactionMetadata(mockReq, mockRes, noop) + expect(mockVfnService.getTransactionMetadata).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.OK) + expect(mockRes.json).toHaveBeenCalledWith(transaction) + }) + + test('returns 404 on error', async () => { + mockVfnService.getTransactionMetadata.mockImplementationOnce(() => { + const error = new Error(notFoundError) + error.name = notFoundError + throw error + }) + await getTransactionMetadata(mockReq, mockRes, noop) + expect(mockVfnService.getTransactionMetadata).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.NOT_FOUND) + expect(mockRes.json).toHaveBeenCalledWith(notFoundError) + }) + }) + + describe('resetFieldInTransaction', () => { + let mockReq + afterEach(() => jest.clearAllMocks()) + + beforeAll(() => { + mockReq = { + body: { fieldId: MOCK_FIELD_ID }, + params: { transactionId: MOCK_TRANSACTION_ID }, + } + }) + + test('correctly calls service', async () => { + const transaction = new Verification({ formId: new ObjectId() }) + mockVfnService.getTransaction.mockReturnValueOnce( + Promise.resolve(transaction), + ) + await resetFieldInTransaction(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.resetFieldInTransaction).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + ) + expect(mockRes.sendStatus).toHaveBeenCalledWith(HttpStatus.OK) + }) + + test('returns 404 on error', async () => { + mockVfnService.getTransaction.mockImplementationOnce(() => { + const error = new Error(notFoundError) + error.name = notFoundError + throw error + }) + await resetFieldInTransaction(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.NOT_FOUND) + expect(mockRes.json).toHaveBeenCalledWith(notFoundError) + }) + }) + + describe('getNewOtp', () => { + let mockReq, transaction + afterEach(() => jest.clearAllMocks()) + + beforeAll(() => { + mockReq = { + body: { fieldId: MOCK_FIELD_ID, answer: MOCK_ANSWER }, + params: { transactionId: MOCK_TRANSACTION_ID }, + } + transaction = new Verification({ formId: new ObjectId() }) + mockVfnService.getTransaction.mockReturnValue( + Promise.resolve(transaction), + ) + }) + + test('calls service correctly', async () => { + await getNewOtp(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.getNewOtp).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + MOCK_ANSWER, + ) + expect(mockRes.sendStatus).toHaveBeenCalledWith(HttpStatus.CREATED) + }) + + test('returns 404 on not found error', async () => { + mockVfnService.getNewOtp.mockImplementationOnce(() => { + const error = new Error(notFoundError) + error.name = notFoundError + throw error + }) + await getNewOtp(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.getNewOtp).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + MOCK_ANSWER, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.NOT_FOUND) + expect(mockRes.json).toHaveBeenCalledWith(notFoundError) + }) + + test('returns 202 on WaitForOtp error', async () => { + mockVfnService.getNewOtp.mockImplementationOnce(() => { + const error = new Error(waitOtpError) + error.name = waitOtpError + throw error + }) + await getNewOtp(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.getNewOtp).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + MOCK_ANSWER, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.ACCEPTED) + expect(mockRes.json).toHaveBeenCalledWith(waitOtpError) + }) + + test('returns 400 on OTP failed error', async () => { + mockVfnService.getNewOtp.mockImplementationOnce(() => { + const error = new Error(sendOtpError) + error.name = sendOtpError + throw error + }) + await getNewOtp(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.getNewOtp).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + MOCK_ANSWER, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST) + expect(mockRes.json).toHaveBeenCalledWith(sendOtpError) + }) + }) + + describe('verifyOtp', () => { + let mockReq, transaction + + afterEach(() => jest.clearAllMocks()) + + beforeAll(() => { + mockReq = { + body: { fieldId: MOCK_FIELD_ID, otp: MOCK_OTP }, + params: { transactionId: MOCK_TRANSACTION_ID }, + } + transaction = new Verification({ formId: new ObjectId() }) + mockVfnService.getTransaction.mockReturnValue( + Promise.resolve(transaction), + ) + }) + + test('calls service correctly', async () => { + mockVfnService.verifyOtp.mockReturnValue(Promise.resolve(MOCK_DATA)) + await verifyOtp(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.verifyOtp).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + MOCK_OTP, + ) + expect(mockRes.status).toHaveBeenCalledWith(HttpStatus.OK) + expect(mockRes.json).toHaveBeenCalledWith(MOCK_DATA) + }) + + test('returns 422 for invalid OTP', async () => { + mockVfnService.verifyOtp.mockImplementationOnce(() => { + const error = new Error(invalidOtpError) + error.name = invalidOtpError + throw error + }) + await verifyOtp(mockReq, mockRes, noop) + expect(mockVfnService.getTransaction).toHaveBeenCalledWith( + MOCK_TRANSACTION_ID, + ) + expect(mockVfnService.verifyOtp).toHaveBeenCalledWith( + transaction, + MOCK_FIELD_ID, + MOCK_OTP, + ) + expect(mockRes.status).toHaveBeenCalledWith( + HttpStatus.UNPROCESSABLE_ENTITY, + ) + expect(mockRes.json).toHaveBeenCalledWith(invalidOtpError) + }) + }) +}) diff --git a/tests/unit/backend/modules/verification/verification.service.spec.ts b/tests/unit/backend/modules/verification/verification.service.spec.ts new file mode 100644 index 0000000000..80cc4926b9 --- /dev/null +++ b/tests/unit/backend/modules/verification/verification.service.spec.ts @@ -0,0 +1,488 @@ +import bcrypt from 'bcrypt' +import { ObjectId } from 'bson' +import mongoose from 'mongoose' +import { mocked } from 'ts-jest/utils' + +import smsFactory from 'src/app/factories/sms.factory' +import getFormModel from 'src/app/models/form.server.model' +import getVerificationModel from 'src/app/models/verification.server.model' +import { + createTransaction, + getNewOtp, + getTransactionMetadata, + resetFieldInTransaction, + verifyOtp, +} from 'src/app/modules/verification/verification.service' +import MailService from 'src/app/services/mail.service' +import { otpGenerator } from 'src/config/config' +import formsgSdk from 'src/config/formsg-sdk' +import { BasicFieldType, IUserSchema, IVerificationSchema } from 'src/types' + +import dbHandler from '../../helpers/jest-db' + +const Form = getFormModel(mongoose) +const Verification = getVerificationModel(mongoose) +const MOCK_FORM_TITLE = 'Verification service tests' + +// Set up mocks +jest.mock('src/config/config') +const mockOtpGenerator = mocked(otpGenerator, true) +jest.mock('src/config/formsg-sdk') +const mockFormsgSdk = mocked(formsgSdk, true) +jest.mock('src/app/factories/sms.factory') +const mockSmsFactory = mocked(smsFactory, true) +jest.mock('src/app/services/mail.service') +const mockMailService = mocked(MailService, true) +jest.mock('bcrypt') +const mockBcrypt = mocked(bcrypt, true) + +describe('Verification service', () => { + let user: IUserSchema + beforeAll(async () => { + await dbHandler.connect() + }) + beforeEach(async () => { + const preloadedDocuments = await dbHandler.insertFormCollectionReqs({}) + user = preloadedDocuments.user + }) + afterAll(async () => await dbHandler.closeDatabase()) + + describe('createTransaction', () => { + afterEach(async () => await dbHandler.clearDatabase()) + + test('should return null when form_fields does not exist', async () => { + const testForm = new Form({ + admin: user, + title: MOCK_FORM_TITLE, + }) + await testForm.save() + await expect(createTransaction(testForm._id)).resolves.toBe(null) + // Document should not have been created + await expect( + Verification.findOne({ formId: testForm._id }), + ).resolves.toBe(null) + }) + + test('should return null when there are no verifiable fields', async () => { + const testForm = new Form({ + form_fields: [{ fieldType: BasicFieldType.YesNo }], + admin: user, + title: MOCK_FORM_TITLE, + }) + await testForm.save() + await expect(createTransaction(testForm._id)).resolves.toBe(null) + // Document should not have been created + await expect( + Verification.findOne({ formId: testForm._id }), + ).resolves.toBe(null) + }) + + test('should correctly save and return transaction', async () => { + const testForm = new Form({ + form_fields: [{ fieldType: BasicFieldType.Email, isVerifiable: true }], + admin: user, + title: MOCK_FORM_TITLE, + }) + await testForm.save() + const returnedTransaction = await createTransaction(testForm._id) + const foundTransaction = await Verification.findOne({ + formId: testForm._id, + }) + expect(foundTransaction).toBeTruthy() + expect(returnedTransaction).toEqual({ + transactionId: foundTransaction._id, + expireAt: foundTransaction.expireAt, + }) + }) + }) + + describe('getTransactionMetadata', () => { + afterEach(async () => await dbHandler.clearDatabase()) + + test('should throw error when transaction does not exist', async () => { + return expect( + getTransactionMetadata(String(new ObjectId())), + ).rejects.toThrowError('TRANSACTION_NOT_FOUND') + }) + + test('should correctly return metadata', async () => { + const formId = new ObjectId() + const expireAt = new Date() + const testVerification = new Verification({ formId, expireAt }) + await testVerification.save() + const actual = await getTransactionMetadata(testVerification._id) + expect(actual.toObject()).toEqual({ + _id: testVerification._id, + formId, + expireAt, + }) + }) + }) + + describe('resetFieldInTransaction', () => { + afterEach(async () => await dbHandler.clearDatabase()) + + test('should correctly reset one field', async () => { + const testForm = new Form({ + admin: user, + title: MOCK_FORM_TITLE, + form_fields: [ + { fieldType: BasicFieldType.Email, isVerifiable: true }, + { fieldType: BasicFieldType.Mobile, isVerifiable: true }, + ], + }) + const formId = testForm._id + const hashCreatedAt = new Date() + const hashedOtp = 'hash' + const signedData = 'signedData' + const hashRetries = 1 + const transaction = new Verification({ + formId, + fields: testForm.form_fields.map(({ _id, fieldType }) => ({ + _id, + fieldType, + hashCreatedAt, + hashedOtp, + signedData, + hashRetries, + })), + }) + await transaction.save() + await resetFieldInTransaction(transaction, testForm.form_fields[0]._id) + const actual = await Verification.findOne({ formId }) + expect(actual.fields[0].toObject()).toEqual({ + _id: String(testForm.form_fields[0]._id), + fieldType: testForm.form_fields[0].fieldType, + hashCreatedAt: null, + hashedOtp: null, + signedData: null, + hashRetries: 0, + }) + expect(actual.fields[1].toObject()).toEqual({ + _id: String(testForm.form_fields[1]._id), + fieldType: testForm.form_fields[1].fieldType, + hashCreatedAt, + hashedOtp, + signedData, + hashRetries, + }) + }) + + test('should throw error if field ID does not exist', async () => { + const transaction = new Verification({ formId: new ObjectId() }) + await transaction.save() + return expect( + resetFieldInTransaction(transaction, String(new ObjectId())), + ).rejects.toThrowError('Field not found in transaction') + }) + + test('should throw error if transaction ID does not exist', async () => { + const transaction = new Verification({ formId: new ObjectId() }) + return expect( + resetFieldInTransaction(transaction, String(new ObjectId())), + ).rejects.toThrowError('Field not found in transaction') + }) + }) + + describe('getNewOtp', () => { + let transaction: IVerificationSchema, mockAnswer: string, mockOtp: string + let hashedOtp: string, signedData: string, hashRetries: number + let hashCreatedAt: Date + beforeEach(() => { + jest.clearAllMocks() + mockAnswer = 'answer' + mockOtp = '123456' + hashedOtp = 'hash' + signedData = 'signedData' + hashRetries = 1 + hashCreatedAt = new Date() + const defaultParams = { + hashCreatedAt, + hashedOtp, + signedData, + hashRetries, + isVerifiable: true, + } + transaction = new Verification({ + formId: new ObjectId(), + fields: [ + { + fieldType: BasicFieldType.Email, + ...defaultParams, + _id: new ObjectId(), + }, + { + fieldType: BasicFieldType.Mobile, + ...defaultParams, + _id: new ObjectId(), + }, + ], + expireAt: new Date(Date.now() + 6e5), // so it won't expire in tests + }) + mockOtpGenerator.mockReturnValue(mockOtp) + mockBcrypt.hash.mockReturnValue(Promise.resolve(hashedOtp)) + mockFormsgSdk.verification.generateSignature.mockReturnValue(signedData) + }) + + afterEach(async () => await dbHandler.clearDatabase()) + + test('should throw error for expired transaction', async () => { + transaction.expireAt = new Date(1) + await transaction.save() + return expect( + getNewOtp(transaction, transaction.fields[0]._id, mockAnswer), + ).rejects.toThrowError('TRANSACTION_NOT_FOUND') + }) + + test('should throw error for invalid field ID', async () => { + return expect( + getNewOtp(transaction, String(new ObjectId()), mockAnswer), + ).rejects.toThrowError('Field not found in transaction') + }) + + test('should throw error for OTP requested too soon', async () => { + // 1min in the future + transaction.fields[0].hashCreatedAt = new Date(Date.now() + 6e4) + // Actual error is 'Wait for _ seconds before requesting' + return expect( + getNewOtp(transaction, transaction.fields[0]._id, mockAnswer), + ).rejects.toThrowError('seconds before requesting for a new otp') + }) + + test('should send OTP for email field', async () => { + // Reset field so we can test update later on + transaction.fields[0].hashedOtp = null + transaction.fields[0].signedData = null + transaction.fields[0].hashCreatedAt = null + transaction.fields[0].hashRetries = 1 + await transaction.save() + await getNewOtp(transaction, transaction.fields[0]._id, mockAnswer) + expect(mockOtpGenerator).toHaveBeenCalled() + expect(mockBcrypt.hash.mock.calls[0][0]).toBe(mockOtp) + expect( + mockFormsgSdk.verification.generateSignature.mock.calls[0][0], + ).toEqual({ + transactionId: transaction._id, + formId: transaction.formId, + fieldId: transaction.fields[0]._id, + answer: mockAnswer, + }) + expect(mockMailService.sendVerificationOtp.mock.calls[0]).toEqual([ + mockAnswer, + mockOtp, + ]) + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashCreatedAt).toBeInstanceOf(Date) + expect(foundTransaction.fields[0].hashedOtp).toBe(hashedOtp) + expect(foundTransaction.fields[0].signedData).toBe(signedData) + expect(foundTransaction.fields[0].hashRetries).toBe(0) + }) + + test('should send OTP for mobile field', async () => { + // Reset field so we can test update later on + transaction.fields[1].hashedOtp = null + transaction.fields[1].signedData = null + transaction.fields[1].hashCreatedAt = null + transaction.fields[1].hashRetries = 1 + await transaction.save() + await getNewOtp(transaction, transaction.fields[1]._id, mockAnswer) + expect(mockOtpGenerator).toHaveBeenCalled() + expect(mockBcrypt.hash.mock.calls[0][0]).toBe(mockOtp) + expect( + mockFormsgSdk.verification.generateSignature.mock.calls[0][0], + ).toEqual({ + transactionId: transaction._id, + formId: transaction.formId, + fieldId: transaction.fields[1]._id, + answer: mockAnswer, + }) + expect(mockSmsFactory.sendVerificationOtp.mock.calls[0]).toEqual([ + mockAnswer, + mockOtp, + transaction.formId, + ]) + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[1].hashCreatedAt).toBeInstanceOf(Date) + expect(foundTransaction.fields[1].hashedOtp).toBe(hashedOtp) + expect(foundTransaction.fields[1].signedData).toBe(signedData) + expect(foundTransaction.fields[1].hashRetries).toBe(0) + }) + + test('should catch and re-throw errors thrown when sending email', async () => { + // So we don't trigger WAIT_FOR_SECONDS error + transaction.fields[0].hashCreatedAt = null + transaction.fields[0].signedData = null + transaction.fields[0].hashCreatedAt = null + transaction.fields[0].hashRetries = 1 + await transaction.save() + const myErrorMsg = "I'd like to have an argument please" + mockMailService.sendVerificationOtp.mockImplementationOnce(() => { + throw new Error(myErrorMsg) + }) + return expect( + getNewOtp(transaction, transaction.fields[0]._id, mockAnswer), + ).rejects.toThrowError(myErrorMsg) + }) + + test('should catch and re-throw errors thrown when sending sms', async () => { + // So we don't trigger WAIT_FOR_SECONDS error + transaction.fields[1].hashCreatedAt = null + transaction.fields[1].signedData = null + transaction.fields[1].hashCreatedAt = null + transaction.fields[1].hashRetries = 1 + await transaction.save() + const myErrorMsg = 'Tis but a scratch!' + mockSmsFactory.sendVerificationOtp.mockImplementationOnce(() => { + throw new Error(myErrorMsg) + }) + return expect( + getNewOtp(transaction, transaction.fields[1]._id, mockAnswer), + ).rejects.toThrowError(myErrorMsg) + }) + }) + + describe('verifyOtp', () => { + let mockOtp: string, transaction: IVerificationSchema, hashRetries: number + let signedData: string + beforeEach(() => { + jest.clearAllMocks() + mockOtp = '123456' + hashRetries = 0 + signedData = 'signedData' + const defaultParams = { + hashCreatedAt: new Date(), + hashedOtp: 'hash', + signedData, + hashRetries, + isVerifiable: true, + } + transaction = new Verification({ + formId: new ObjectId(), + fields: [ + { + fieldType: BasicFieldType.Email, + ...defaultParams, + _id: new ObjectId(), + }, + { + fieldType: BasicFieldType.Mobile, + ...defaultParams, + _id: new ObjectId(), + }, + ], + expireAt: new Date(Date.now() + 6e5), // so it won't expire in tests + }) + }) + + afterEach(async () => await dbHandler.clearDatabase()) + + test('should throw error for expired transaction', async () => { + transaction.expireAt = new Date(1) + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).rejects.toThrowError('TRANSACTION_NOT_FOUND') + // Check that database was not updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries) + }) + + test('should throw error for invalid field ID', async () => { + await transaction.save() + await expect( + verifyOtp(transaction, String(new ObjectId()), mockOtp), + ).rejects.toThrowError('Field not found in transaction') + // Check that database was not updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries) + }) + + test('should throw error for invalid hashed OTP', async () => { + transaction.fields[0].hashedOtp = null + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).rejects.toThrowError('RESEND_OTP') + // Check that database was not updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries) + }) + + test('should throw error for invalid hashCreatedAt', async () => { + transaction.fields[0].hashCreatedAt = null + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).rejects.toThrowError('RESEND_OTP') + // Check that database was not updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries) + }) + + test('should throw error for expired hash', async () => { + // 10min 10s ago + transaction.fields[0].hashCreatedAt = new Date(Date.now() - 6.1e5) + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).rejects.toThrowError('RESEND_OTP') + // Check that database was not updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries) + }) + + test('should throw error if too many retries', async () => { + const tooManyRetries = 4 + transaction.fields[0].hashRetries = tooManyRetries + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).rejects.toThrowError('RESEND_OTP') + // Check that database was not updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(tooManyRetries) + }) + + test('should reject invalid OTP', async () => { + mockBcrypt.compare.mockReturnValueOnce(Promise.resolve(false)) + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).rejects.toThrowError('INVALID_OTP') + // Check that database was updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries + 1) + }) + + test('should accept valid OTP', async () => { + mockBcrypt.compare.mockReturnValueOnce(Promise.resolve(true)) + await transaction.save() + await expect( + verifyOtp(transaction, transaction.fields[0]._id, mockOtp), + ).resolves.toBe(signedData) + // Check that database was updated + const foundTransaction = await Verification.findOne({ + _id: transaction._id, + }) + expect(foundTransaction.fields[0].hashRetries).toBe(hashRetries + 1) + }) + }) +}) From 9c79b80eb698609147aca5ecb12d9000574836a3 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Sat, 22 Aug 2020 00:49:44 +0800 Subject: [PATCH 14/27] feat: log form ID in GA event labels (#154) --- .../core/services/gtag.client.service.js | 29 +++++++++++-------- .../view-responses.client.controller.js | 5 +++- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/public/modules/core/services/gtag.client.service.js b/src/public/modules/core/services/gtag.client.service.js index 73fa8602b4..0536d7ebf4 100644 --- a/src/public/modules/core/services/gtag.client.service.js +++ b/src/public/modules/core/services/gtag.client.service.js @@ -139,7 +139,7 @@ function GTag($rootScope, $window) { _gtagEvents('search', { event_category: 'Examples', event_action: 'Click Open Template', - event_label: form.title, + event_label: `${form.title} (${form._id})`, form_id: form._id, }) } @@ -154,7 +154,7 @@ function GTag($rootScope, $window) { _gtagEvents('search', { event_category: 'Examples', event_action: 'Click Close Template', - event_label: form.title, + event_label: `${form.title} (${form._id})`, form_id: form._id, }) } @@ -168,7 +168,7 @@ function GTag($rootScope, $window) { _gtagEvents('search', { event_category: 'Examples', event_action: 'Click Create New Form', - event_label: form.title, + event_label: `${form.title} (${form._id})`, form_id: form._id, }) } @@ -244,7 +244,7 @@ function GTag($rootScope, $window) { let eventParams = { event_category: 'Public Form', event_action: eventAction, - event_label: form.title, + event_label: `${form.title} (${form._id})`, form_id: form._id, } @@ -343,7 +343,8 @@ function GTag($rootScope, $window) { * Logs the start of a storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.formTitle The title of the form + * @param {String} params.formTitle The title of the form + * @param {String} params.userEmail The email of the user downloading * @param {number} expectedNumSubmissions The expected number of submissions to download * @param {number} numWorkers The number of decryption workers * @return {Void} @@ -356,7 +357,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Download start', - event_label: params.formTitle, + event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, form_id: params.formId, num_workers: numWorkers, num_submissions: expectedNumSubmissions, @@ -367,7 +368,8 @@ function GTag($rootScope, $window) { * Logs a successful storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.formTitle The title of the form + * @param {String} params.formTitle The title of the form + * @param {String} params.userEmail The email of the user downloading * @param {number} downloadedNumSubmissions The number of submissions downloaded * @param {number} numWorkers The number of decryption workers * @param {number} duration The duration taken by the download @@ -382,7 +384,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Download success', - event_label: params.formTitle, + event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, form_id: params.formId, duration: duration, num_workers: numWorkers, @@ -394,7 +396,8 @@ function GTag($rootScope, $window) { * Logs a failed storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.formTitle The title of the form + * @param {String} params.formTitle The title of the form + * @param {String} params.userEmail The email of the user downloading * @param {number} numWorkers The number of decryption workers * @param {number} expectedNumSubmissions The expected number of submissions * @param {number} duration The duration taken by the download @@ -411,7 +414,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Download failure', - event_label: params.formTitle, + event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, form_id: params.formId, duration: duration, num_workers: numWorkers, @@ -424,6 +427,7 @@ function GTag($rootScope, $window) { * Logs a failed attempt to even start storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form + * @param {String} params.userEmail The email of the user downloading * @param {String} params.formTitle The title of the form * @param {string} errorMessage The error message for the failure * @return {Void} @@ -432,7 +436,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Network failure', - event_label: params.formTitle, + event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, form_id: params.formId, message: errorMessage, }) @@ -442,6 +446,7 @@ function GTag($rootScope, $window) { * Logs partial (or full) decryption failure when downloading responses. * @param {Object} params The response params object * @param {String} params.formId ID of the form + * @param {String} params.userEmail The email of the user downloading * @param {String} params.formTitle The title of the form * @param {number} numWorkers The number of decryption workers * @param {number} expectedNumSubmissions The expected number of submissions @@ -459,7 +464,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Partial decrypt error', - event_label: params.formTitle, + event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, form_id: params.formId, duration: duration, num_workers: numWorkers, diff --git a/src/public/modules/forms/admin/controllers/view-responses.client.controller.js b/src/public/modules/forms/admin/controllers/view-responses.client.controller.js index 551e2ded4e..626afdcbe1 100644 --- a/src/public/modules/forms/admin/controllers/view-responses.client.controller.js +++ b/src/public/modules/forms/admin/controllers/view-responses.client.controller.js @@ -16,6 +16,7 @@ angular '$timeout', 'moment', 'FormSgSdk', + '$window', ViewResponsesController, ]) @@ -29,6 +30,7 @@ function ViewResponsesController( $timeout, moment, FormSgSdk, + $window, ) { const vm = this @@ -61,11 +63,12 @@ function ViewResponsesController( // Trigger for export CSV vm.exportCsv = function () { + const userDetails = JSON.parse($window.localStorage.getItem("user")) let params = { formId: vm.myform._id, formTitle: vm.myform.title, + userEmail: userDetails.email, } - if (vm.datePicker.date.startDate && vm.datePicker.date.endDate) { params.startDate = moment(new Date(vm.datePicker.date.startDate)).format( 'YYYY-MM-DD', From ecbfa40334d73c21705c77b313f21f6fc883f5b0 Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Mon, 24 Aug 2020 10:52:30 +0800 Subject: [PATCH 15/27] feat: migrate `util/response` to new Submission module (service, utils, etc) (#176) * refactor: rename BasicFieldType to BasicField * feat: combine old FieldType with BasicField type in shared/resources * refactor(LogicUtils): use export instead of module.exports * refactor: update type definition of baseField for discriminator types * feat: migrate response util to Typescript * feat: add typing for possible form field type * fix(ResponseUtils): fix incorrect validation sequence * test(ResponseUtils): add initial tests (some empty) * refactor(ResponseUtils): use imported FieldResponse type instead * test(ResponseUtils): add success flow tests * refactor: move response utils to submission module * tests(SubmissionService): add tests * test(SubmissionUtils): add tests * fix: update verification tests to include responseMode in form type This is just a bandaid fix, but there is no point rehauling all the tests in this commit since new tests are going to be written soon during the controller refactor. --- .../email-submissions.server.controller.js | 13 +- .../encrypt-submissions.server.controller.js | 12 +- src/app/models/field/baseField.ts | 8 +- src/app/models/field/tableField.ts | 8 +- src/app/models/form.server.model.ts | 59 ++- .../modules/submission/submission.service.ts | 103 +++++ .../modules/submission/submission.types.ts | 6 + .../modules/submission/submission.utils.ts | 30 ++ src/app/utils/response.js | 94 ----- src/shared/resources/basic/index.ts | 61 +-- src/shared/resources/myinfo/index.ts | 68 ++-- src/shared/util/logic.ts | 12 +- src/shared/util/verification.ts | 4 +- src/types/field/baseField.ts | 7 +- src/types/field/emailField.ts | 4 +- src/types/field/fieldTypes.ts | 2 +- src/types/field/index.ts | 41 ++ src/types/field/mobileField.ts | 4 +- src/types/field/tableField.ts | 6 +- src/types/response.ts | 4 +- ...mail-submissions.server.controller.spec.js | 3 + tests/unit/backend/helpers/jest-db.ts | 4 +- .../backend/models/form_fields.schema.spec.ts | 6 +- .../submission/submission.service.spec.ts | 367 ++++++++++++++++++ .../submission/submission.utils.spec.ts | 55 +++ .../verification/verification.service.spec.ts | 18 +- 26 files changed, 732 insertions(+), 267 deletions(-) create mode 100644 src/app/modules/submission/submission.service.ts create mode 100644 src/app/modules/submission/submission.types.ts create mode 100644 src/app/modules/submission/submission.utils.ts delete mode 100644 src/app/utils/response.js create mode 100644 tests/unit/backend/modules/submission/submission.service.spec.ts create mode 100644 tests/unit/backend/modules/submission/submission.utils.spec.ts diff --git a/src/app/controllers/email-submissions.server.controller.js b/src/app/controllers/email-submissions.server.controller.js index c5465b39c2..77270307dd 100644 --- a/src/app/controllers/email-submissions.server.controller.js +++ b/src/app/controllers/email-submissions.server.controller.js @@ -9,8 +9,6 @@ const mongoose = require('mongoose') const { getEmailSubmissionModel } = require('../models/submission.server.model') const emailSubmission = getEmailSubmissionModel(mongoose) const HttpStatus = require('http-status-codes') -const { FIELDS_TO_REJECT } = require('../utils/field-validation/config') -const { getParsedResponses } = require('../utils/response') const { getRequestIp } = require('../utils/request') const { ConflictError } = require('../utils/custom-errors') const { MB } = require('../constants/filesize') @@ -22,6 +20,9 @@ const { mapAttachmentsFromParsedResponses, } = require('../utils/attachment') const config = require('../../config/config') +const { + getProcessedResponses, +} = require('../modules/submission/submission.service') const logger = require('../../config/logger').createLoggerWithLabel( 'email-submissions', ) @@ -209,13 +210,7 @@ exports.validateEmailSubmission = function (req, res, next) { if (req.body.responses) { try { - const emailModeFilter = (arr) => - arr.filter(({ fieldType }) => !FIELDS_TO_REJECT.includes(fieldType)) - req.body.parsedResponses = getParsedResponses( - form, - req.body.responses, - emailModeFilter, - ) + req.body.parsedResponses = getProcessedResponses(form, req.body.responses) delete req.body.responses // Prevent downstream functions from using responses by deleting it } catch (err) { logger.error(`ip="${getRequestIp(req)}" error=`, err) diff --git a/src/app/controllers/encrypt-submissions.server.controller.js b/src/app/controllers/encrypt-submissions.server.controller.js index 8adf8c91ea..daf23d9647 100644 --- a/src/app/controllers/encrypt-submissions.server.controller.js +++ b/src/app/controllers/encrypt-submissions.server.controller.js @@ -14,7 +14,6 @@ const Submission = getSubmissionModel(mongoose) const encryptSubmission = getEncryptSubmissionModel(mongoose) const { checkIsEncryptedEncoding } = require('../utils/encryption') -const { getParsedResponses } = require('../utils/response') const { ConflictError } = require('../utils/custom-errors') const { getRequestIp } = require('../utils/request') const { isMalformedDate, createQueryWithDateParam } = require('../utils/date') @@ -24,6 +23,9 @@ const logger = require('../../config/logger').createLoggerWithLabel( const { aws: { attachmentS3Bucket, s3 }, } = require('../../config/config') +const { + getProcessedResponses, +} = require('../modules/submission/submission.service') /** * Extracts relevant fields, injects questions, verifies visibility of field and validates answers @@ -51,13 +53,7 @@ exports.validateEncryptSubmission = function (req, res, next) { if (req.body.responses) { try { - const encryptModeFilter = (arr) => - arr.filter(({ fieldType }) => ['mobile', 'email'].includes(fieldType)) // For autoreplies - req.body.parsedResponses = getParsedResponses( - form, - req.body.responses, - encryptModeFilter, - ) + req.body.parsedResponses = getProcessedResponses(form, req.body.responses) delete req.body.responses // Prevent downstream functions from using responses by deleting it } catch (err) { logger.error(`ip="${getRequestIp(req)}" error=`, err) diff --git a/src/app/models/field/baseField.ts b/src/app/models/field/baseField.ts index fe8dd609a6..0ba8534cdf 100644 --- a/src/app/models/field/baseField.ts +++ b/src/app/models/field/baseField.ts @@ -7,7 +7,7 @@ import { } from '../../../app/utils/beta-permissions' import { AuthType, - BasicFieldType, + BasicField, IFieldSchema, IMyInfoSchema, ITableFieldSchema, @@ -18,7 +18,7 @@ import getUserModel from '../user.server.model' const uidgen3 = new UIDGenerator(256, UIDGenerator.BASE62) -const VALID_FIELD_TYPES = Object.values(BasicFieldType) +const VALID_FIELD_TYPES = Object.values(BasicField) export const MyInfoSchema = new Schema( { @@ -67,7 +67,7 @@ const createBaseFieldSchema = (db: Mongoose) => { }, fieldType: { type: String, - enum: Object.values(BasicFieldType), + enum: Object.values(BasicField), required: true, }, }, @@ -143,7 +143,7 @@ const createBaseFieldSchema = (db: Mongoose) => { // Typeguards const isTableField = (field: IFieldSchema): field is ITableFieldSchema => { - return field.fieldType === BasicFieldType.Table + return field.fieldType === BasicField.Table } export default createBaseFieldSchema diff --git a/src/app/models/field/tableField.ts b/src/app/models/field/tableField.ts index cd0c0ac7e7..4a22dc25c4 100644 --- a/src/app/models/field/tableField.ts +++ b/src/app/models/field/tableField.ts @@ -1,11 +1,7 @@ import { isEmpty } from 'lodash' import { Schema } from 'mongoose' -import { - BasicFieldType, - IColumnSchema, - ITableFieldSchema, -} from '../../../types' +import { BasicField, IColumnSchema, ITableFieldSchema } from '../../../types' const createColumnSchema = () => { const ColumnSchema = new Schema( @@ -27,7 +23,7 @@ const createColumnSchema = () => { ) ColumnSchema.pre('validate', function (next) { - let columnTypes = [BasicFieldType.ShortText, BasicFieldType.Dropdown] + let columnTypes = [BasicField.ShortText, BasicField.Dropdown] let index = columnTypes.indexOf(this.columnType) if (index > -1) { return next() diff --git a/src/app/models/form.server.model.ts b/src/app/models/form.server.model.ts index 83300d8241..85dee57b08 100644 --- a/src/app/models/form.server.model.ts +++ b/src/app/models/form.server.model.ts @@ -7,7 +7,7 @@ import { FORM_DUPLICATE_KEYS } from '../../shared/constants' import { validateWebhookUrl } from '../../shared/util/webhook-validation' import { AuthType, - BasicFieldType, + BasicField, Colors, FormLogoState, IEmailFormSchema, @@ -322,58 +322,43 @@ const compileFormModel = (db: Mongoose): IFormModel => { const TableFieldSchema = createTableFieldSchema() - FormFieldPath.discriminator(BasicFieldType.Email, createEmailFieldSchema()) - FormFieldPath.discriminator(BasicFieldType.Rating, createRatingFieldSchema()) + FormFieldPath.discriminator(BasicField.Email, createEmailFieldSchema()) + FormFieldPath.discriminator(BasicField.Rating, createRatingFieldSchema()) FormFieldPath.discriminator( - BasicFieldType.Attachment, + BasicField.Attachment, createAttachmentFieldSchema(), ) + FormFieldPath.discriminator(BasicField.Dropdown, createDropdownFieldSchema()) + FormFieldPath.discriminator(BasicField.Radio, createRadioFieldSchema()) + FormFieldPath.discriminator(BasicField.Checkbox, createCheckboxFieldSchema()) FormFieldPath.discriminator( - BasicFieldType.Dropdown, - createDropdownFieldSchema(), - ) - FormFieldPath.discriminator(BasicFieldType.Radio, createRadioFieldSchema()) - FormFieldPath.discriminator( - BasicFieldType.Checkbox, - createCheckboxFieldSchema(), - ) - FormFieldPath.discriminator( - BasicFieldType.ShortText, + BasicField.ShortText, createShortTextFieldSchema(), ) - FormFieldPath.discriminator(BasicFieldType.HomeNo, createHomenoFieldSchema()) - FormFieldPath.discriminator(BasicFieldType.Mobile, createMobileFieldSchema()) + FormFieldPath.discriminator(BasicField.HomeNo, createHomenoFieldSchema()) + FormFieldPath.discriminator(BasicField.Mobile, createMobileFieldSchema()) + FormFieldPath.discriminator(BasicField.LongText, createLongTextFieldSchema()) + FormFieldPath.discriminator(BasicField.Number, createNumberFieldSchema()) + FormFieldPath.discriminator(BasicField.Decimal, createDecimalFieldSchema()) + FormFieldPath.discriminator(BasicField.Image, createImageFieldSchema()) + FormFieldPath.discriminator(BasicField.Date, createDateFieldSchema()) + FormFieldPath.discriminator(BasicField.Nric, createNricFieldSchema()) + FormFieldPath.discriminator(BasicField.YesNo, createYesNoFieldSchema()) FormFieldPath.discriminator( - BasicFieldType.LongText, - createLongTextFieldSchema(), - ) - FormFieldPath.discriminator(BasicFieldType.Number, createNumberFieldSchema()) - FormFieldPath.discriminator( - BasicFieldType.Decimal, - createDecimalFieldSchema(), - ) - FormFieldPath.discriminator(BasicFieldType.Image, createImageFieldSchema()) - FormFieldPath.discriminator(BasicFieldType.Date, createDateFieldSchema()) - FormFieldPath.discriminator(BasicFieldType.Nric, createNricFieldSchema()) - FormFieldPath.discriminator(BasicFieldType.YesNo, createYesNoFieldSchema()) - FormFieldPath.discriminator( - BasicFieldType.Statement, + BasicField.Statement, createStatementFieldSchema(), ) - FormFieldPath.discriminator( - BasicFieldType.Section, - createSectionFieldSchema(), - ) - FormFieldPath.discriminator(BasicFieldType.Table, TableFieldSchema) + FormFieldPath.discriminator(BasicField.Section, createSectionFieldSchema()) + FormFieldPath.discriminator(BasicField.Table, TableFieldSchema) const TableColumnPath = TableFieldSchema.path( 'columns', ) as Schema.Types.DocumentArray TableColumnPath.discriminator( - BasicFieldType.ShortText, + BasicField.ShortText, createShortTextFieldSchema(), ) TableColumnPath.discriminator( - BasicFieldType.Dropdown, + BasicField.Dropdown, createDropdownFieldSchema(), ) diff --git a/src/app/modules/submission/submission.service.ts b/src/app/modules/submission/submission.service.ts new file mode 100644 index 0000000000..fbfdcfb8fb --- /dev/null +++ b/src/app/modules/submission/submission.service.ts @@ -0,0 +1,103 @@ +import _ from 'lodash' + +import { + getLogicUnitPreventingSubmit, + getVisibleFieldIds, +} from '../../../shared/util/logic' +import { FieldResponse, IFieldSchema, IFormSchema } from '../../../types' +import { ConflictError } from '../../utils/custom-errors' +import validateField from '../../utils/field-validation' + +import { ProcessedFieldResponse } from './submission.types' +import { getModeFilter } from './submission.utils' + +/** + * Filter allowed form field responses from given responses and return the + * array of responses with duplicates removed. + * + * @param form The form document + * @param responses the responses that corresponds to the given form + * @returns filtered list of allowed responses with duplicates (if any) removed + * @throws ConflictError if the given form's form field ids count do not match given responses' + */ +const getFilteredResponses = ( + form: IFormSchema, + responses: FieldResponse[], +) => { + const modeFilter = getModeFilter(form.responseMode) + + // _id must be transformed to string as form response is jsonified. + const fieldIds = modeFilter(form.form_fields).map((field) => ({ + _id: String(field._id), + })) + const uniqueResponses = _.uniqBy(modeFilter(responses), '_id') + const results = _.intersectionBy(uniqueResponses, fieldIds, '_id') + + if (results.length < fieldIds.length) { + const onlyInForm = _.differenceBy(fieldIds, results, '_id').map( + ({ _id }) => _id, + ) + throw new ConflictError( + `formId="${form._id}" message="Some form fields are missing" onlyInForm="${onlyInForm}"`, + ) + } + return results +} + +/** + * Injects response metadata such as the question, visibility state. In + * addition, validation such as input validation or signature validation on + * verified fields are also performed on the response. + * @param form The form document corresponding to the responses + * @param responses The responses to process and validate + * @returns field responses with additional metadata injected. + * @throws Error if response validation fails + */ +export const getProcessedResponses = ( + form: IFormSchema, + originalResponses: FieldResponse[], +) => { + const filteredResponses = getFilteredResponses(form, originalResponses) + + // Set of all visible fields + const visibleFieldIds = getVisibleFieldIds(filteredResponses, form) + + // Guard against invalid form submissions that should have been prevented by + // logic. + if (getLogicUnitPreventingSubmit(filteredResponses, form, visibleFieldIds)) { + throw new Error('Submission prevented by form logic') + } + + // Create a map keyed by field._id for easier access + const fieldMap = form.form_fields.reduce<{ + [fieldId: string]: IFieldSchema + }>((acc, field) => { + acc[field._id] = field + return acc + }, {}) + + // Validate each field in the form and inject metadata into the responses. + const processedResponses = filteredResponses.map((response) => { + const responseId = response._id + const formField = fieldMap[responseId] + if (!formField) { + throw new Error('Response ID does not match form field IDs') + } + + const processingResponse: ProcessedFieldResponse = { + ...response, + isVisible: visibleFieldIds.has(responseId), + question: formField.getQuestion(), + } + + if (formField.isVerifiable) { + processingResponse.isUserVerified = formField.isVerifiable + } + + // Error will be thrown if the processed response is not valid. + validateField(form._id, formField, processingResponse) + return processingResponse + }) + + return processedResponses +} diff --git a/src/app/modules/submission/submission.types.ts b/src/app/modules/submission/submission.types.ts new file mode 100644 index 0000000000..7be81cf527 --- /dev/null +++ b/src/app/modules/submission/submission.types.ts @@ -0,0 +1,6 @@ +import { FieldResponse } from 'src/types' + +export type ProcessedFieldResponse = FieldResponse & { + isVisible?: boolean + isUserVerified?: boolean +} diff --git a/src/app/modules/submission/submission.utils.ts b/src/app/modules/submission/submission.utils.ts new file mode 100644 index 0000000000..2dce7da9ac --- /dev/null +++ b/src/app/modules/submission/submission.utils.ts @@ -0,0 +1,30 @@ +import { BasicField, ResponseMode } from '../../../types' +import { FIELDS_TO_REJECT } from '../../utils/field-validation/config' + +type ModeFilterParam = { + fieldType: BasicField +} + +export const getModeFilter = (responseMode: ResponseMode) => { + switch (responseMode) { + case ResponseMode.Email: + return emailModeFilter + case ResponseMode.Encrypt: + return encryptModeFilter + default: + throw Error('getResponsesForEachField: Invalid response mode parameter') + } +} + +const emailModeFilter = (responses: T[]) => { + return responses.filter( + ({ fieldType }) => !FIELDS_TO_REJECT.includes(fieldType), + ) +} + +const encryptModeFilter = (responses: T[] = []) => { + // To filter for autoreply-able fields. + return responses.filter(({ fieldType }) => + [BasicField.Mobile, BasicField.Email].includes(fieldType), + ) +} diff --git a/src/app/utils/response.js b/src/app/utils/response.js deleted file mode 100644 index b1dd5473f3..0000000000 --- a/src/app/utils/response.js +++ /dev/null @@ -1,94 +0,0 @@ -const validateField = require('./field-validation') -const logicUtil = require('../../shared/util/logic') -const _ = require('lodash') -const { ConflictError } = require('./custom-errors.js') - -/** - * For each form field, select the first response that is available for that field - * - * @param {Object} form - * @param {Mongoose.ObjectId} form._id - * @param {Array} form.form_fields - * @param {Array} responses - * @param {Function} modeFilter - * @returns {Array} one response for each field id - */ -const getResponsesForEachField = function (form, responses, modeFilter) { - const fieldIds = modeFilter(form.form_fields).map((field) => { - return { _id: String(field._id) } - }) - const uniqueResponses = _.uniqBy(modeFilter(responses), '_id') - const results = _.intersectionBy(uniqueResponses, fieldIds, '_id') - if (results.length < fieldIds.length) { - const onlyInForm = _.differenceBy(fieldIds, results, '_id').map( - ({ _id }) => _id, - ) - throw new ConflictError( - `formId="${form._id}" message="Some form fields are missing" onlyInForm="${onlyInForm}"`, - ) - } - return results -} - -/** - * Check that a form submission should have been prevented by logic. If so, - * throw an error. - * @param {Object} form - * @param {Array} responses - * @param {Set} visibleFieldIds - * @throws {Error} Throws an error if form submission should have been prevented by logic. - */ -const validatePreventSubmitLogic = function (form, responses, visibleFieldIds) { - if ( - logicUtil.getLogicUnitPreventingSubmit(responses, form, visibleFieldIds) - ) { - throw new Error('Submission prevented by form logic') - } -} - -/** - * Construct parsed responses by checking visibility and injecting questions - * - * @param {Object} form - * @param {Array} bodyResponses - * @param {Function} modeFilter - * @returns {Array} - */ -const getParsedResponses = function (form, bodyResponses, modeFilter) { - const responses = getResponsesForEachField(form, bodyResponses, modeFilter) - - // Create a map keyed by field._id for easier access - const fieldMap = form.form_fields.reduce((acc, field) => { - acc[field._id] = field - return acc - }, {}) - - // Set of all visible fields - const visibleFieldIds = logicUtil.getVisibleFieldIds(responses, form) - validatePreventSubmitLogic(form, responses, visibleFieldIds) - - // Validate each field in the form and construct parsed responses for downstream processing - const parsedResponses = responses.map((response) => { - const { _id } = response - // In FormValidator, we have checked that all the form field ids exist, so - // this wont be null. - const formField = fieldMap[_id] - response.isVisible = visibleFieldIds.has(_id) - validateField(form._id, formField, response) - // Instance method of base field schema. - response.question = formField.getQuestion() - if (formField.isVerifiable) { - // This is only correct because validateField should have thrown an error - // if the signature was wrong. - response.isUserVerified = true - } - - return response - }) - - return parsedResponses -} - -module.exports = { - getParsedResponses, -} diff --git a/src/shared/resources/basic/index.ts b/src/shared/resources/basic/index.ts index d774f3ee1c..9cde47cbf8 100644 --- a/src/shared/resources/basic/index.ts +++ b/src/shared/resources/basic/index.ts @@ -1,123 +1,104 @@ -export type FieldType = - | 'section' - | 'statement' - | 'email' - | 'mobile' - | 'homeno' - | 'number' - | 'decimal' - | 'image' - | 'textfield' - | 'textarea' - | 'dropdown' - | 'yes_no' - | 'checkbox' - | 'radiobutton' - | 'attachment' - | 'date' - | 'rating' - | 'nric' - | 'table' +import { BasicField } from '../../../types' interface IBasicFieldType { - name: FieldType + name: BasicField value: string submitted: boolean } export const types: IBasicFieldType[] = [ { - name: 'section', + name: BasicField.Section, value: 'Header', submitted: true, }, { - name: 'statement', + name: BasicField.Statement, value: 'Statement', submitted: false, }, { - name: 'email', + name: BasicField.Email, value: 'Email', submitted: true, }, { - name: 'mobile', + name: BasicField.Mobile, value: 'Mobile Number', submitted: true, }, { - name: 'homeno', + name: BasicField.HomeNo, value: 'Home Number', submitted: true, }, { - name: 'number', + name: BasicField.Number, value: 'Number', submitted: true, }, { - name: 'decimal', + name: BasicField.Decimal, value: 'Decimal', submitted: true, }, { - name: 'image', + name: BasicField.Image, value: 'Image', submitted: false, }, { - name: 'textfield', + name: BasicField.ShortText, value: 'Short Text', submitted: true, }, { - name: 'textarea', + name: BasicField.LongText, value: 'Long Text', submitted: true, }, { - name: 'dropdown', + name: BasicField.Dropdown, value: 'Dropdown', submitted: true, }, { - name: 'yes_no', + name: BasicField.YesNo, value: 'Yes/No', submitted: true, }, { - name: 'checkbox', + name: BasicField.Checkbox, value: 'Checkbox', submitted: true, }, { - name: 'radiobutton', + name: BasicField.Radio, value: 'Radio', submitted: true, }, { - name: 'attachment', + name: BasicField.Attachment, value: 'Attachment', submitted: true, }, { - name: 'date', + name: BasicField.Date, value: 'Date', submitted: true, }, { - name: 'rating', + name: BasicField.Rating, value: 'Rating', submitted: true, }, { - name: 'nric', + name: BasicField.Nric, value: 'NRIC', submitted: true, }, { - name: 'table', + name: BasicField.Table, value: 'Table', submitted: true, }, diff --git a/src/shared/resources/myinfo/index.ts b/src/shared/resources/myinfo/index.ts index 4077d89f72..844af60035 100644 --- a/src/shared/resources/myinfo/index.ts +++ b/src/shared/resources/myinfo/index.ts @@ -1,4 +1,4 @@ -import { FieldType } from '../basic' +import { BasicField } from '../../../types' import COUNTRIES from './myinfo-countries' import DIALECTS from './myinfo-dialects' @@ -47,7 +47,7 @@ interface IMyInfoFieldType { verified: MyInfoVerifiedType[] source: string description: string - fieldType: FieldType + fieldType: BasicField fieldOptions?: string[] ValidationOptions?: object } @@ -68,7 +68,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Immigration & Checkpoints Authority / Ministry of Manpower', description: 'The registered name of the form-filler. This field is verified by ICA for Singaporeans/PRs & foreigners on Long-Term Visit Pass, and by MOM for Employment Pass holders.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'sex', @@ -78,7 +78,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Immigration & Checkpoints Authority / Ministry of Manpower', description: 'The gender of the form-filler. This field is verified by ICA for Singaporeans/PRs & foreigners on Long-Term Visit Pass, and by MOM for Employment Pass holders.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: ['FEMALE', 'MALE', 'UNKNOWN'], }, { @@ -89,7 +89,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Immigration & Checkpoints Authority / Ministry of Manpower', description: 'The registered name of the form-filler. This field is verified by ICA for Singaporeans/PRs & foreigners on Long-Term Visit Pass, and by MOM for Employment Pass holders.', - fieldType: 'date', + fieldType: BasicField.Date, }, { name: 'race', @@ -99,7 +99,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Immigration & Checkpoints Authority / Ministry of Manpower', description: 'The race of the form-filler. This field is verified by ICA for Singaporean/PRs & foreigners on Long-Term Visit Pass, and by MOM for Employment Pass holders.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: RACES, }, { @@ -110,7 +110,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Immigration & Checkpoints Authority / Ministry of Manpower', description: 'The nationality of the form-filler. This field is verified by ICA for Singaporeans/PRs & foreigners on Long-Term Visit Pass, and by MOM for Employment Pass holders.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: NATIONALITIES, }, { @@ -121,14 +121,14 @@ export const types: IMyInfoFieldType[] = [ source: 'Immigration & Checkpoints Authority / Ministry of Manpower', description: 'The birth country of the form-filler. This field is verified by ICA for Singaporeans/PRs & foreigners on Long-Term Visit Pass, and by MOM for Employment Pass holders.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: COUNTRIES, }, // { // name: 'secondaryrace', // value: 'Race (Secondary)', // category: "personal", - // fieldType: 'dropdown' + // fieldType: BasicField.Dropdown // }, { name: 'residentialstatus', @@ -137,7 +137,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['SG', 'PR'], source: 'Immigration and Checkpoints Authority', description: 'The residential status of the form-filler.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: ['Alien', 'Citizen', 'NOT APPLICABLE', 'PR', 'Unknown'], }, { @@ -147,7 +147,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['SG', 'PR'], source: 'Immigration and Checkpoints Authority', description: 'The dialect group of the form-filler.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: DIALECTS, }, { @@ -158,7 +158,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Housing Development Board / Urban Redevelopment Authority', description: 'The type of housing that the form-filler lives in. This information is verified by HDB for public housing, and by URA for private housing.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: [ 'APARTMENT', 'CONDOMINIUM', @@ -175,7 +175,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['SG', 'PR'], source: 'Housing Development Board', description: 'The type of HDB flat that the form-filler lives in.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: [ '1-ROOM FLAT (HDB)', '2-ROOM FLAT (HDB)', @@ -193,7 +193,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['SG'], source: 'Immigration & Checkpoints Authority', description: 'The passport number of the form-filler.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'passportexpirydate', @@ -202,7 +202,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['SG'], source: 'Immigration & Checkpoints Authority', description: 'The passport expiry date of the form-filler.', - fieldType: 'date', + fieldType: BasicField.Date, }, { name: 'marital', @@ -212,7 +212,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Social and Family Development', description: 'The marital status of the form-filler. This field is treated as unverified, as data provided by MSF may be outdated in cases of marriages in a foreign country.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: ['SINGLE', 'MARRIED', 'WIDOWED', 'DIVORCED'], }, { @@ -222,7 +222,7 @@ export const types: IMyInfoFieldType[] = [ verified: [], source: 'User-provided', description: 'Highest education level of form-filler.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: [ 'NO FORMAL QUALIFICATION / PRE-PRIMARY / LOWER PRIMARY', 'PRIMARY', @@ -245,7 +245,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Social and Family Development', description: 'The country of marriage of the form-filler. This field is treated as unverified, as data provided by MSF may be outdated in cases of marriages in a foreign country.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: COUNTRIES, }, // { @@ -255,7 +255,7 @@ export const types: IMyInfoFieldType[] = [ // verified: [], // source: "User-provided" // description: "", - // fieldType: 'dropdown', + // fieldType: BasicField.Dropdown, // }, // { // name: 'marriedname', @@ -288,7 +288,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['SG', 'PR'], source: 'Immigration & Checkpoints Authority', description: 'The registered address of the form-filler.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'mailadd', @@ -297,7 +297,7 @@ export const types: IMyInfoFieldType[] = [ verified: [], source: 'User-provided', description: 'The mailing address of the form-filler.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'billadd', @@ -306,7 +306,7 @@ export const types: IMyInfoFieldType[] = [ verified: [], source: 'User-provided', description: 'The billing address of the form-filler.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'schoolname', @@ -316,7 +316,7 @@ export const types: IMyInfoFieldType[] = [ source: 'User-provided', description: 'List of primary, secondary and tertiary educational institutions in Singapore. Does not include private or international educational institutions.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: SCHOOLS, }, { @@ -327,7 +327,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Manpower', description: 'The occupation of the form-filler. Verified for foreigners with SingPass only.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: OCCUPATIONS, }, { @@ -338,7 +338,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Manpower', description: "The name of the form-filler's employer. Verified for foreigners with SingPass only.", - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'vehno', @@ -347,7 +347,7 @@ export const types: IMyInfoFieldType[] = [ verified: [], source: 'User-provided', description: 'Vehicle plate number of the form-filler.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'marriagecertno', @@ -357,7 +357,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Social and Family Development', description: 'Marriage Certificate Number of form-filler. This field is treated as unverified, as data provided by MSF may be outdated in cases of marriages in a foreign country.', - fieldType: 'textfield', + fieldType: BasicField.ShortText, }, { name: 'marriagedate', @@ -367,7 +367,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Social and Family Development', description: 'The date of marriage of the form-filler. This field is treated as unverified, as data provided by MSF may be outdated in cases of marriages in a foreign country.', - fieldType: 'date', + fieldType: BasicField.Date, }, { name: 'divorcedate', @@ -377,7 +377,7 @@ export const types: IMyInfoFieldType[] = [ source: 'Ministry of Social and Family Development', description: 'The date of divorce of the form-filler. This field is treated as unverified, as data provided by MSF may be outdated in cases of marriages in a foreign country.', - fieldType: 'date', + fieldType: BasicField.Date, }, { name: 'workpassstatus', @@ -386,7 +386,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['F'], source: 'Ministry of Manpower', description: 'Workpass application status of foreigner.', - fieldType: 'dropdown', + fieldType: BasicField.Dropdown, fieldOptions: ['Live', 'Approved'], }, { @@ -396,7 +396,7 @@ export const types: IMyInfoFieldType[] = [ verified: ['F'], source: 'Ministry of Manpower', description: 'The workpass expiry date of the form-filler.', - fieldType: 'date', + fieldType: BasicField.Date, }, { name: 'mobileno', @@ -405,7 +405,7 @@ export const types: IMyInfoFieldType[] = [ verified: [], source: 'User-provided', description: 'Mobile telephone number of form-filler.', - fieldType: 'mobile', + fieldType: BasicField.Mobile, }, { name: 'homeno', @@ -414,7 +414,7 @@ export const types: IMyInfoFieldType[] = [ verified: [], source: 'User-provided', description: 'Home telephone number of form-filler.', - fieldType: 'homeno', + fieldType: BasicField.HomeNo, }, { name: 'gradyear', @@ -424,7 +424,7 @@ export const types: IMyInfoFieldType[] = [ source: 'User-provided', description: "Graduation year of form filler's last attended educational institution.", - fieldType: 'number', + fieldType: BasicField.Number, ValidationOptions: { selectedValidation: 'Exact', customVal: 4, diff --git a/src/shared/util/logic.ts b/src/shared/util/logic.ts index 772f9a4fcc..cc60255450 100644 --- a/src/shared/util/logic.ts +++ b/src/shared/util/logic.ts @@ -63,7 +63,7 @@ const isPreventSubmitLogic = ( * @param {Array} form.form_fields : An array of form fields containing the ids of the fields * @returns {Object} Object containing fields to be displayed and their corresponding conditions, keyed by id of the displayable field */ -const groupLogicUnitsByField = (form: IForm): GroupedLogic => { +export const groupLogicUnitsByField = (form: IForm): GroupedLogic => { const formId = form._id const formLogics = form.form_logics.filter(isShowFieldsLogic) const formFieldIds = new Set( @@ -126,7 +126,7 @@ const getPreventSubmitConditions = ( * provided, the function recomputes it. * @returns {Object} Condition if submission is to prevented, otherwise undefined */ -const getLogicUnitPreventingSubmit = ( +export const getLogicUnitPreventingSubmit = ( submission: LogicFieldArray, form: IForm, visibleFieldIds?: FieldIdSet, @@ -166,7 +166,7 @@ const allConditionsExist = ( * @var {Array} logicUnits - Array of logic units * @returns {Set} Set of IDs of visible fields */ -const getVisibleFieldIds = ( +export const getVisibleFieldIds = ( submission: LogicFieldArray, form: IForm, ): FieldIdSet => { @@ -307,9 +307,3 @@ const findConditionField = ( (submittedField) => String(submittedField._id) === String(fieldId), ) } - -module.exports = { - groupLogicUnitsByField, - getVisibleFieldIds, - getLogicUnitPreventingSubmit, -} diff --git a/src/shared/util/verification.ts b/src/shared/util/verification.ts index 382e8d3a30..7373f92888 100644 --- a/src/shared/util/verification.ts +++ b/src/shared/util/verification.ts @@ -1,6 +1,6 @@ -import { BasicFieldType } from '../../types' +import { BasicField } from '../../types' -export const VERIFIED_FIELDTYPES = [BasicFieldType.Email, BasicFieldType.Mobile] +export const VERIFIED_FIELDTYPES = [BasicField.Email, BasicField.Mobile] export const SALT_ROUNDS = 10 export const TRANSACTION_EXPIRE_AFTER_SECONDS = 14400 // 4 hours export const HASH_EXPIRE_AFTER_SECONDS = 600 // 10 minutes diff --git a/src/types/field/baseField.ts b/src/types/field/baseField.ts index 321c4eb179..4f346fb43a 100644 --- a/src/types/field/baseField.ts +++ b/src/types/field/baseField.ts @@ -2,7 +2,7 @@ import { Document } from 'mongoose' import { IFormSchema } from '../form' -import { BasicFieldType, MyInfoAttribute } from './fieldTypes' +import { BasicField, MyInfoAttribute } from './fieldTypes' export interface IMyInfo { attr: MyInfoAttribute @@ -21,7 +21,7 @@ export interface IField { description: string required: boolean disabled: boolean - fieldType: BasicFieldType + fieldType: BasicField myInfo?: IMyInfo _id: Document['_id'] } @@ -33,6 +33,9 @@ export interface IFieldSchema extends IField, Document { /** Returns this sub-documents parent document. */ parent(): IFormSchema + // Discriminatable parameter + isVerifiable?: boolean + // Instance methods /** * Returns the string to be displayed as the asked question in form diff --git a/src/types/field/emailField.ts b/src/types/field/emailField.ts index 20773ad571..0e076d3e35 100644 --- a/src/types/field/emailField.ts +++ b/src/types/field/emailField.ts @@ -13,4 +13,6 @@ export interface IEmailField extends IField { isVerifiable: boolean } -export interface IEmailFieldSchema extends IEmailField, IFieldSchema {} +export interface IEmailFieldSchema extends IEmailField, IFieldSchema { + isVerifiable: boolean +} diff --git a/src/types/field/fieldTypes.ts b/src/types/field/fieldTypes.ts index 7f3ad3b9ac..cff5b880c3 100644 --- a/src/types/field/fieldTypes.ts +++ b/src/types/field/fieldTypes.ts @@ -1,4 +1,4 @@ -export enum BasicFieldType { +export enum BasicField { Section = 'section', Statement = 'statement', Email = 'email', diff --git a/src/types/field/index.ts b/src/types/field/index.ts index f8c5a56b90..c5221ebd44 100644 --- a/src/types/field/index.ts +++ b/src/types/field/index.ts @@ -1,3 +1,23 @@ +import { IAttachmentField } from './attachmentField' +import { ICheckboxField } from './checkboxField' +import { IDateField } from './dateField' +import { IDecimalField } from './decimalField' +import { IDropdownField } from './dropdownField' +import { IEmailField } from './emailField' +import { IHomenoField } from './homeNoField' +import { IImageField } from './imageField' +import { ILongTextField } from './longTextField' +import { IMobileField } from './mobileField' +import { INricField } from './nricField' +import { INumberField } from './numberField' +import { IRadioField } from './radioField' +import { IRatingField } from './ratingField' +import { ISectionField } from './sectionField' +import { IShortTextField } from './shortTextField' +import { IStatementField } from './statementField' +import { ITableField } from './tableField' +import { IYesNoField } from './yesNoField' + export * from './fieldTypes' export * from './baseField' export * from './attachmentField' @@ -19,3 +39,24 @@ export * from './shortTextField' export * from './statementField' export * from './tableField' export * from './yesNoField' + +export type PossibleField = + | IAttachmentField + | ICheckboxField + | IDateField + | IDecimalField + | IDropdownField + | IEmailField + | IHomenoField + | IImageField + | ILongTextField + | IMobileField + | INricField + | INumberField + | IRadioField + | IRatingField + | ISectionField + | IShortTextField + | IStatementField + | ITableField + | IYesNoField diff --git a/src/types/field/mobileField.ts b/src/types/field/mobileField.ts index fe583271b4..c83d242b15 100644 --- a/src/types/field/mobileField.ts +++ b/src/types/field/mobileField.ts @@ -5,4 +5,6 @@ export interface IMobileField extends IField { isVerifiable: boolean } -export interface IMobileFieldSchema extends IMobileField, IFieldSchema {} +export interface IMobileFieldSchema extends IMobileField, IFieldSchema { + isVerifiable: boolean +} diff --git a/src/types/field/tableField.ts b/src/types/field/tableField.ts index 069be54df7..2f4fb31812 100644 --- a/src/types/field/tableField.ts +++ b/src/types/field/tableField.ts @@ -3,14 +3,14 @@ import { Document } from 'mongoose' import { IFormSchema } from '../form' import { IField, IFieldSchema } from './baseField' -import { BasicFieldType } from './fieldTypes' +import { BasicField } from './fieldTypes' export interface IColumn { title: string required: boolean - // Allow all BasicFieldTypes, but pre-validate hook will block non-dropdown/ + // Allow all BasicFields, but pre-validate hook will block non-dropdown/ // non-textfield types. - columnType: BasicFieldType + columnType: BasicField } // Manual override since mongoose types don't have generics yet. diff --git a/src/types/response.ts b/src/types/response.ts index d52cc433e0..24fb110d74 100644 --- a/src/types/response.ts +++ b/src/types/response.ts @@ -1,10 +1,10 @@ -import { BasicFieldType, IFieldSchema, IMyInfo } from './field' +import { BasicField, IFieldSchema, IMyInfo } from './field' export type AttachmentsMap = Record interface IBaseResponse { _id: IFieldSchema['_id'] - fieldType: BasicFieldType + fieldType: BasicField question: string myInfo?: IMyInfo } diff --git a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js index cb13ef5f25..816733ecc1 100644 --- a/tests/unit/backend/controllers/email-submissions.server.controller.spec.js +++ b/tests/unit/backend/controllers/email-submissions.server.controller.spec.js @@ -2832,6 +2832,7 @@ describe('Email Submissions Controller', () => { title: 'Test Form', emails: 'test@test.gov.sg', admin: testUser._id, + responseMode: 'email', form_fields: [{ title: 'Email', fieldType: 'email' }], }) testForm @@ -2869,6 +2870,7 @@ describe('Email Submissions Controller', () => { testForm = new Form({ title: 'Test Form', emails: 'test@test.gov.sg', + responseMode: 'email', admin: testUser._id, form_fields: [ { title: 'Email', fieldType: 'email', isVerifiable: true }, @@ -3028,6 +3030,7 @@ describe('Email Submissions Controller', () => { let form = new Form({ title: 'Test Form', emails: 'test@test.gov.sg', + responseMode: 'email', admin: testUser._id, form_fields: [field, yesNoField], form_logics: [ diff --git a/tests/unit/backend/helpers/jest-db.ts b/tests/unit/backend/helpers/jest-db.ts index a3dc159ddf..c3db14d9c0 100644 --- a/tests/unit/backend/helpers/jest-db.ts +++ b/tests/unit/backend/helpers/jest-db.ts @@ -56,7 +56,7 @@ const insertFormCollectionReqs = async ({ userId?: ObjectID mailName?: string mailDomain?: string -}) => { +} = {}) => { const Agency = getAgencyModel(mongoose) const User = getUserModel(mongoose) @@ -69,7 +69,7 @@ const insertFormCollectionReqs = async ({ const user = await User.create({ email: `${mailName}@${mailDomain}`, - _id: userId, + _id: userId ?? new ObjectID(), agency: agency.id, }) diff --git a/tests/unit/backend/models/form_fields.schema.spec.ts b/tests/unit/backend/models/form_fields.schema.spec.ts index c1e5302d26..ff0cd9b410 100644 --- a/tests/unit/backend/models/form_fields.schema.spec.ts +++ b/tests/unit/backend/models/form_fields.schema.spec.ts @@ -2,7 +2,7 @@ import { ObjectID } from 'bson' import mongoose from 'mongoose' import getFormModel from 'src/app/models/form.server.model' -import { BasicFieldType, ResponseMode } from 'src/types' +import { BasicField, ResponseMode } from 'src/types' import dbHandler from '../helpers/jest-db' @@ -38,12 +38,12 @@ describe('Form Field Schema', () => { it('should return field title when field type is not a table field', async () => { // Arrange // Get all field types - const fieldTypes = Object.values(BasicFieldType) + const fieldTypes = Object.values(BasicField) // Asserts fieldTypes.forEach(async (type) => { // Skip table field. - if (type === BasicFieldType.Table) return + if (type === BasicField.Table) return const fieldTitle = `test ${type} field title` const field = await createAndReturnFormField({ fieldType: type, diff --git a/tests/unit/backend/modules/submission/submission.service.spec.ts b/tests/unit/backend/modules/submission/submission.service.spec.ts new file mode 100644 index 0000000000..97936c8e34 --- /dev/null +++ b/tests/unit/backend/modules/submission/submission.service.spec.ts @@ -0,0 +1,367 @@ +import { ObjectID } from 'bson' +import { cloneDeep } from 'lodash' +import mongoose from 'mongoose' + +import getFormModel from 'src/app/models/form.server.model' +import * as SubmissionService from 'src/app/modules/submission/submission.service' +import { ProcessedFieldResponse } from 'src/app/modules/submission/submission.types' +import { FIELDS_TO_REJECT } from 'src/app/utils/field-validation/config' +import * as LogicUtil from 'src/shared/util/logic' +import { + AttachmentSize, + BasicField, + FieldResponse, + IEmailFormSchema, + IEncryptedFormSchema, + IPreventSubmitLogicSchema, + ISingleAnswerResponse, + LogicType, + PossibleField, + ResponseMode, +} from 'src/types' + +import dbHandler from '../../helpers/jest-db' + +const Form = getFormModel(mongoose) + +const MOCK_ADMIN_ID = new ObjectID() +const MOCK_FORM_PARAMS = { + title: 'Test Form', + admin: MOCK_ADMIN_ID, +} +const MOCK_ENCRYPTED_FORM_PARAMS = { + ...MOCK_FORM_PARAMS, + publicKey: 'mockPublicKey', + responseMode: 'encrypt', +} +const MOCK_EMAIL_FORM_PARAMS = { + ...MOCK_FORM_PARAMS, + emails: ['test@example.com'], + responseMode: 'email', +} + +// Declare here so the array is static. +const FIELD_TYPES = Object.values(BasicField) +const TYPE_TO_INDEX_MAP = (() => { + const map: { [field: string]: number } = {} + FIELD_TYPES.forEach((type, index) => { + map[type] = index + }) + return map +})() + +describe('submission.service', () => { + let defaultEmailForm: IEmailFormSchema + let defaultEmailResponses: FieldResponse[] + let defaultEncryptForm: IEncryptedFormSchema + let defaultEncryptResponses: FieldResponse[] + + beforeAll(async () => { + await dbHandler.connect() + await dbHandler.insertFormCollectionReqs({ userId: MOCK_ADMIN_ID }) + + const defaultFormFields = generateDefaultFields() + + defaultEmailForm = (await createAndReturnFormWithFields( + defaultFormFields, + ResponseMode.Email, + )) as IEmailFormSchema + defaultEncryptForm = (await createAndReturnFormWithFields( + defaultFormFields, + ResponseMode.Encrypt, + )) as IEncryptedFormSchema + + // Process default responses + defaultEmailResponses = defaultEmailForm.form_fields.map((field) => { + return { + _id: String(field._id), + fieldType: field.fieldType, + question: field.getQuestion(), + answer: '', + } + }) + defaultEncryptResponses = defaultEncryptForm.form_fields.map((field) => { + return { + _id: String(field._id), + fieldType: field.fieldType, + question: field.getQuestion(), + answer: '', + } + }) + }) + afterAll(async () => await dbHandler.closeDatabase()) + + describe('getProcessedResponses', () => { + it('should return list of parsed responses for encrypted form submission successfully', async () => { + // Arrange + // Only mobile and email fields are parsed, since the other fields are + // e2e encrypted from the browser. + const mobileFieldIndex = TYPE_TO_INDEX_MAP[BasicField.Mobile] + const emailFieldIndex = TYPE_TO_INDEX_MAP[BasicField.Email] + + // Add answers to both mobile and email fields + const updatedResponses = cloneDeep(defaultEncryptResponses) + const newEmailResponse: ISingleAnswerResponse = { + ...updatedResponses[emailFieldIndex], + answer: 'test@example.com', + } + const newMobileResponse: ISingleAnswerResponse = { + ...updatedResponses[mobileFieldIndex], + answer: '+6587654321', + } + updatedResponses[mobileFieldIndex] = newMobileResponse + updatedResponses[emailFieldIndex] = newEmailResponse + + // Act + const actual = SubmissionService.getProcessedResponses( + defaultEncryptForm, + updatedResponses, + ) + + // Assert + const expectedParsed: ProcessedFieldResponse[] = [ + { ...newEmailResponse, isVisible: true }, + { ...newMobileResponse, isVisible: true }, + ] + // Should only have email and mobile fields for encrypted forms. + expect(actual).toEqual(expectedParsed) + }) + + it('should return list of parsed responses for email form submission successfully', async () => { + // Arrange + // Add answer to subset of field types + const shortTextFieldIndex = TYPE_TO_INDEX_MAP[BasicField.ShortText] + const decimalFieldIndex = TYPE_TO_INDEX_MAP[BasicField.Decimal] + + // Add answers to both selected fields. + const updatedResponses = cloneDeep(defaultEmailResponses) + const newShortTextResponse: ISingleAnswerResponse = { + ...updatedResponses[shortTextFieldIndex], + answer: 'the quick brown fox jumps over the lazy dog', + } + const newDecimalResponse: ISingleAnswerResponse = { + ...updatedResponses[decimalFieldIndex], + answer: '3.142', + } + updatedResponses[shortTextFieldIndex] = newShortTextResponse + updatedResponses[decimalFieldIndex] = newDecimalResponse + + // Act + const actual = SubmissionService.getProcessedResponses( + defaultEmailForm, + updatedResponses, + ) + + // Assert + // Expect metadata to be injected to all responses (except fields to + // reject). + const expectedParsed: ProcessedFieldResponse[] = [] + updatedResponses.forEach((response, index) => { + if (FIELDS_TO_REJECT.includes(response.fieldType)) { + return + } + const expectedProcessed: ProcessedFieldResponse = { + ...response, + isVisible: true, + } + + if (defaultEmailForm.form_fields[index].isVerifiable) { + expectedProcessed.isUserVerified = true + } + expectedParsed.push(expectedProcessed) + }) + + expect(actual).toEqual(expectedParsed) + }) + + it('should throw error when email form has more fields than responses', async () => { + // Arrange + const extraFieldForm = cloneDeep(defaultEmailForm) + const secondMobileField = cloneDeep( + extraFieldForm.form_fields[TYPE_TO_INDEX_MAP[BasicField.Mobile]], + ) + secondMobileField._id = new ObjectID() + extraFieldForm.form_fields.push(secondMobileField) + + // Act + Assert + expect(() => + SubmissionService.getProcessedResponses( + extraFieldForm, + defaultEmailResponses, + ), + ).toThrowError('Some form fields are missing') + }) + + it('should throw error when encrypt form has more fields than responses', async () => { + // Arrange + const extraFieldForm = cloneDeep(defaultEncryptForm) + const secondMobileField = cloneDeep( + extraFieldForm.form_fields[TYPE_TO_INDEX_MAP[BasicField.Mobile]], + ) + secondMobileField._id = new ObjectID() + extraFieldForm.form_fields.push(secondMobileField) + + // Act + Assert + expect(() => + SubmissionService.getProcessedResponses( + extraFieldForm, + defaultEncryptResponses, + ), + ).toThrowError('Some form fields are missing') + }) + + it('should throw error when any responses are not valid for encrypted form submission ', async () => { + // Arrange + // Only mobile and email fields are parsed, since the other fields are + // e2e encrypted from the browser. + const mobileFieldIndex = TYPE_TO_INDEX_MAP[BasicField.Mobile] + + const requireMobileEncryptForm = cloneDeep(defaultEncryptForm) + requireMobileEncryptForm.form_fields[mobileFieldIndex].required = true + + // Act + Assert + expect(() => + SubmissionService.getProcessedResponses( + requireMobileEncryptForm, + defaultEncryptResponses, + ), + ).toThrowError('Invalid answer submitted') + }) + + it('should throw error when any responses are not valid for email form submission ', async () => { + // Arrange + // Set NRIC field in form as required. + const nricFieldIndex = TYPE_TO_INDEX_MAP[BasicField.Nric] + const requireNricEmailForm = cloneDeep(defaultEmailForm) + requireNricEmailForm.form_fields[nricFieldIndex].required = true + + // Act + Assert + expect(() => + SubmissionService.getProcessedResponses( + requireNricEmailForm, + defaultEmailResponses, + ), + ).toThrowError('Invalid answer submitted') + }) + + it('should throw error when encrypted form submission is prevented by logic', async () => { + // Arrange + // Mock logic util to return non-empty to check if error is thrown + jest + .spyOn(LogicUtil, 'getLogicUnitPreventingSubmit') + .mockReturnValueOnce({ + preventSubmitMessage: 'mock prevent submit', + conditions: [], + logicType: LogicType.PreventSubmit, + _id: 'some id', + } as IPreventSubmitLogicSchema) + + // Act + Assert + expect(() => + SubmissionService.getProcessedResponses( + defaultEncryptForm, + defaultEncryptResponses, + ), + ).toThrowError('Submission prevented by form logic') + }) + + it('should throw error when email form submission is prevented by logic', async () => { + // Arrange + // Mock logic util to return non-empty to check if error is thrown + jest + .spyOn(LogicUtil, 'getLogicUnitPreventingSubmit') + .mockReturnValueOnce({ + preventSubmitMessage: 'mock prevent submit', + conditions: [], + logicType: LogicType.PreventSubmit, + _id: 'some id', + } as IPreventSubmitLogicSchema) + + // Act + Assert + expect(() => + SubmissionService.getProcessedResponses( + defaultEmailForm, + defaultEmailResponses, + ), + ).toThrowError('Submission prevented by form logic') + }) + }) +}) + +const createAndReturnFormWithFields = async ( + formFieldParamsList: Partial[], + formType: ResponseMode = ResponseMode.Email, +) => { + let baseParams + + switch (formType) { + case ResponseMode.Email: + baseParams = MOCK_EMAIL_FORM_PARAMS + break + case ResponseMode.Encrypt: + baseParams = MOCK_ENCRYPTED_FORM_PARAMS + break + default: + baseParams = MOCK_FORM_PARAMS + } + + const processedParamList = formFieldParamsList.map((params) => { + // Insert required params if they do not exist. + if (params.fieldType === 'attachment') { + params = { attachmentSize: AttachmentSize.ThreeMb, ...params } + } + if (params.fieldType === 'image') { + params = { + url: 'http://example.com', + fileMd5Hash: 'some hash', + name: 'test image name', + size: 'some size', + ...params, + } + } + + return params + }) + + const formParam = { + ...baseParams, + form_fields: processedParamList, + } + const form = await Form.create(formParam) + + return form +} + +const generateDefaultFields = () => { + // Get all field types + const formFields: Partial[] = FIELD_TYPES.map((fieldType) => { + const fieldTitle = `test ${fieldType} field title` + if (fieldType === BasicField.Table) { + return { + title: fieldTitle, + minimumRows: 1, + columns: [ + { + title: 'Test Column Title 1', + required: false, + columnType: BasicField.ShortText, + }, + { + title: 'Test Column Title 2', + required: false, + columnType: BasicField.Dropdown, + }, + ], + fieldType, + } + } + + return { + fieldType, + title: fieldTitle, + required: false, + } + }) + + return formFields +} diff --git a/tests/unit/backend/modules/submission/submission.utils.spec.ts b/tests/unit/backend/modules/submission/submission.utils.spec.ts new file mode 100644 index 0000000000..854914e77b --- /dev/null +++ b/tests/unit/backend/modules/submission/submission.utils.spec.ts @@ -0,0 +1,55 @@ +import { getModeFilter } from 'src/app/modules/submission/submission.utils' +import { BasicField, ResponseMode } from 'src/types' + +describe('submission.utils', () => { + describe('getModeFilter', () => { + const ALL_FIELD_TYPES = Object.values(BasicField).map((fieldType) => ({ + fieldType, + })) + + it('should return emailMode filter when ResponseMode.Email is passed', async () => { + // Act + const modeFilter = getModeFilter(ResponseMode.Email) + const actual = modeFilter(ALL_FIELD_TYPES) + + // Assert + expect(modeFilter.name).toEqual('emailModeFilter') + // Should filter out image and statement fields in email mode. + const typesToBeFiltered = [BasicField.Image, BasicField.Statement] + typesToBeFiltered.forEach((fieldType) => { + expect(actual).not.toContainEqual( + expect.objectContaining({ + fieldType, + }), + ) + }) + expect(actual.length).toEqual( + ALL_FIELD_TYPES.length - typesToBeFiltered.length, + ) + }) + + it('should return encryptMode filter when ResponseMode.Encrypt is passed', async () => { + // Act + const modeFilter = getModeFilter(ResponseMode.Encrypt) + const actual = modeFilter(ALL_FIELD_TYPES) + + // Assert + expect(modeFilter.name).toEqual('encryptModeFilter') + // Should only return verifiable fields. + expect(actual).toEqual( + expect.arrayContaining([ + { fieldType: BasicField.Mobile }, + { fieldType: BasicField.Email }, + ]), + ) + expect(actual.length).toEqual(2) + }) + + it('should throw error if called with invalid responseMode', async () => { + // Act + Assert + expect(() => getModeFilter(undefined)).toThrowError( + 'getResponsesForEachField: Invalid response mode parameter', + ) + }) + }) +}) diff --git a/tests/unit/backend/modules/verification/verification.service.spec.ts b/tests/unit/backend/modules/verification/verification.service.spec.ts index 80cc4926b9..ae39e714bc 100644 --- a/tests/unit/backend/modules/verification/verification.service.spec.ts +++ b/tests/unit/backend/modules/verification/verification.service.spec.ts @@ -16,7 +16,7 @@ import { import MailService from 'src/app/services/mail.service' import { otpGenerator } from 'src/config/config' import formsgSdk from 'src/config/formsg-sdk' -import { BasicFieldType, IUserSchema, IVerificationSchema } from 'src/types' +import { BasicField, IUserSchema, IVerificationSchema } from 'src/types' import dbHandler from '../../helpers/jest-db' @@ -65,7 +65,7 @@ describe('Verification service', () => { test('should return null when there are no verifiable fields', async () => { const testForm = new Form({ - form_fields: [{ fieldType: BasicFieldType.YesNo }], + form_fields: [{ fieldType: BasicField.YesNo }], admin: user, title: MOCK_FORM_TITLE, }) @@ -79,7 +79,7 @@ describe('Verification service', () => { test('should correctly save and return transaction', async () => { const testForm = new Form({ - form_fields: [{ fieldType: BasicFieldType.Email, isVerifiable: true }], + form_fields: [{ fieldType: BasicField.Email, isVerifiable: true }], admin: user, title: MOCK_FORM_TITLE, }) @@ -127,8 +127,8 @@ describe('Verification service', () => { admin: user, title: MOCK_FORM_TITLE, form_fields: [ - { fieldType: BasicFieldType.Email, isVerifiable: true }, - { fieldType: BasicFieldType.Mobile, isVerifiable: true }, + { fieldType: BasicField.Email, isVerifiable: true }, + { fieldType: BasicField.Mobile, isVerifiable: true }, ], }) const formId = testForm._id @@ -207,12 +207,12 @@ describe('Verification service', () => { formId: new ObjectId(), fields: [ { - fieldType: BasicFieldType.Email, + fieldType: BasicField.Email, ...defaultParams, _id: new ObjectId(), }, { - fieldType: BasicFieldType.Mobile, + fieldType: BasicField.Mobile, ...defaultParams, _id: new ObjectId(), }, @@ -364,12 +364,12 @@ describe('Verification service', () => { formId: new ObjectId(), fields: [ { - fieldType: BasicFieldType.Email, + fieldType: BasicField.Email, ...defaultParams, _id: new ObjectId(), }, { - fieldType: BasicFieldType.Mobile, + fieldType: BasicField.Mobile, ...defaultParams, _id: new ObjectId(), }, From 2ab4c0844c43017c08c9fdd7ec7aceea30227baf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 13:36:53 +0800 Subject: [PATCH 16/27] fix(deps): bump font-awesome from 4.6.1 to 4.7.0 (#186) Bumps [font-awesome](https://github.com/FortAwesome/Font-Awesome) from 4.6.1 to 4.7.0. - [Release notes](https://github.com/FortAwesome/Font-Awesome/releases) - [Changelog](https://github.com/FortAwesome/Font-Awesome/blob/master/CHANGELOG.md) - [Commits](https://github.com/FortAwesome/Font-Awesome/compare/v4.6.1...v4.7.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index de73070aad..9f0a680384 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12156,9 +12156,9 @@ } }, "font-awesome": { - "version": "4.6.1", - "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.6.1.tgz", - "integrity": "sha1-VHJl+0xFu+2Qq4vE93qXs3uFKhI=" + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz", + "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM=" }, "for-in": { "version": "1.0.2", diff --git a/package.json b/package.json index 2369093670..1f1d1c9e2b 100644 --- a/package.json +++ b/package.json @@ -116,7 +116,7 @@ "fetch-readablestream": "^0.2.0", "file-loader": "^4.0.0", "file-saver": "^2.0.2", - "font-awesome": "4.6.1", + "font-awesome": "4.7.0", "glob": "^7.1.2", "has-ansi": "^4.0.0", "helmet": "^3.21.3", From 47630e293e997d8f466a28dd4fde92578f2759d9 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Mon, 24 Aug 2020 14:20:08 +0800 Subject: [PATCH 17/27] chore(deps-dev): bump typescript to 4.0.2 (#196) --- package-lock.json | 12 +++++++++--- package.json | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9f0a680384..423c3b94db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23990,6 +23990,12 @@ "is-number": "^3.0.0", "repeat-string": "^1.6.1" } + }, + "typescript": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", + "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "dev": true } } }, @@ -24947,9 +24953,9 @@ } }, "typescript": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.0.2.tgz", + "integrity": "sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ==", "dev": true }, "uglify-js": { diff --git a/package.json b/package.json index 1f1d1c9e2b..8b0156e047 100644 --- a/package.json +++ b/package.json @@ -229,7 +229,7 @@ "ts-mock-imports": "^1.3.0", "ts-node": "^8.10.2", "ts-node-dev": "^1.0.0-pre.44", - "typescript": "^3.9.7", + "typescript": "^4.0.2", "url-loader": "^1.1.2", "webpack": "^4.43.0", "webpack-cli": "^3.3.12", From a1582372c1b8e65a36ea3a7a2759f4f09fcbcc73 Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Mon, 24 Aug 2020 14:20:37 +0800 Subject: [PATCH 18/27] feat: add core ApplicationError for express app (#195) --- src/app/modules/core/core.errors.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/app/modules/core/core.errors.ts diff --git a/src/app/modules/core/core.errors.ts b/src/app/modules/core/core.errors.ts new file mode 100644 index 0000000000..9f43d02939 --- /dev/null +++ b/src/app/modules/core/core.errors.ts @@ -0,0 +1,28 @@ +/** + * A custom base error class that encapsulates the name, message, status code, + * and logging meta string (if any) for the error. + */ +export class ApplicationError extends Error { + /** + * Http status code for the error to be returned in the response. + */ + status: number + /** + * Meta string to be logged by the application logger, if any. + */ + meta?: string + + constructor(message?: string, status?: number, meta?: string) { + super() + + Error.captureStackTrace(this, this.constructor) + + this.name = this.constructor.name + + this.message = message || 'Something went wrong. Please try again.' + + this.status = status || 500 + + this.meta = meta + } +} From 5a49e0da14085718b55e0b7bcb72b37df2af155e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 15:52:02 +0800 Subject: [PATCH 19/27] chore(deps-dev): bump @typescript-eslint/eslint-plugin (#197) Bumps [@typescript-eslint/eslint-plugin](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/eslint-plugin) from 3.3.0 to 3.9.1. - [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases) - [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/master/packages/eslint-plugin/CHANGELOG.md) - [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v3.9.1/packages/eslint-plugin) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 62 ++++++++++++++++++++++++++++++++++++++++++++--- package.json | 2 +- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 423c3b94db..c11c87f3f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5154,18 +5154,57 @@ } }, "@typescript-eslint/eslint-plugin": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-3.3.0.tgz", - "integrity": "sha512-Ybx/wU75Tazz6nU2d7nN6ll0B98odoiYLXwcuwS5WSttGzK46t0n7TPRQ4ozwcTv82UY6TQoIvI+sJfTzqK9dQ==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-3.9.1.tgz", + "integrity": "sha512-XIr+Mfv7i4paEdBf0JFdIl9/tVxyj+rlilWIfZ97Be0lZ7hPvUbS5iHt9Glc8kRI53dsr0PcAEudbf8rO2wGgg==", "dev": true, "requires": { - "@typescript-eslint/experimental-utils": "3.3.0", + "@typescript-eslint/experimental-utils": "3.9.1", + "debug": "^4.1.1", "functional-red-black-tree": "^1.0.1", "regexpp": "^3.0.0", "semver": "^7.3.2", "tsutils": "^3.17.1" }, "dependencies": { + "@typescript-eslint/experimental-utils": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-3.9.1.tgz", + "integrity": "sha512-lkiZ8iBBaYoyEKhCkkw4SAeatXyBq9Ece5bZXdLe1LWBUwTszGbmbiqmQbwWA8cSYDnjWXp9eDbXpf9Sn0hLAg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/types": "3.9.1", + "@typescript-eslint/typescript-estree": "3.9.1", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + } + }, + "@typescript-eslint/typescript-estree": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-3.9.1.tgz", + "integrity": "sha512-IqM0gfGxOmIKPhiHW/iyAEXwSVqMmR2wJ9uXHNdFpqVvPaQ3dWg302vW127sBpAiqM9SfHhyS40NKLsoMpN2KA==", + "dev": true, + "requires": { + "@typescript-eslint/types": "3.9.1", + "@typescript-eslint/visitor-keys": "3.9.1", + "debug": "^4.1.1", + "glob": "^7.1.6", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, "semver": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", @@ -5198,6 +5237,12 @@ "eslint-visitor-keys": "^1.1.0" } }, + "@typescript-eslint/types": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-3.9.1.tgz", + "integrity": "sha512-15JcTlNQE1BsYy5NBhctnEhEoctjXOjOK+Q+rk8ugC+WXU9rAcS2BYhoh6X4rOaXJEpIYDl+p7ix+A5U0BqPTw==", + "dev": true + }, "@typescript-eslint/typescript-estree": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-3.3.0.tgz", @@ -5230,6 +5275,15 @@ } } }, + "@typescript-eslint/visitor-keys": { + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-3.9.1.tgz", + "integrity": "sha512-zxdtUjeoSh+prCpogswMwVUJfEFmCOjdzK9rpNjNBfm6EyPt99x3RrJoBOGZO23FCt0WPKUCOL5mb/9D5LjdwQ==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, "@uirouter/core": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/@uirouter/core/-/core-6.0.5.tgz", diff --git a/package.json b/package.json index 8b0156e047..38c6547500 100644 --- a/package.json +++ b/package.json @@ -183,7 +183,7 @@ "@types/uid-generator": "^2.0.2", "@types/uuid": "^8.0.0", "@types/validator": "^13.0.0", - "@typescript-eslint/eslint-plugin": "^3.3.0", + "@typescript-eslint/eslint-plugin": "^3.9.1", "@typescript-eslint/parser": "^3.3.0", "axios-mock-adapter": "^1.18.1", "babel-loader": "^8.0.5", From e6a4db64d97163ac9802c737fdf5c3e4311f013c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 16:46:58 +0800 Subject: [PATCH 20/27] fix(deps): bump puppeteer-core from 4.0.0 to 5.2.1 (#188) Bumps [puppeteer-core](https://github.com/puppeteer/puppeteer) from 4.0.0 to 5.2.1. - [Release notes](https://github.com/puppeteer/puppeteer/releases) - [Commits](https://github.com/puppeteer/puppeteer/compare/v4.0.0...v5.2.1) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 76 +++++++++++++++++++++++++++++++++++++++-------- package.json | 2 +- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index c11c87f3f9..076e1dbd06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10274,6 +10274,11 @@ "integrity": "sha512-fYXbFSeilT7bnKWFi4OERSPHdtaEoDGn4aUhV5Nly6/I+Tp6JZ/6Icmd7LVIF5euyodGpxz2e/bfUmDnIdSIDw==", "dev": true }, + "devtools-protocol": { + "version": "0.0.781568", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.781568.tgz", + "integrity": "sha512-9Uqnzy6m6zEStluH9iyJ3iHyaQziFnMnLeC8vK0eN6smiJmIx7+yB64d67C2lH/LZra+5cGscJAJsNXO+MdPMg==" + }, "diagnostics": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", @@ -18106,11 +18111,6 @@ "through2": "^2.0.0" } }, - "mitt": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mitt/-/mitt-2.0.1.tgz", - "integrity": "sha512-FhuJY+tYHLnPcBHQhbUFzscD5512HumCPE4URXZUgPi3IvOJi4Xva5IIgy3xX56GqCmw++MAm5UURG6kDBYTdg==" - }, "mixin-deep": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", @@ -20888,15 +20888,16 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "puppeteer-core": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-4.0.0.tgz", - "integrity": "sha512-Tb5FVp9h9wkd2gXpc/qfBFFI7zjLxxe3pw34U5ntpSnIoUV2m9IKIjAf7ou+5N3fU9VPV3MNJ3HQiDVasN/MPQ==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-5.2.1.tgz", + "integrity": "sha512-gLjEOrzwgcnwRH+sm4hS1TBqe2/DN248nRb2hYB7+lZ9kCuLuACNvuzlXILlPAznU3Ob+mEvVEBDcLuFa0zq3g==", "requires": { "debug": "^4.1.0", + "devtools-protocol": "0.0.781568", "extract-zip": "^2.0.0", "https-proxy-agent": "^4.0.0", "mime": "^2.0.3", - "mitt": "^2.0.1", + "pkg-dir": "^4.2.0", "progress": "^2.0.1", "proxy-from-env": "^1.0.0", "rimraf": "^3.0.2", @@ -20913,10 +20914,61 @@ "ms": "^2.1.1" } }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, "mime": { "version": "2.4.6", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz", "integrity": "sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA==" + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "requires": { + "find-up": "^4.0.0" + } } } }, @@ -23385,9 +23437,9 @@ } }, "tar-stream": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz", - "integrity": "sha512-UaF6FoJ32WqALZGOIAApXx+OdxhekNMChu6axLJR85zMMjXKWFGjbIRe+J6P4UnRGg9rAwWvbTT0oI7hD/Un7Q==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.3.tgz", + "integrity": "sha512-Z9yri56Dih8IaK8gncVPx4Wqt86NDmQTSh49XLZgjWpGZL9GK9HKParS2scqHCC4w6X9Gh2jwaU45V47XTKwVA==", "requires": { "bl": "^4.0.1", "end-of-stream": "^1.4.1", diff --git a/package.json b/package.json index 38c6547500..9f307c61f3 100644 --- a/package.json +++ b/package.json @@ -141,7 +141,7 @@ "nodemailer-direct-transport": "~3.3.2", "opossum": "^5.0.0", "proxyquire": "^2.1.0", - "puppeteer-core": "^4.0.0", + "puppeteer-core": "^5.2.1", "raven-js": "^3.27.0", "rimraf": "^3.0.2", "selectize": "0.12.4", From f0a04d8795ad5b803b33631072c9f89d4e4b5d55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 16:49:40 +0800 Subject: [PATCH 21/27] fix(deps): bump uid-generator from 1.0.0 to 2.0.0 (#187) Bumps [uid-generator](https://github.com/nwoltman/node-uid-generator) from 1.0.0 to 2.0.0. - [Release notes](https://github.com/nwoltman/node-uid-generator/releases) - [Commits](https://github.com/nwoltman/node-uid-generator/compare/v1.0.0...v2.0.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 076e1dbd06..84c70f3822 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25094,9 +25094,9 @@ "integrity": "sha1-dIYISKf9i8SU2YVtL2J3bqmGN8E=" }, "uid-generator": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/uid-generator/-/uid-generator-1.0.0.tgz", - "integrity": "sha512-tx0ni0iyF4CZVNP8bAxcK3PNmtQcJPdflpy6lVW1lPIhTEuPoKR+o88auoXi3mFKLZ7nItCeMRp9GyFFbZ4f3w==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/uid-generator/-/uid-generator-2.0.0.tgz", + "integrity": "sha512-XLRw2UyViQueSbd3dOHkswrg4gA4YuhibKzkFiPkilo6cdKEQqOX3K/Yu6Z2WXVMK+npfMNlSSufVSUifbXoOQ==" }, "uid-safe": { "version": "2.1.5", diff --git a/package.json b/package.json index 9f307c61f3..98d62f0d61 100644 --- a/package.json +++ b/package.json @@ -152,7 +152,7 @@ "tweetnacl": "^1.0.1", "twilio": "^3.33.1", "ui-select": "^0.19.8", - "uid-generator": "^1.0.0", + "uid-generator": "^2.0.0", "uuid": "^8.3.0", "validator": "^11.1.0", "web-streams-polyfill": "^2.1.1", From ae5aa8f2c02a5865c9f2a8441d1d866e00b6f6be Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Mon, 24 Aug 2020 17:02:20 +0800 Subject: [PATCH 22/27] feat: remove beta field validations (#194) * refactor: migrate beta-permissions util to Typescript * fix(BaseFieldSchema): correct validation control flow * feat(BetaPermUtil): remove mobile field from beta * feat: remove checking of User beta permissions validation Since there is no more beta flags --- src/app/models/field/baseField.ts | 154 +++++++++--------------- src/app/models/field/index.ts | 4 +- src/app/models/form.server.model.ts | 4 +- src/app/utils/beta-permissions/index.js | 19 --- 4 files changed, 61 insertions(+), 120 deletions(-) delete mode 100644 src/app/utils/beta-permissions/index.js diff --git a/src/app/models/field/baseField.ts b/src/app/models/field/baseField.ts index 0ba8534cdf..c542b4bdd9 100644 --- a/src/app/models/field/baseField.ts +++ b/src/app/models/field/baseField.ts @@ -1,10 +1,6 @@ -import { Mongoose, Schema } from 'mongoose' +import { Schema } from 'mongoose' import UIDGenerator from 'uid-generator' -import { - isBetaField, - userCanCreateField, -} from '../../../app/utils/beta-permissions' import { AuthType, BasicField, @@ -14,7 +10,6 @@ import { MyInfoAttribute, ResponseMode, } from '../../../types' -import getUserModel from '../user.server.model' const uidgen3 = new UIDGenerator(256, UIDGenerator.BASE62) @@ -42,108 +37,75 @@ export const MyInfoSchema = new Schema( }, ) -const createBaseFieldSchema = (db: Mongoose) => { - const User = getUserModel(db) - - const FieldSchema = new Schema( - { - globalId: String, - title: { - type: String, - trim: true, - default: '', - }, - description: { - type: String, - default: '', - }, - required: { - type: Boolean, - default: true, - }, - disabled: { - type: Boolean, - default: false, - }, - fieldType: { - type: String, - enum: Object.values(BasicField), - required: true, - }, +export const BaseFieldSchema = new Schema( + { + globalId: String, + title: { + type: String, + trim: true, + default: '', }, - { - discriminatorKey: 'fieldType', + description: { + type: String, + default: '', }, - ) - - // Hooks - FieldSchema.pre('validate', function (next) { - // Required due to inner scopes - const thisField = this - // Invalid field types - if (!VALID_FIELD_TYPES.includes(thisField.fieldType)) { - return next(Error('Field type is incorrect or unspecified')) - } - - const { responseMode, admin } = thisField.parent() - - // Prevent MyInfo fields from being set in encrypt mode. - if (responseMode === ResponseMode.Encrypt) { - if (thisField.myInfo?.attr) { - return next( - Error('MyInfo fields are not allowed for storage mode forms'), - ) - } - } - - // Check if user is allowed to add this field. - if (isBetaField(thisField)) { - User.findById(admin, function (err, user) { - if (err) { - return next( - Error( - `Error validating user permissions for ${thisField.fieldType} fields`, - ), - ) - } - if (!userCanCreateField(user, thisField)) { - return next( - Error( - `User is not allowed to access the beta feature - ${this.fieldType} fields`, - ), - ) - } - }) - } + required: { + type: Boolean, + default: true, + }, + disabled: { + type: Boolean, + default: false, + }, + fieldType: { + type: String, + enum: Object.values(BasicField), + required: true, + }, + }, + { + discriminatorKey: 'fieldType', + }, +) - return next() - }) +// Hooks +BaseFieldSchema.pre('validate', function (next) { + // Invalid field types + if (!VALID_FIELD_TYPES.includes(this.fieldType)) { + return next(Error('Field type is incorrect or unspecified')) + } - FieldSchema.pre('save', function (next) { - if (!this.globalId) { - this.globalId = uidgen3.generateSync() + // Prevent MyInfo fields from being set in encrypt mode. + if (this.parent().responseMode === ResponseMode.Encrypt) { + if (this.myInfo?.attr) { + return next(Error('MyInfo fields are not allowed for storage mode forms')) } - return next() - }) + } - // Instance methods - FieldSchema.methods.getQuestion = function (this: IFieldSchema) { - // Return concatenation of all column titles as question string. - if (isTableField(this)) { - const columnTitles = this.columns.map((col) => col.title) - return `${this.title} (${columnTitles.join(', ')})` - } + // No errors. + return next() +}) - // Default question is the field title. - return this.title +BaseFieldSchema.pre('save', function (next) { + if (!this.globalId) { + this.globalId = uidgen3.generateSync() + } + return next() +}) + +// Instance methods +BaseFieldSchema.methods.getQuestion = function (this: IFieldSchema) { + // Return concatenation of all column titles as question string. + if (isTableField(this)) { + const columnTitles = this.columns.map((col) => col.title) + return `${this.title} (${columnTitles.join(', ')})` } - return FieldSchema + // Default question is the field title. + return this.title } // Typeguards const isTableField = (field: IFieldSchema): field is ITableFieldSchema => { return field.fieldType === BasicField.Table } - -export default createBaseFieldSchema diff --git a/src/app/models/field/index.ts b/src/app/models/field/index.ts index 4b27192729..f84c6dd2b2 100644 --- a/src/app/models/field/index.ts +++ b/src/app/models/field/index.ts @@ -1,5 +1,5 @@ import createAttachmentFieldSchema from './attachmentField' -import createBaseFieldSchema from './baseField' +import { BaseFieldSchema } from './baseField' import createCheckboxFieldSchema from './checkboxField' import createDateFieldSchema from './dateField' import createDecimalFieldSchema from './decimalField' @@ -39,5 +39,5 @@ export { createStatementFieldSchema, createTableFieldSchema, createYesNoFieldSchema, - createBaseFieldSchema, + BaseFieldSchema, } diff --git a/src/app/models/form.server.model.ts b/src/app/models/form.server.model.ts index 85dee57b08..62ff489d1a 100644 --- a/src/app/models/form.server.model.ts +++ b/src/app/models/form.server.model.ts @@ -24,8 +24,8 @@ import { MB } from '../constants/filesize' import getAgencyModel from './agency.server.model' import { + BaseFieldSchema, createAttachmentFieldSchema, - createBaseFieldSchema, createCheckboxFieldSchema, createDateFieldSchema, createDecimalFieldSchema, @@ -133,8 +133,6 @@ const compileFormModel = (db: Mongoose): IFormModel => { const Agency = getAgencyModel(db) const User = getUserModel(db) - const BaseFieldSchema = createBaseFieldSchema(db) - // Schema const FormSchema = new Schema( { diff --git a/src/app/utils/beta-permissions/index.js b/src/app/utils/beta-permissions/index.js deleted file mode 100644 index 5c13e5fb23..0000000000 --- a/src/app/utils/beta-permissions/index.js +++ /dev/null @@ -1,19 +0,0 @@ -const BETA_FIELDS = ['mobile'] - -const userCanCreateField = (user, field) => { - switch (field.fieldType) { - // Add cases if there are beta fields - default: - return true - } -} - -const isBetaField = (field) => { - // All MyInfo fields should not be a beta field. - return !field.myInfo && BETA_FIELDS.includes(field.fieldType) -} - -module.exports = { - isBetaField, - userCanCreateField, -} From 6c69af0973081c735fe2e1c97ca1fb6313c06c93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 17:57:43 +0800 Subject: [PATCH 23/27] chore(deps-dev): bump @opengovsg/mockpass from 2.2.0 to 2.4.6 (#198) Bumps [@opengovsg/mockpass](https://github.com/opengovsg/mockpass) from 2.2.0 to 2.4.6. - [Release notes](https://github.com/opengovsg/mockpass/releases) - [Commits](https://github.com/opengovsg/mockpass/compare/v2.2.0...v2.4.6) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 37 ++++++++++++++++++++++++++++--------- package.json | 2 +- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 84c70f3822..829972bf0b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4387,14 +4387,15 @@ } }, "@opengovsg/mockpass": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@opengovsg/mockpass/-/mockpass-2.2.0.tgz", - "integrity": "sha512-Rqb6uO6U44q4YzvBkOZXGtDjvCQq0oLGv9L/RVnDCIyLvkWNwSjLU1HCZoY/ShYGatjrlWL/smCG7RACj191rg==", + "version": "2.4.6", + "resolved": "https://registry.npmjs.org/@opengovsg/mockpass/-/mockpass-2.4.6.tgz", + "integrity": "sha512-+B57K1Os7mvFMm+8A+M0bXdmySo3QEjB/4wlTP4RtYv5+KOyHCGvUJar9AkLiVvYFS9S/lM+xMb8AcGHVIYBLw==", "dev": true, "requires": { "base-64": "^0.1.0", "cookie-parser": "^1.4.3", "dotenv": "^8.1.0", + "expiry-map": "^1.1.0", "express": "^4.16.3", "jsonwebtoken": "^8.4.0", "lodash": "^4.17.11", @@ -4415,12 +4416,6 @@ "integrity": "sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==", "dev": true }, - "uuid": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.1.0.tgz", - "integrity": "sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg==", - "dev": true - }, "xml-encryption": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/xml-encryption/-/xml-encryption-1.2.0.tgz", @@ -11574,6 +11569,15 @@ "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.2.0.tgz", "integrity": "sha512-6SK3MG/Bbhm8MsgyJAylg+ucIOU71/FzyFalcfu5nY19dH8y/z0tBJU0wrNBXD4B27EoQtqPF/9wqH0iYAd04g==" }, + "expiry-map": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/expiry-map/-/expiry-map-1.1.0.tgz", + "integrity": "sha512-EviBh1pKXf8TuPIf88HS27Ti2biOZA1Ci4FeWhXzgY/tCBv05FA+7gO0jXt7AF5JgaiReAld1ag3jd5vzvKupw==", + "dev": true, + "requires": { + "map-age-cleaner": "^0.1.0" + } + }, "express": { "version": "4.17.1", "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", @@ -17683,6 +17687,15 @@ "tmpl": "1.0.x" } }, + "map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "dev": true, + "requires": { + "p-defer": "^1.0.0" + } + }, "map-cache": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", @@ -19557,6 +19570,12 @@ "os-tmpdir": "^1.0.0" } }, + "p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "dev": true + }, "p-each-series": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-2.1.0.tgz", diff --git a/package.json b/package.json index 98d62f0d61..57becacaab 100644 --- a/package.json +++ b/package.json @@ -163,7 +163,7 @@ "devDependencies": { "@babel/core": "^7.4.3", "@babel/preset-env": "^7.11.0", - "@opengovsg/mockpass": "^2.1.1", + "@opengovsg/mockpass": "^2.4.6", "@shelf/jest-mongodb": "^1.2.2", "@types/compression": "^1.7.0", "@types/convict": "^5.2.1", From e6c722193d2fb1fe1c0067d6e09991cae49cd49c Mon Sep 17 00:00:00 2001 From: shuli-ogp <63710093+shuli-ogp@users.noreply.github.com> Date: Mon, 24 Aug 2020 19:56:36 +0800 Subject: [PATCH 24/27] fix: shift userEmail retrieval to GA service (#192) * fix: shift userEmail retrieval to GA service * lint * fix: retrieve userEmail only when tracker is activated * refactor: use helper function to generate userEmail * refactor: rename as getUserEmail() * chore: add try-catch block * refactor: use Auth.getUser() --- .../core/services/gtag.client.service.js | 40 ++++++++++--------- .../view-responses.client.controller.js | 4 -- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/public/modules/core/services/gtag.client.service.js b/src/public/modules/core/services/gtag.client.service.js index 0536d7ebf4..af844fa90e 100644 --- a/src/public/modules/core/services/gtag.client.service.js +++ b/src/public/modules/core/services/gtag.client.service.js @@ -1,10 +1,15 @@ -angular.module('core').factory('GTag', ['$rootScope', '$window', GTag]) +angular.module('core').factory('GTag', ['Auth', '$rootScope', '$window', GTag]) -function GTag($rootScope, $window) { +function GTag(Auth, $rootScope, $window) { // Google Analytics tracking ID provided on signup. const GATrackingID = $window.GATrackingID let gtagService = {} + const getUserEmail = () => { + const user = Auth.getUser() + return user && user.email + } + /** * Internal wrapper function to initialise GA with some globals * @@ -50,7 +55,11 @@ function GTag($rootScope, $window) { * start with a slash (/) character. * @return {Void} */ - const _gtagPageview = ({ pageTitle, pagePath, pageLocation }) => { + const _gtagPageview = ({ + pageTitle, + pagePath, + pageLocation + }) => { if (GATrackingID) { $window.gtag('config', GATrackingID, { page_title: pageTitle, @@ -343,8 +352,7 @@ function GTag($rootScope, $window) { * Logs the start of a storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.formTitle The title of the form - * @param {String} params.userEmail The email of the user downloading + * @param {String} params.formTitle The title of the form * @param {number} expectedNumSubmissions The expected number of submissions to download * @param {number} numWorkers The number of decryption workers * @return {Void} @@ -357,7 +365,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Download start', - event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, + event_label: `${params.formTitle} (${params.formId}), ${getUserEmail()}`, form_id: params.formId, num_workers: numWorkers, num_submissions: expectedNumSubmissions, @@ -368,8 +376,7 @@ function GTag($rootScope, $window) { * Logs a successful storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.formTitle The title of the form - * @param {String} params.userEmail The email of the user downloading + * @param {String} params.formTitle The title of the form * @param {number} downloadedNumSubmissions The number of submissions downloaded * @param {number} numWorkers The number of decryption workers * @param {number} duration The duration taken by the download @@ -384,7 +391,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Download success', - event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, + event_label: `${params.formTitle} (${params.formId}), ${getUserEmail()}`, form_id: params.formId, duration: duration, num_workers: numWorkers, @@ -396,8 +403,7 @@ function GTag($rootScope, $window) { * Logs a failed storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.formTitle The title of the form - * @param {String} params.userEmail The email of the user downloading + * @param {String} params.formTitle The title of the form * @param {number} numWorkers The number of decryption workers * @param {number} expectedNumSubmissions The expected number of submissions * @param {number} duration The duration taken by the download @@ -414,7 +420,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Download failure', - event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, + event_label: `${params.formTitle} (${params.formId}), ${getUserEmail()}`, form_id: params.formId, duration: duration, num_workers: numWorkers, @@ -427,7 +433,6 @@ function GTag($rootScope, $window) { * Logs a failed attempt to even start storage mode responses download. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.userEmail The email of the user downloading * @param {String} params.formTitle The title of the form * @param {string} errorMessage The error message for the failure * @return {Void} @@ -436,7 +441,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Network failure', - event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, + event_label: `${params.formTitle} (${params.formId}), ${getUserEmail()}`, form_id: params.formId, message: errorMessage, }) @@ -446,7 +451,6 @@ function GTag($rootScope, $window) { * Logs partial (or full) decryption failure when downloading responses. * @param {Object} params The response params object * @param {String} params.formId ID of the form - * @param {String} params.userEmail The email of the user downloading * @param {String} params.formTitle The title of the form * @param {number} numWorkers The number of decryption workers * @param {number} expectedNumSubmissions The expected number of submissions @@ -464,7 +468,7 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Partial decrypt error', - event_label: `${params.formTitle} (${params.formId}), ${params.userEmail}`, + event_label: `${params.formTitle} (${params.formId}), ${getUserEmail()}`, form_id: params.formId, duration: duration, num_workers: numWorkers, @@ -480,9 +484,9 @@ function GTag($rootScope, $window) { _gtagEvents('storage', { event_category: 'Storage Mode Form', event_action: 'Secret key mailto clicked', - event_label: formTitle + event_label: formTitle, }) } return gtagService -} +} \ No newline at end of file diff --git a/src/public/modules/forms/admin/controllers/view-responses.client.controller.js b/src/public/modules/forms/admin/controllers/view-responses.client.controller.js index 626afdcbe1..fd2235ffe6 100644 --- a/src/public/modules/forms/admin/controllers/view-responses.client.controller.js +++ b/src/public/modules/forms/admin/controllers/view-responses.client.controller.js @@ -16,7 +16,6 @@ angular '$timeout', 'moment', 'FormSgSdk', - '$window', ViewResponsesController, ]) @@ -30,7 +29,6 @@ function ViewResponsesController( $timeout, moment, FormSgSdk, - $window, ) { const vm = this @@ -63,11 +61,9 @@ function ViewResponsesController( // Trigger for export CSV vm.exportCsv = function () { - const userDetails = JSON.parse($window.localStorage.getItem("user")) let params = { formId: vm.myform._id, formTitle: vm.myform.title, - userEmail: userDetails.email, } if (vm.datePicker.date.startDate && vm.datePicker.date.endDate) { params.startDate = moment(new Date(vm.datePicker.date.startDate)).format( From aa341141d47a806ece786fcccbe0faef0945ccfc Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Tue, 25 Aug 2020 10:34:50 +0800 Subject: [PATCH 25/27] chore: bump version to v4.32.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 829972bf0b..b5da361109 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "FormSG", - "version": "4.31.0", + "version": "4.32.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 57becacaab..645d1b90c2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "FormSG", "description": "Form Manager for Government", - "version": "4.31.0", + "version": "4.32.0", "homepage": "https://form.gov.sg", "authors": [ "FormSG " From 824380ef2a015674b5931cc3f9516036eb80a917 Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Thu, 27 Aug 2020 18:42:18 +0800 Subject: [PATCH 26/27] fix: split mail by semicolon in addition to comma when validating --- src/app/utils/mail.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app/utils/mail.ts b/src/app/utils/mail.ts index b8c8208bc3..3f02ce0b4a 100644 --- a/src/app/utils/mail.ts +++ b/src/app/utils/mail.ts @@ -115,7 +115,9 @@ export const isToFieldValid = (addresses: string | string[]) => { const mails = flattenDeep( flattenDeep([addresses]).map((addrString) => String(addrString) - .split(',') + // Split by both commas and semicolons, as some legacy emails are + // delimited by semicolons. + .split(/,|;/) .map((addr) => addr.trim()), ), ) From 0bf07cfc9b804a2e602a096032065d73805acfea Mon Sep 17 00:00:00 2001 From: Kar Rui Lau Date: Thu, 27 Aug 2020 18:42:56 +0800 Subject: [PATCH 27/27] chore: bump version to v4.32.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b5da361109..9a7d28612a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "FormSG", - "version": "4.32.0", + "version": "4.32.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 645d1b90c2..3c29fa2eb0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "FormSG", "description": "Form Manager for Government", - "version": "4.32.0", + "version": "4.32.1", "homepage": "https://form.gov.sg", "authors": [ "FormSG "