refactor: use convict for configuration (#190)

* Move session secret to config * Move port to config * Move otpLifeSpan, bounceLifeSpan and submissionsTopUp to config * Move appConfig to config * Move formsgSdkMode to config * Move awsConfig to config * Move types out * Move cspReportUri and chromiumBin to config * Fix bugs with config loading * Move banners and custom watch grp to config * Move dbHost to config * Remove deprecated env vars * Move ses to convict and enforce that prod ses are defined * Remove repeated code to check bucket url * Reorganise banner env vars under banner key/header * Add missing env var to documentation and re-organise CSP_REPORT_URI' * Reorganise based on order in documentation * Move nodeEnv to convict * Move bucket urls to own convict that gets validated after * Move schemas to separate file * Fix format of nodeEnv * Edit nodemailer configuration to be based on environment * Reorganise and add comments * Remove convict config to Vars for clarity * Add types to convict schemas * Fix typing issues * Add documentation on FORMSG_LOCALSTACK_ENDPT * Reorganise for better readability * Fix bug with db host * Remove test env vars that no longer need to be unset * Use getProperties so that typing can be enforced * Remove warn validation on dev environment * Remove default for session secret * Add defaults for chromium bin * Add validation for db host * Split schemas into option, compulsory and prodOnly * Fix typing * Use new schemas and load dbUri based on environment * Update package * Remove env vars that don't need to be defined during tests * Move cspReportUri to sentry feature, enforce type url and only pass to Helmet if defined * Add missing documentation * Simplify logic * Remove otpGenerator * Define defaults directly in schema * Validate aws endpoint * Move defaults to constant file * Fix imports for constants file * Update default for bounceLifeSpan * Mark the relevant env vars as being sensitive * Add todo to clean up MyInfo env vars * Reference issue in todo * Move aws default endpoint to aws endpoint env var and enforce region in production urls * Fix rebase issues * Remove FORMSG_LOCALSTACK_ENDPT * Remove logger from config * Require config in logger now that config does not use logger * Remove aws region default * Use config.nodeEnv instead of process.env * Update docs/DEPLOYMENT_SETUP.md Co-authored-by: Antariksh Mahajan <[email protected]> * Update docs/DEPLOYMENT_SETUP.md Co-authored-by: Antariksh Mahajan <[email protected]> * Add AWS_ENDPOINT to docker file for dev purposes * Add comments to specify how convict defaults work * Add a session secret default for dev environment * Add default for aws endpoint in docker dev file Co-authored-by: Arshad Ali <[email protected]> Co-authored-by: Antariksh Mahajan <[email protected]>
opengovsg · Sep 3, 2020 · 68a9ea5 · 68a9ea5
1 parent 1bfb064
commit 68a9ea5
Show file tree

Hide file tree

Showing 21 changed files with 754 additions and 473 deletions.
diff --git a/.template-env b/.template-env
@@ -25,7 +25,7 @@ AWS_ACCESS_KEY_ID=
 FORMSG_SDK_MODE=
 
 
-#### Optional variables, some have defaults defined here, as well as in `config/defaults`
+#### Optional variables, some have defaults defined here, as well as in `config/schema`
 
 ## App Config
 # APP_NAME=FormSG

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -28,6 +28,7 @@ services:
       - BOUNCE_LIFE_SPAN=1800000
       - AWS_ACCESS_KEY_ID=fakeKey
       - AWS_SECRET_ACCESS_KEY=fakeSecret
+      - SESSION_SECRET=thisisasecret
       - GA_TRACKING_ID
       - SENTRY_CONFIG_URL
       - TWILIO_ACCOUNT_SID
@@ -63,6 +64,7 @@ services:
       - IS_SP_MAINTENANCE
       - IS_CP_MAINTENANCE
       - AGGREGATE_COLLECTION
+      - AWS_ENDPOINT=http://localhost:4572
 
   mockpass:
     build: https://github.com/opengovsg/mockpass.git

diff --git a/docs/DEPLOYMENT_SETUP.md b/docs/DEPLOYMENT_SETUP.md
@@ -96,7 +96,7 @@ The following env variables are set in Travis:
 
 ### Core Features
 
-#### App and Database
+#### App Config
 
 | Variable                 | Description                                                                                                                                                                                                           |
 | :----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -105,19 +105,27 @@ The following env variables are set in Travis:
 | `APP_URL`                | Defaults to `'https://form.gov.sg'`.                                                                                                                                                                                  |
 | `APP_KEYWORDS`           | Defaults to `'forms, formbuilder, nodejs'`.                                                                                                                                                                           |
 | `APP_IMAGES`             | Defaults to `'/public/modules/core/img/og/img_metatag.png,/public/modules/core/img/og/logo-vertical-color.png'`.                                                                                                      |
-| `APP_TWITTER_IMAGE`      | ath to Twitter image. Defaults to `'/public/modules/core/img/og/logo-vertical-color.png'`.                                                                                                                            |
+| `APP_TWITTER_IMAGE`      | Path to Twitter image. Defaults to `'/public/modules/core/img/og/logo-vertical-color.png'`.                                                                                                                            |
+
+#### App and Database
+
+| Variable                 | Description                                                                                                                                                                                                           |
+| :----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `DB_HOST`                | A MongoDB URI.                                                                                                                                                                                                        |
 | `OTP_LIFE_SPAN`          | Time in milliseconds that admin login OTP is valid for. Defaults to 900000ms or 15 minutes.                                                                                                                           |
-| `BOUNCE_LIFE_SPAN`       | Time in milliseconds that bounces are tracked for each form. Defaults to 1800000ms or 30 minutes. Only relevant if you have set up AWS to send bounce and delivery notifications to the /emailnotifications endpoint. |
 | `PORT`                   | Server port. Defaults to `5000`.                                                                                                                                                                                      |
 | `NODE_ENV`               | [Express environment mode](https://expressjs.com/en/advanced/best-practice-performance.html#set-node_env-to-production). Defaults to `'development'`. This should always be set to a production environment           |
 | `SESSION_SECRET`         | Secret for `express-session`. Defaults to `'sandcrawler-138577'`. This should always be set in a production environment.                                                                                              |
+| `SUBMISSIONS_TOP_UP`     | Use this to inflate the number of submissions displayed on the landing page. Defaults to `0`.                                                                                                                         |
+
+#### Banners
+
+| Variable                 | Description                                                                                                                                                                                                           |
+| :----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `SITE_BANNER_CONTENT`    | If set, displays a banner message on both private routes that `ADMIN_BANNER_CONTENT` covers **and** public form routes that `IS_GENERAL_MAINTENANCE` covers. Overrides all other banner environment variables         |
 | `ADMIN_BANNER_CONTENT`   | If set, displays a banner message on private admin routes such as the form list page as well as form builder pages.                                                                                                   |
+| `IS_LOGIN_BANNER`   | If set, displays a banner message on the login page                              |
 | `IS_GENERAL_MAINTENANCE` | If set, displays a banner message on all forms. Overrides `IS_SP_MAINTENANCE` and `IS_CP_MAINTENANCE`.                                                                                                                |
-| `IS_SP_MAINTENANCE`      | If set, displays a banner message on SingPass forms. Overrides `IS_CP_MAINTENANCE`.                                                                                                                                   |
-| `IS_CP_MAINTENANCE`      | If set, displays a banner message on SingPass forms.                                                                                                                                                                  |
-| `SUBMISSIONS_TOP_UP`     | Use this to inflate the number of submissions displayed on the landing page. Defaults to `0`.                                                                                                                         |
 
 #### AWS services
 
@@ -126,6 +134,7 @@ The following env variables are set in Travis:
 | `AWS_REGION`                  | AWS region.                                                                                                                         |
 | `AWS_ACCESS_KEY_ID`           | AWS IAM access key ID used to access S3.                                                                                            |
 | `AWS_SECRET_ACCESS_KEY`       | AWS IAM access secret used to access S3.                                                                                            |
+| `AWS_ENDPOINT`                 | AWS S3 bucket endpoint.                                                                                                                         |
 | `IMAGE_S3_BUCKET`             | Name of S3 bucket for image field uploads.                                                                                          |
 | `LOGO_S3_BUCKET`              | Name of S3 bucket for form logo uploads.                                                                                            |
 | `LOGO_S3_BUCKET`              | Name of S3 bucket for form logo uploads.                                                                                            |
@@ -152,6 +161,7 @@ The following env variables are set in Travis:
 | `MAIL_LOGGER`         | If set to true then logs to console. If value is not set or is false then nothing is logged.                                                                                                   |
 | `MAIL_DEBUG`          | If set to `true`, then logs SMTP traffic, otherwise logs only transaction events.                                                                                                              |
 | `CHROMIUM_BIN`        | Filepath to chromium binary. Required for email autoreply PDF generation with Puppeteer.                                                                                                       |
+| `BOUNCE_LIFE_SPAN`       | Time in milliseconds that bounces are tracked for each form. Defaults to 10800000ms or 3 hours. Only relevant if you have set up AWS to send bounce and delivery notifications to the /emailnotifications endpoint. |
 
 ### Additional Features
 
@@ -180,7 +190,8 @@ If this feature is enabled, client-side error events will be piped to [sentry.io
 
 | Variable            | Description                                                                                           |
 | :------------------ | ----------------------------------------------------------------------------------------------------- |
-| `SENTRY_CONFIG_URL` | Sentry.io URL for configuring the Sentry SDK.                                                         |
+| `CSP_REPORT_URI`    | Reporting URL for Content Security Policy violdations. Can be configured to use a Sentry.io endpoint. |
+| `SENTRY_CONFIG_URL` | Sentry.io URL for configuring the Raven SDK.                                                          |
 | `CSP_REPORT_URI`    | Reporting URL for Content Security Policy violdations. Can be configured to use a Sentry.io endpoint. |
 
 #### Examples page Using Pre-Computed Results
@@ -235,6 +246,8 @@ Note that MyInfo is currently not supported for storage mode forms and enabling
 | `MYINFO_CLIENT_CONFIG`          | Configures [MyInfoGovClient](https://github.com/opengovsg/myinfo-gov-client). Set this to either`stg` or `prod` to fetch MyInfo data from the corresponding endpoints. |
 | `MYINFO_FORMSG_KEY_PATH`        | Filepath to MyInfo private key, which is used to decrypt returned responses.                                                                                           |
 | `MYINFO_APP_KEY`                | (deprecated) Directly specify contents of the MyInfo FormSG private key. Only works if `NODE_ENV` is set to `development`.                                             |
+| `IS_SP_MAINTENANCE`      | If set, displays a banner message on SingPass forms. Overrides `IS_CP_MAINTENANCE`.                                                                                                                                   |
+| `IS_CP_MAINTENANCE`      | If set, displays a banner message on CorpPass forms.                                                                                                                                                                  |
 
 #### Verified Emails/SMSes
 
@@ -258,9 +271,9 @@ If this feature is enabled, storage mode forms will also support authentication
 
 ### Tests
 
-| Variable                   | Description                                                                                                                                     |
-| :------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
-| `MONGO_BINARY_VERSION`     | Version of the Mongo binary used. Defaults to `'latest'` according to [MongoMemoryServer](https://github.com/nodkz/mongodb-memory-server) docs. |
-| `PWD`                      | Path of working directory.                                                                                                                      |
-| `MOCK_WEBHOOK_CONFIG_FILE` | Path of configuration file for mock webhook server                                                                                              |
-| `MOCK_WEBHOOK_PORT`        | Port of mock webhook server                                                                                                                     |
+| Variable               | Description                                                                                                                                     |
+| :--------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| `MONGO_BINARY_VERSION` | Version of the Mongo binary used. Defaults to `'latest'` according to [MongoMemoryServer](https://github.com/nodkz/mongodb-memory-server) docs. |
+| `PWD`                  | Path of working directory.                                                                                                                      |
+| `MOCK_WEBHOOK_CONFIG_FILE`                  | Path of configuration file for mock webhook server                                                                                                                      |
+| `MOCK_WEBHOOK_PORT`                  | Port of mock webhook server                                                                                                                      |
diff --git a/init-localstack.sh b/init-localstack.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 set -x
-until $(curl --output /dev/null --silent --head --fail http://localhost:4572); do
+until $(curl --output /dev/null --silent --head --fail $AWS_ENDPOINT); do
   printf 'Waiting for Localstack to be ready...'
   sleep 5
 done
-awslocal --endpoint-url=http://localhost:4572 s3 mb s3://$IMAGE_S3_BUCKET
-awslocal --endpoint-url=http://localhost:4572 s3 mb s3://$LOGO_S3_BUCKET
-awslocal --endpoint-url=http://localhost:4572 s3 mb s3://$ATTACHMENT_S3_BUCKET
+awslocal --endpoint-url=$AWS_ENDPOINT s3 mb s3://$IMAGE_S3_BUCKET
+awslocal --endpoint-url=$AWS_ENDPOINT s3 mb s3://$LOGO_S3_BUCKET
+awslocal --endpoint-url=$AWS_ENDPOINT s3 mb s3://$ATTACHMENT_S3_BUCKET
 set +x
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -87,6 +87,7 @@
     "angular-ui-router": "~1.0.22",
     "async": "~1.5.2",
     "await-to-js": "^2.1.1",
+    "aws-info": "^1.1.0",
     "aws-sdk": "^2.734.0",
     "axios": "^0.20.0",
     "bcrypt": "^5.0.0",
@@ -131,6 +132,7 @@
     "lodash": "^4.17.20",
     "mobile-detect": "^1.4.2",
     "moment-timezone": "0.5.31",
+    "mongodb-uri": "^0.9.7",
     "mongoose": "^5.9.10",
     "multiparty": ">=4.1.3",
     "ng-infinite-scroll": "^1.3.0",

diff --git a/src/app/controllers/authentication.server.controller.js b/src/app/controllers/authentication.server.controller.js
@@ -16,7 +16,7 @@ const validator = require('validator')
 const { StatusCodes } = require('http-status-codes')
 
 const config = require('../../config/config')
-const defaults = require('../../config/defaults').default
+const { LINKS } = require('../../shared/constants')
 const PERMISSIONS = require('../utils/permission-levels').default
 const { getRequestIp } = require('../utils/request')
 const logger = require('../../config/logger').createLoggerWithLabel(module)
@@ -55,7 +55,7 @@ exports.validateDomain = function (req, res, next) {
       return res
         .status(StatusCodes.INTERNAL_SERVER_ERROR)
         .send(
-          `Unable to validate email domain. If this issue persists, please submit a Support Form (${defaults.links.supportFormLink}).`,
+          `Unable to validate email domain. If this issue persists, please submit a Support Form (${LINKS.supportFormLink}).`,
         )
     }
     // Agency not found
@@ -286,7 +286,7 @@ exports.verifyOtp = function (req, res, next) {
         return res
           .status(StatusCodes.INTERNAL_SERVER_ERROR)
           .send(
-            `Unable to login at this time. Please submit a Support Form (${defaults.links.supportFormLink}).`,
+            `Unable to login at this time. Please submit a Support Form (${LINKS.supportFormLink}).`,
           )
       }
       if (!updatedRecord) {
@@ -412,7 +412,7 @@ exports.signIn = function (req, res) {
         return res
           .status(StatusCodes.INTERNAL_SERVER_ERROR)
           .send(
-            `User signin failed. Please try again later and if the problem persists, submit our Support Form (${defaults.links.supportFormLink}).`,
+            `User signin failed. Please try again later and if the problem persists, submit our Support Form (${LINKS.supportFormLink}).`,
           )
       }
       let userObj = {

diff --git a/src/app/factories/spcp-myinfo.factory.js b/src/app/factories/spcp-myinfo.factory.js
@@ -56,6 +56,8 @@ const spcpFactory = ({ isEnabled, props }) => {
       singpassEserviceId: props.spEsrvcId,
     }
     let myInfoGovClient
+    // TODO: These env vars should move to spcp-myinfo.config and be validated
+    // as part of convict (Issue #255)
     if (config.nodeEnv === 'production') {
       let myInfoPrefix =
         process.env.MYINFO_CLIENT_CONFIG === 'stg' ? 'STG2-' : 'PROD2-'