chore: update travis to have multiple ci stages (#742)

* chore: update travis to have multiple build stages * chore: use correct paths * chore: remove create * fix: missing script tag * chore: correct order of stages * chore: experiment with only building node_modules once * chore: add default language without install step * chore: use node_js 12 in all Travis stages * chore: run lint-ci concurrently * chore: copy all in coverage workspace * chore: run coveralls after_success * chore: use node testEnvironment when running Jest tests Allows for speed up of Jest tests See: https://itnext.io/how-to-make-your-sluggish-jest-v23-tests-go-faster-1d4f3388bcdd * chore: increase number of parallel workers in Jest to 4 * chore: remove unnecessary coverage reporters in Jest
opengovsg · Nov 30, 2020 · 111f1fa · 111f1fa
1 parent 0fb552c
commit 111f1fa
Show file tree

Hide file tree

Showing 5 changed files with 127 additions and 99 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,17 +2,11 @@ os: linux
 dist: xenial
 
 services:
-  - docker
   - xvfb
 
 language: node_js
 node_js: '12'
-cache:
-  - npm
-  - pip
-
-addons:
-  chrome: stable
+install: true # skip installation, perform in build stage.
 
 notifications:
   email:
@@ -23,90 +17,124 @@ notifications:
     on_success: always
     on_failure: always
 
-before_script:
-  - export NODE_OPTIONS=--max-old-space-size=4096
-
-script:
-  - set -e
-  - npm_config_mode=yes npx lockfile-lint --type npm --path package.json --validate-https --allowed-hosts npm
-  - npm run lint-ci
-  - npm run build
-  - travis_retry npm run test-ci
-  - npm run test-e2e-ci
-
-before_deploy:
-  # Workaround to run before_deploy only once
-  - >
-    if ! [ "$TAG" ]; then
-      pip install --user awscli
-      # Put AWS in path
-      export PATH=$PATH:$HOME/.local/bin
-      # Login to AWS ECR, credentials defined in $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY
-      $(aws ecr get-login --no-include-email --region ap-southeast-1)
-      export TAG=travis-$TRAVIS_COMMIT-$TRAVIS_BUILD_NUMBER
-      docker build -f Dockerfile.production -t $REPO:$TAG .
-      docker tag $REPO:$TAG $REPO:$TRAVIS_BRANCH
-      docker push $REPO
-      # Add TAG to Dockerrun
-      sed -i -e "s/@TAG/$TAG/g" Dockerrun.aws.json 
-      zip -r "$TAG.zip" .ebextensions Dockerrun.aws.json
-    fi
-  - export ELASTIC_BEANSTALK_LABEL="$TAG-$(env TZ=Asia/Singapore date "+%Y%m%d%H%M%S")"
+jobs:
+  include:
+    - stage: Build application
+      install: npm ci
+      cache:
+        - npm
+        - pip
+      before_script:
+        - export NODE_OPTIONS=--max-old-space-size=4096
+      script:
+        - set -e
+        - npm_config_mode=yes npx lockfile-lint --type npm --path package.json --validate-https --allowed-hosts npm
+        - npm run lint-ci
+        - npm run build
+      workspaces:
+        create:
+          name: build
+          paths: .
+    - stage: Tests
+      name: Javascript tests
+      workspaces:
+        use: build
+      script:
+        - travis_retry npm run test-backend-jasmine
+        - npm run test-frontend
+    - name: Typescript tests
+      workspaces:
+        use: build
+      script:
+        - travis_retry npm run test-backend-jest
+      after_success:
+        - cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js
+    - name: End-to-end tests
+      workspaces:
+        use: build
+      addons:
+        chrome: stable
+      script:
+        - npm run test-e2e-ci
+    - stage: Deploy
+      services:
+        - docker
+      workspaces:
+        use: build
+      script: skip
+      before_deploy:
+        # Workaround to run before_deploy only once
+        - >
+          if ! [ "$TAG" ]; then
+            pip install --user awscli
+            # Put AWS in path
+            export PATH=$PATH:$HOME/.local/bin
+            # Login to AWS ECR, credentials defined in $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY
+            $(aws ecr get-login --no-include-email --region ap-southeast-1)
+            export TAG=travis-$TRAVIS_COMMIT-$TRAVIS_BUILD_NUMBER
+            docker build -f Dockerfile.production -t $REPO:$TAG .
+            docker tag $REPO:$TAG $REPO:$TRAVIS_BRANCH
+            docker push $REPO
+            # Add TAG to Dockerrun
+            sed -i -e "s/@TAG/$TAG/g" Dockerrun.aws.json 
+            zip -r "$TAG.zip" .ebextensions Dockerrun.aws.json
+          fi
+        - export ELASTIC_BEANSTALK_LABEL="$TAG-$(env TZ=Asia/Singapore date "+%Y%m%d%H%M%S")"
+      deploy:
+        - provider: elasticbeanstalk
+          access_key_id: $AWS_ACCESS_KEY_ID
+          secret_access_key: $AWS_SECRET_ACCESS_KEY
+          region: $AWS_REGION
+          app: $STAGING_APP_NAME
+          env: $UAT_DEPLOY_ENV
+          bucket: $STAGING_BUCKET_NAME
+          zip_file: '$TAG.zip'
+          on:
+            branch: $UAT_BRANCH
 
-deploy:
-  - provider: elasticbeanstalk
-    skip_cleanup: true
-    access_key_id: $AWS_ACCESS_KEY_ID
-    secret_access_key: $AWS_SECRET_ACCESS_KEY
-    region: $AWS_REGION
-    app: $STAGING_APP_NAME
-    env: $UAT_DEPLOY_ENV
-    bucket: $STAGING_BUCKET_NAME
-    zip_file: '$TAG.zip'
-    on:
-      branch: $UAT_BRANCH
+        - provider: elasticbeanstalk
+          access_key_id: $AWS_ACCESS_KEY_ID
+          secret_access_key: $AWS_SECRET_ACCESS_KEY
+          region: $AWS_REGION
+          app: $STAGING_APP_NAME
+          env: $STAGING_DEPLOY_ENV
+          bucket: $STAGING_BUCKET_NAME
+          zip_file: '$TAG.zip'
+          on:
+            branch: $STAGING_BRANCH
 
-  - provider: elasticbeanstalk
-    skip_cleanup: true
-    access_key_id: $AWS_ACCESS_KEY_ID
-    secret_access_key: $AWS_SECRET_ACCESS_KEY
-    region: $AWS_REGION
-    app: $STAGING_APP_NAME
-    env: $STAGING_DEPLOY_ENV
-    bucket: $STAGING_BUCKET_NAME
-    zip_file: '$TAG.zip'
-    on:
-      branch: $STAGING_BRANCH
+        - provider: elasticbeanstalk
+          access_key_id: $AWS_ACCESS_KEY_ID
+          secret_access_key: $AWS_SECRET_ACCESS_KEY
+          region: $AWS_REGION
+          app: $STAGING_APP_NAME
+          env: $STAGING_ALT_DEPLOY_ENV
+          bucket: $STAGING_BUCKET_NAME
+          zip_file: '$TAG.zip'
+          on:
+            branch: $STAGING_ALT_BRANCH
 
-  - provider: elasticbeanstalk
-    skip_cleanup: true
-    access_key_id: $AWS_ACCESS_KEY_ID
-    secret_access_key: $AWS_SECRET_ACCESS_KEY
-    region: $AWS_REGION
-    app: $STAGING_APP_NAME
-    env: $STAGING_ALT_DEPLOY_ENV
-    bucket: $STAGING_BUCKET_NAME
-    zip_file: '$TAG.zip'
-    on:
-      branch: $STAGING_ALT_BRANCH
+        - provider: elasticbeanstalk
+          access_key_id: $AWS_ACCESS_KEY_ID
+          secret_access_key: $AWS_SECRET_ACCESS_KEY
+          region: $AWS_REGION
+          app: $PROD_APP_NAME
+          env: $PROD_DEPLOY_ENV
+          bucket: $PROD_BUCKET_NAME
+          zip_file: '$TAG.zip'
+          on:
+            branch: $PROD_BRANCH
 
-  - provider: elasticbeanstalk
-    skip_cleanup: true
-    access_key_id: $AWS_ACCESS_KEY_ID
-    secret_access_key: $AWS_SECRET_ACCESS_KEY
-    region: $AWS_REGION
-    app: $PROD_APP_NAME
-    env: $PROD_DEPLOY_ENV
-    bucket: $PROD_BUCKET_NAME
-    zip_file: '$TAG.zip'
-    on:
-      branch: $PROD_BRANCH
+      after_deploy:
+        - >
+          if [[ "$SENTRY_PROJECT" &&  "$SENTRY_AUTH_TOKEN" && "$SENTRY_ORG" && "$SENTRY_URL" ]]; then
+            curl -sL https://sentry.io/get-cli/ | bash
+            sentry-cli releases --org $SENTRY_ORG --project $SENTRY_PROJECT new "$TAG"
+            sentry-cli releases files "$TAG" upload-sourcemaps ./ --rewrite --ignore-file .sentryignore
+            sentry-cli releases finalize "$TAG"
+          fi
 
-after_deploy:
-  - >
-    if [[ "$SENTRY_PROJECT" &&  "$SENTRY_AUTH_TOKEN" && "$SENTRY_ORG" && "$SENTRY_URL" ]]; then
-      curl -sL https://sentry.io/get-cli/ | bash
-      sentry-cli releases --org $SENTRY_ORG --project $SENTRY_PROJECT new "$TAG"
-      sentry-cli releases files "$TAG" upload-sourcemaps ./ --rewrite --ignore-file .sentryignore
-      sentry-cli releases finalize "$TAG"
-    fi
+stages:
+  - Build application
+  - Tests
+  - Deploy
diff --git a/jest.config.js b/jest.config.js
@@ -11,9 +11,11 @@ module.exports = {
   },
   collectCoverageFrom: ['./src/**/*.{ts,js}', '!**/__tests__/**'],
   coveragePathIgnorePatterns: ['./node_modules/', './tests'],
+  coverageReporters: ['lcov', 'text'],
   coverageThreshold: {
     global: {
       statements: 25, // Increase this percentage as test coverage improves
     },
   },
+  testEnvironment: 'node',
 }
diff --git a/package.json b/package.json
@@ -16,6 +16,11 @@
     "npm": "~6.4.0"
   },
   "scripts": {
+    "test-backend-jest": "env-cmd -f tests/.test-full-env jest --coverage --maxWorkers=4",
+    "test-backend-jest:watch": "env-cmd -f tests/.test-full-env jest --watch",
+    "test-backend-jasmine": "env-cmd -f tests/.test-full-env --use-shell \"npm run download-binary && jasmine --config=tests/unit/backend/jasmine.json\"",
+    "test-frontend": "jest --config=tests/unit/frontend/jest.config.js",
+    "test-backend": "npm run test-backend-jasmine && npm run test-backend-jest",
     "build": "npm run build-backend && npm run build-frontend",
     "build-backend": "tsc -p tsconfig.build.json",
     "build-frontend": "webpack --config webpack.prod.js",
@@ -24,12 +29,6 @@
     "start": "node dist/backend/server.js",
     "dev": "docker-compose up --build",
     "docker-dev": "npm run build-frontend-dev:watch & ts-node-dev --respawn --transpile-only --inspect=0.0.0.0 --exit-child -- src/server.ts",
-    "test-backend": "npm run test-backend-jasmine && npm run test-backend-jest",
-    "test-backend-jest": "env-cmd -f tests/.test-full-env jest --coverage --maxWorkers=2",
-    "test-backend-jest:watch": "env-cmd -f tests/.test-full-env jest --watch",
-    "test-backend-jasmine": "env-cmd -f tests/.test-full-env --use-shell \"npm run download-binary && jasmine --config=tests/unit/backend/jasmine.json\"",
-    "test-frontend": "jest --config=tests/unit/frontend/jest.config.js",
-    "test-ci": "npm run test-backend && npm run test-frontend && coveralls < coverage/lcov.info",
     "test": "npm run build-backend && npm run test-backend && npm run test-frontend",
     "test-e2e-build": "npm run build-backend && npm run build-frontend-dev",
     "test-run": "concurrently --success last --kill-others \"mockpass\" \"maildev\" \"node dist/backend/server.js\" \"node ./tests/mock-webhook-server.js\"",
@@ -44,7 +43,7 @@
     "lint-style": "stylelint '*/**/*.css' --quiet --fix",
     "lint-html": "htmlhint && prettier --write './src/public/**/*.html' --ignore-path './dist/**' --loglevel silent",
     "lint": "npm run lint-code && npm run lint-style && npm run lint-html",
-    "lint-ci": "eslint src/ --quiet && stylelint '*/**/*.css' --quiet && htmlhint && prettier --c './src/public/**/*.html' --ignore-path './dist/**'",
+    "lint-ci": "concurrently \"eslint src/ --quiet\" \"stylelint '*/**/*.css' --quiet\" \"htmlhint\" \"prettier --c './src/public/**/*.html' --ignore-path './dist/**'\"",
     "version": "auto-changelog -p && git add CHANGELOG.md"
   },
   "husky": {

diff --git a/src/app/modules/analytics/__tests__/analytics.controller.spec.ts b/src/app/modules/analytics/__tests__/analytics.controller.spec.ts
@@ -1,8 +1,8 @@
-import { DatabaseError } from 'dist/backend/app/modules/core/core.errors'
 import { errAsync, okAsync } from 'neverthrow'
 
 import expressHandler from 'tests/unit/backend/helpers/jest-express'
 
+import { DatabaseError } from '../../core/core.errors'
 import * as AnalyticsController from '../analytics.controller'
 import { AnalyticsFactory } from '../analytics.factory'
 import * as AnalyticsService from '../analytics.service'

diff --git a/tests/unit/backend/utils/field-validation/email-validation.spec.ts b/tests/unit/backend/utils/field-validation/email-validation.spec.ts
@@ -1,10 +1,9 @@
+import { ValidateFieldError } from 'src/app/modules/submission/submission.errors'
 import { validateField } from 'src/app/utils/field-validation'
 import EmailValidator from 'src/app/utils/field-validation/validators/EmailValidator.class'
 import { BasicField } from 'src/types/field/fieldTypes'
 import { ISingleAnswerResponse } from 'src/types/response'
 
-import { ValidateFieldError } from '../../../../../dist/backend/app/modules/submission/submission.errors'
-
 describe('Email field validation', () => {
   beforeEach(() => {
     jest