Feature/jest parallel integration tests

.circleci/config.yml

@@ -25,6 +25,7 @@ defaults_Dependencies: &defaults_Dependencies |
   npm config set unsafe-perm true
   npm install -g node-gyp
 
+
 defaults_awsCliDependencies: &defaults_awsCliDependencies |
   apk upgrade --no-cache
   apk --no-cache add \
@@ -54,7 +55,8 @@ executors:
 
   default-machine:
     machine:
-      image: ubuntu-1604:201903-01
+      docker_layer_caching: true
+      image: ubuntu-2004:202008-01
 
 ##
 # Jobs
@@ -85,84 +87,66 @@ jobs:
           key: dependency-cache-{{ checksum "package.json" }}
           paths:
             - node_modules
-
-  test-unit:
-    executor: default-docker
-    steps:
-      - checkout
-      - run:
-          name: Install general dependencies
-          command: *defaults_Dependencies
-      - restore_cache:
-          keys:
-            - dependency-cache-{{ checksum "package.json" }}
-      - run:
-          name: Create dir for test results
-          command: mkdir -p ./test/results
-      - run:
-          name: Install tape and tap-xunit
-          command: npm install -g tape tap-xunit
-      - run:
-          name: Install tapes
-          command: npm install -g tapes
-      - run:
-          name: Execute unit tests
-          command: npm -s run test:xunit > ./test/results/tape.xml
-      - store_artifacts:
-          path: ./test/results
-          prefix: test
-      - store_test_results:
-          path: ./test/results
-
-  test-coverage:
-    executor: default-docker
-    steps:
-      - checkout
-      - run:
-          name: Install general dependencies
-          command: *defaults_Dependencies
-      - run:
-          name: Install AWS CLI dependencies
-          command: *defaults_awsCliDependencies
-      - restore_cache:
-          keys:
-            - dependency-cache-{{ checksum "package.json" }}
-      - run:
-          name: Execute code coverage check
-          command: npm -s run test:coverage
-      - store_artifacts:
-          path: coverage
-          prefix: test
-      - store_test_results:
-          path: coverage
-      - run:
-          name: Copy code coverage to SonarQube
-          command: |
-            if [ "${CIRCLE_BRANCH}" == "master" ];
-            then
-                echo "Sending lcov.info to SonarQube..."
-                aws s3 cp coverage/lcov.info $AWS_S3_DIR_SONARQUBE/central-settlement/lcov.info
-            else
-                echo "Not a release (env CIRCLE_BRANCH != 'master'), skipping sending lcov.info to SonarQube."
-            fi
-
-  test-integration:
-    executor: default-machine
-    steps:
-      - checkout
-      - restore_cache:
-          key: dependency-cache-{{ checksum "package.json" }}
-      - run:
-          name: Create dir for test results
-          command: mkdir -p ./test/results
-      - run:
-          name: Execute integration tests
-          command: npm -s run test:integration
-      - store_artifacts:
-          path: ./test/results
-          prefix: test
-      - store_test_results:
-          path: ./test/results
+  tests:
+      executor: default-machine
+      steps:
+        - checkout
+        - restore_cache:
+            keys:
+              - v1-docker-{{ checksum "package.json" }}
+        - run:
+            name: Load Docker layers cache
+            command: |
+              set +o pipefail
+              docker load -i ~/docker-layers.tar | true
+        - run:
+            name: Prepare docker environment
+            command: npm -s run test:int:new:ci:prepare
+        - run:
+            name: Save Docker layers cache
+            command: |
+              DOCKER_IMAGES=$(docker images --format "{{.Repository}}" --filter=reference="${CIRCLE_PROJECT_REPONAME}_*")
+              DOCKER_LAYERS=$(for image in $DOCKER_IMAGES; do docker history $image -q | grep -v missing; done)
+              docker save -o ~/docker-layers.tar $DOCKER_LAYERS
+        - save_cache:
+            key: v1-docker-{{ checksum "package.json" }}
+            paths:
+              - ~/docker-layers.tar
+        - run:
+            name: Execute unit tests
+            command: docker exec -it cs_central-settlement sh -c "mkdir -p ./test/results && npm -s run test:xunit > ./test/results/tape.xml"
+        - run:
+            name: Execute unit tests with coverage
+            command: docker exec -it cs_central-settlement sh -c "npm run test:coverage"
+        - run:
+            name: Copy unit test results
+            command: mkdir -p ./test/results && docker cp cs_central-settlement:/opt/central-settlement/test/results/ .
+        - run:
+            name: Execute narrow integration tests
+            command: docker exec -it cs_central-settlement sh -c "npm run test:int:new:series:run"
+        - run:
+            name: Execute combined tests coverage check
+            command: docker exec -it cs_central-settlement sh -c  "npm run test:coverage:merge && npm run test:coverage:merge:check"
+        - run:
+            name: Copy combined coverage results
+            command: docker cp cs_central-settlement:/opt/central-settlement/coverage_merged/ .
+        - store_artifacts:
+            path: ./test/results
+            prefix: test
+        - store_test_results:
+            path: ./test/results
+        - store_artifacts:
+            path: ./coverage_merged
+        - run:
+            name: Copy code coverage to SonarQube
+            command: |
+              if [ "${CIRCLE_BRANCH}" == "master" ];
+              then
+                  echo "Sending lcov.info to SonarQube..."
+                  aws s3 cp ./coverage_merged/lcov.info $AWS_S3_DIR_SONARQUBE/central-settlement/lcov.info
+              else
+                  echo "Not a release (env CIRCLE_BRANCH != 'master'), skipping sending lcov.info to SonarQube."
+              fi
 
   vulnerability-check:
     executor: default-docker
@@ -361,32 +345,8 @@ workflows:
               ignore:
                 - /feature*/
                 - /bugfix*/
-      - test-unit:
+      - tests:
           context: org-global
-          requires:
-            - setup
-          filters:
-            tags:
-              only: /.*/
-            branches:
-              ignore:
-                - /feature*/
-                - /bugfix*/
-      - test-coverage:
-          context: org-global
-          requires:
-            - setup
-          filters:
-            tags:
-              only: /.*/
-            branches:
-              ignore:
-                - /feature*/
-                - /bugfix*/
-      - test-integration:
-          context: org-global
-          requires:
-            - setup
           filters:
             tags:
               only: /.*/
@@ -420,9 +380,7 @@ workflows:
           context: org-global
           requires:
             - setup
-            - test-unit
-            - test-coverage
-            - test-integration
+            - tests
             - vulnerability-check
             - audit-licenses
           filters:
@@ -471,4 +429,4 @@ workflows:
               only: /v[0-9]+(\.[0-9]+)*(\-snapshot)?/
             branches:
               ignore:
-                - /.*/
+                - /.*/

.gitignore

@@ -17,7 +17,7 @@ lib-cov
 
 # Coverage directory used by tools like istanbul
 coverage
-
+coverage_*
 #Docs folder created for documentation
 docs
 
@@ -58,3 +58,6 @@ jspm_packages
 .history
 /.gitlab-ci.env
 junit.xml
+migrations
+seeds
+tmp

.gitlab-ci.yml

@@ -13,8 +13,6 @@ stages:
   - apk --no-cache add curl
   - apk --no-cache add openssh-client
   - apk add --no-cache -t build-dependencies make gcc g++ python libtool autoconf automake
-  - npm config set unsafe-perm true
-  - npm install -g node-gyp
 
 # Build the image and push it to gitlab package registry
 Build:
@@ -30,25 +28,43 @@ Build:
     - docker push "$CI_REGISTRY_IMAGE:$IMAGE_TAG"
 
 # Execute unit tests
-Unit Test:
+Run Test:
   stage: test
-  image: node:12.16.0-alpine
+  image: docker:latest
+  services:
+    - docker:dind
+  before_script:
+    - docker info
+    - apk update
+    - apk upgrade
+    - apk add curl jq python3 python3-dev build-base libffi-dev libressl-dev gettext
+    - curl -O https://bootstrap.pypa.io/get-pip.py
+    - python3 get-pip.py
+    - pip install docker-compose
+    - apk --no-cache add ca-certificates
+    - apk --no-cache add curl
+    - apk --no-cache add openssh-client
   script:
-    - *defaults_Dependencies
-    # - cd $(npm root -g)/npm
-    # - npm install --no-save github:interledgerjs/five-bells-ledger-api-tests
-    - npm install
+    - apk update && apk --no-cache add git
+    - mkdir -p ~/.ssh && ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+    - sh ./test/int/scripts/pre_start_ci.sh
+    - docker exec -i cs_central-settlement sh -c "mkdir -p ./test/results && npm -s run test:xunit > ./test/results/tape.xml"
+    - docker exec -i cs_central-settlement sh -c "npm run test:coverage"
+    - mkdir -p ./test/results && docker cp cs_central-settlement:/opt/central-settlement/test/results/ .
+    - docker exec -i cs_central-settlement sh -c "npm run test:int:new:series:run"
+    - docker exec -i cs_central-settlement sh -c  "npm run test:coverage:merge && npm run test:coverage:merge:check"
+    - docker cp cs_central-settlement:/opt/central-settlement/coverage_combined/ .
     # - apk del build-dependencies
     # - mkdir -p ./test/results
     #- npm install -g tape tap-xunit
     #- npm install -g tapes
-    - npm -s run test:coverage-check
-  #artifacts:
-  #  paths:
-  #    - ./test/results/*.*
-  #  reports:
-  #    junit:
-  #      - ./test/results/tape.xml
+    # - npm -s run test:coverage-check
+  artifacts:
+   paths:
+     - ./test/results/*.*
+   reports:
+     junit:
+       - ./test/results/tape.xml
 
 
 # Deployment requires the mojaloop cluster to be configured in gitlab
@@ -69,7 +85,7 @@ Deploy to Dev:
     # This allows the cluster to pull images from the private repo
     # without needing to modify the upstream OSS helm chart to allow specifying image pull credentials
     # NOTE: This step requires a deploy token to be created in this project called gitlab-deploy-token
-    # See https://docs.gitlab.com/ee/user/project/deploy_tokens/index.html#gitlab-deploy-token for details  
+    # See https://docs.gitlab.com/ee/user/project/deploy_tokens/index.html#gitlab-deploy-token for details
     # Install coreutils so that we can use the GNU version of base64 and disable line wrapping (-w)
     - apk update && apk add coreutils
     # Set the required variables and base64 them

.nycrc.combined.yml

@@ -0,0 +1,23 @@
+# temp-directory: "./.nyc_output"
+check-coverage: true
+per-file: true
+lines: 90
+statements: 90
+functions: 90
+branches: 90
+all: true
+include: [
+  "src/**/*.js"
+]
+instrument: true
+reporter: [
+  "lcov",
+  "text-summary",
+  "html"
+]
+report-dir: "./coverage_combined"
+exclude: [
+  "**/node_modules/**",
+  '**/migrations/**',
+  '**/docs/**',
+]

.nycrc.integration.yml

@@ -1,5 +1,5 @@
 temp-directory: "./.nyc_output"
-check-coverage: true
+check-coverage: false
 per-file: true
 lines: 90
 statements: 90

.nycrc.yml

@@ -1,5 +1,5 @@
 temp-directory: "./.nyc_output"
-check-coverage: true
+check-coverage: false
 per-file: true
 lines: 90
 statements: 90