Report test coverage to CodeClimate (#1004)

This PR enables the reporting of test coverage of all the test jobs (`eth` and `tests`) to CodeClimate. This uses S3 to temporarily store results between jobs and later upload them to CC.

Fixes #1000

.travis.yml

@@ -6,9 +6,15 @@ language: python
 python:
   - 3.6.5
 
+stages:
+  - prepare
+  - test
+  - submit
+
 env:
   global:
     - CC_TEST_REPORTER_ID=db72f1ed59628c16eb0c00cbcd629c4c71f68aa1892ef42d18c7c2b8326f460a
+    - JOB_COUNT=2 # Two jobs generate test coverage
   matrix:
     - TEST_TYPE=examples
     - TEST_TYPE=tests
@@ -25,16 +31,27 @@ cache:
   - $HOME/virtualenv/python3.6.5/lib/python3.6/site-packages
   - $HOME/virtualenv/python3.6.5/bin/
 
+jobs:
+  include:
+    - stage: prepare
+      env: TEST_TYPE=env
+      script:
+        - ./cc-test-reporter before-build
+      after_success:
+    - stage: submit
+      env: TEST_TYPE=env
+      script:
+        - aws s3 sync "s3://manticore-testdata/coverage/$TRAVIS_COMMIT" coverage/ 
+        - ./cc-test-reporter sum-coverage --output - --parts $JOB_COUNT coverage/codeclimate.*.json | ./cc-test-reporter upload-coverage --input -
+      after_success:
+
 install:
   - scripts/travis_install.sh $TEST_TYPE
 
-before_script:
-  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  - chmod +x ./cc-test-reporter
-  - ./cc-test-reporter before-build
-
 script:
   - scripts/travis_test.sh $TEST_TYPE
 
-after_script:
-  - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
+after_success:
+  - ./cc-test-reporter format-coverage -t coverage.py -o "coverage/codeclimate.$TEST_TYPE.json"
+  - aws s3 sync coverage/ "s3://manticore-testdata/coverage/$TRAVIS_COMMIT"
+

scripts/travis_install.sh

@@ -5,16 +5,33 @@ function install_solc {
     sudo chmod +x /usr/bin/solc
 }
 
-install_solc
+function install_mcore {
+    pip install -U pip
+    pip uninstall -y Manticore || echo "Manticore not cached"  # uninstall any old, cached Manticore
 
-pip install -U pip
-pip uninstall -y Manticore || echo "Manticore not cached"  # uninstall any old, cached Manticore
 
-# We only need to install keystone if we're just running regular tests
-EXTRAS="dev-noks"
-if [ "$1" = "tests" ]; then
-    EXTRAS="dev"
+    # We only need to install keystone if we're just running regular tests
+    EXTRAS="dev-noks"
+    if [ "$1" = "tests" ]; then
+        EXTRAS="dev"
+    fi
+
+    pip install --no-binary keystone-engine -e .[$EXTRAS]  # ks can have pip install issues
+}
+
+function install_cc_env {
+    curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+    chmod +x ./cc-test-reporter
+
+    pip install awscli
+}
+
+# install CodeClimate env in all conditions
+install_cc_env
+
+if [ "$1" != "env" ]; then
+    install_solc
+    install_mcore $1
 fi
 
-pip install --no-binary keystone-engine -e .[$EXTRAS]  # ks can have pip install issues
 

scripts/travis_test.sh

@@ -124,6 +124,8 @@ if [ "$should_run_eth_tests" = true ] ; then
     if [[ "${DID_OK}" != OK* ]]; then
         echo "Some functionality tests failed :("
         exit 2
+    else
+        coverage xml
     fi
 fi
 
@@ -134,6 +136,8 @@ if [ "$should_run_tests" = true ]; then
     if [[ "${DID_OK}" != OK* ]]; then
         echo "Some non-eth functionality tests failed :("
         exit 2
+    else
+        coverage xml
     fi
 
     echo "Measuring code coverage..."