Build simplification

.travis.yml

@@ -5,14 +5,12 @@ cache: ccache
 git:
   depth: 1
 
-dist: trusty
+dist: xenial
 
 sudo: true
 
 install:
- - echo "deb http://de.archive.ubuntu.com/ubuntu xenial main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
- - sudo apt-get update
- - sudo apt-get install --allow-unauthenticated g++ libboost-all-dev cmake libreadline-dev libssl-dev autoconf parallel ccache
+ - sudo apt-get install --allow-unauthenticated libboost-thread-dev libboost-iostreams-dev libboost-date-time-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev libboost-chrono-dev libboost-test-dev libboost-context-dev libboost-regex-dev libboost-coroutine-dev cmake parallel
 
 addons:
   sonarcloud:
@@ -27,16 +25,21 @@ env:
     - CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
 
 script:
- - 'echo $((`date +%s` - 120)) >_start_time'
+ - programs/build_helpers/buildstep -s 3500
  - ccache -s
- - '( [ `ccache -s | grep "files in cache" | cut -c 20-` = 0 ] && touch _empty_cache ) || true'
- - sed -i '/tests/d' libraries/fc/CMakeLists.txt
- - cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON .
- - 'which build-wrapper-linux-x86-64 && build-wrapper-linux-x86-64 --out-dir bw-output make -j 2 cli_wallet witness_node chain_test cli_test || make -j 2 cli_wallet witness_node chain_test cli_test'
+ - programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt"
+ - programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
+ - programs/build_helpers/buildstep make.cli_wallet 1600 "programs/build_helpers/make_with_sonar bw-output -j 2 cli_wallet"
+ - programs/build_helpers/buildstep make.witness_node 300 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node"
+ - programs/build_helpers/buildstep make.serializer 45 "programs/build_helpers/make_with_sonar bw-output -j 2 js_operation_serializer"
+ - programs/build_helpers/buildstep make.get_dev_key 10 "programs/build_helpers/make_with_sonar bw-output -j 2 get_dev_key"
+ - programs/build_helpers/buildstep make.chain_test 900 "programs/build_helpers/make_with_sonar bw-output -j 2 chain_test"
+ - programs/build_helpers/buildstep make.cli_test 200 "programs/build_helpers/make_with_sonar bw-output -j 2 cli_test"
+ - programs/build_helpers/buildstep make.perf_test 120 "programs/build_helpers/make_with_sonar bw-output -j 2 performance_test"
  - set -o pipefail
- - '[ $((`date +%s` - `cat _start_time`)) -gt $((42 * 60)) ] && touch _empty_cache || true'
- - '[ -r _empty_cache ] || tests/chain_test 2>&1 | cat'
- - '[ -r _empty_cache ] || tests/cli_test 2>&1 | cat'
- - 'find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d | while read d; do gcov -o "$d" "${d/CMakeFiles*.dir//}"/*.cpp; done >/dev/null'
- - '[ -r _empty_cache ] || ( which sonar-scanner && sonar-scanner || true )'
- - '[ ! -r _empty_cache ] || ( echo "WARNING! Skipped some tests due to compile time! Please restart with populated cache." && false )'
+ - programs/build_helpers/buildstep run.chain_test 240 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test"
+ - programs/build_helpers/buildstep run.cli_test 30 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test"
+ - programs/build_helpers/buildstep prepare.sonar 20 'find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d | while read d; do gcov -o "$d" "${d/CMakeFiles*.dir//}"/*.cpp; done >/dev/null'
+ - programs/build_helpers/buildstep run.sonar 400 "which sonar-scanner && sonar-scanner || true"
+ - programs/build_helpers/buildstep end 0
+ - ccache -s

CMakeLists.txt

@@ -48,11 +48,9 @@ LIST(APPEND BOOST_COMPONENTS thread
                              system
                              filesystem
                              program_options
-                             serialization
                              chrono
                              unit_test_framework
-                             context
-                             locale)
+                             context)
 SET( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 
 IF( WIN32 )

Dockerfile

@@ -10,16 +10,25 @@ RUN \
       cmake \
       git \
       libbz2-dev \
-      libreadline-dev \
-      libboost-all-dev \
       libcurl4-openssl-dev \
       libssl-dev \
       libncurses-dev \
+      libboost-thread-dev \
+      libboost-iostreams-dev \
+      libboost-date-time-dev \
+      libboost-system-dev \
+      libboost-filesystem-dev \
+      libboost-program-options-dev \
+      libboost-chrono-dev \
+      libboost-test-dev \
+      libboost-context-dev \
+      libboost-regex-dev \
+      libboost-coroutine-dev \
+      libtool \
       doxygen \
       ca-certificates \
+      fish \
     && \
-    apt-get update -y && \
-    apt-get install -y fish && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -38,9 +47,10 @@ RUN \
     git submodule update --init --recursive && \
     cmake \
         -DCMAKE_BUILD_TYPE=Release \
+	-DGRAPHENE_DISABLE_UNITY_BUILD=ON \
         . && \
-    make witness_node cli_wallet && \
-    install -s programs/witness_node/witness_node programs/cli_wallet/cli_wallet /usr/local/bin && \
+    make witness_node cli_wallet get_dev_key && \
+    install -s programs/witness_node/witness_node programs/genesis_util/get_dev_key programs/cli_wallet/cli_wallet /usr/local/bin && \
     #
     # Obtain version
     mkdir /etc/bitshares && \

programs/build_helpers/buildstep

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+usage () {
+    echo Usage:
+    echo "  ${0##*/} [-h | --help] 		Display this help message"
+    echo "  ${0##*/} -s | --start <max-time>	Initialize timing"
+    echo "  ${0##*/} <name> <est> <cmd>"
+    echo "The last form executes build step <name> consisting of shell <cmd>"
+    echo "if <est>imated time is still available, otherwise it fails fast."
+    echo "<max-time> and <est> must be specified in seconds."
+    exit $1
+}
+
+if [ "$#" = 0 -o "$1" = "--help" -o "$1" = "-h" ]; then
+    usage `test "$#" = 1; echo $?`
+fi
+
+NOW="$(date +%s)"
+
+if [ "$1" = "--start" -o "$1" = "-s" ]; then
+    if [ "$#" != 2 ]; then
+	usage 1
+    fi
+    echo "$2 $NOW" >_start_time
+    echo "Starting at $(date --date=@$NOW)"
+    exit 0
+fi
+
+NAME="$1"
+EST="$2"
+CMD="$3"
+
+if [ ! -r _start_time ]; then
+    echo "Need to initialize with '$0 -s <max-time>' first!" 1>&2
+    exit 1
+fi
+
+read max begin prev_name prev_begin <_start_time
+
+if [ "$prev_name" != "" ]; then
+    echo "'$prev_name' took $(($NOW - $prev_begin))s"
+fi
+
+if [ "$CMD" != "" ]; then
+    if [ $(($NOW - $begin + $EST)) -lt $max ]; then
+	echo "Running '$NAME' at $NOW..."
+	echo "sh -c '$CMD'"
+	echo "$max $begin $NAME $NOW" >_start_time
+	exec bash -c "$CMD"
+    fi
+    echo "$(($begin + $max - $NOW))s left - insufficient to run '$NAME', exiting!" 1>&2
+    exit 1
+fi
+
+exit 0

programs/build_helpers/make_with_sonar

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+OUT_DIR="$1"
+shift
+if which build-wrapper-linux-x86-64 >/dev/null; then
+    exec build-wrapper-linux-x86-64 --out-dir "$OUT_DIR" make "$@"
+fi
+exec make "$@"