Use docker.sh on the CI

.circleci/config.yml

@@ -25,7 +25,7 @@ jobs:
                 command: ./tools/ci/docker_update
             - run:
                 name: Pull and tag scion_base image
-                command: ./tools/ci/prepare_image 3e44fd367a50d6d46fc9208850fe0a2149c98fc0b54b5c77d94e9b6994783070
+                command: ./tools/ci/prepare_image 89ee7e0b01200680f8620439bba7df9b8e825a04b57258c5ee6758a274443430
                 when: always
             - run:
                 name: Build scion:latest image
@@ -38,7 +38,10 @@ jobs:
             # Build and run tests:
             - run:
                 name: Create and start container
-                command: ./tools/ci/setup_container
+                command: |
+                    set +e -x
+                    mkdir -p /tmp/artifacts/mount
+                    SCION_MOUNT=/tmp/artifacts/mount ./docker.sh start
                 when: always
             - run:
                 name: Build
@@ -71,9 +74,10 @@ jobs:
                     BUILD=$(date +%s)
                     [ -n "$CIRCLE_BUILD_NUM" ] && BUILD="build${CIRCLE_BUILD_NUM}"
                     ARTIFACTS="circleci.${CIRCLE_PROJECT_USERNAME}.${CIRCLE_PROJECT_REPONAME}.${TARGET}.${BUILD}"
-                    mkdir -p "/tmp/artifacts/$ARTIFACTS" "/tmp/artifacts.out"
-                    ./tools/ci/clean_up /tmp/artifacts/$ARTIFACTS
+                    mkdir -p "/tmp/artifacts.out"
+                    mv /tmp/artifacts/{mount,"$ARTIFACTS"}
                     tar caf "/tmp/artifacts.out/$ARTIFACTS.tar.gz" -C /tmp/artifacts "$ARTIFACTS"
+                    ./docker.sh stop
                 when: always
             - store_artifacts:
                 path: /tmp/artifacts.out

docker.sh

@@ -78,8 +78,8 @@ cmd_clean() {
 }
 
 common_args() {
-    # Limit to 4G of ram, don't allow swapping.
-    local args="-h scion -m 4096M --memory-swap=4096M --shm-size=1024M $DOCKER_ARGS"
+    # Limit to 6G of ram, don't allow swapping.
+    local args="-h scion -m 6GB --memory-swap=6GB --shm-size=1024M $DOCKER_ARGS"
     args+=" -v /var/run/docker.sock:/var/run/docker.sock"
     args+=" -v $SCION_MOUNT/gen:/home/scion/go/src/github.com/scionproto/scion/gen"
     args+=" -v $SCION_MOUNT/logs:/home/scion/go/src/github.com/scionproto/scion/logs"
@@ -101,7 +101,7 @@ cmd_run() {
     SCION_MOUNT=${SCION_MOUNT:-$(mktemp -d /tmp/scion_out.XXXXXX)}
     echo "SCION_MOUNT directory: $SCION_MOUNT"
     local args=$(common_args)
-    args+=" -i -t --rm"
+    args+=" -i -t --rm --entrypoint=/docker-entrypoint.sh"
     setup_volumes
     docker run $args scion "$@"
 }
@@ -112,14 +112,16 @@ cmd_start() {
     echo "SCION_MOUNT directory: $SCION_MOUNT"
     local cntr="scion"
     if docker container inspect "$cntr" &>/dev/null; then
-        echo "Removing stale container"
-        docker rm -f "$cntr"
+        echo "Removing stale $cntr container"
+        ./tools/quiet docker rm -f "$cntr"
     fi
     local args=$(common_args)
     args+=" --name $cntr"
     setup_volumes
-    docker container create $args scion -c "tail -f /dev/null"
-    docker start "$cntr"
+    ./tools/quiet docker container create $args scion -c "tail -f /dev/null"
+    ./tools/quiet docker start "$cntr"
+    # Adjust ownership of mounted dirs
+    docker exec scion /docker-entrypoint.sh
 }
 
 cmd_exec() {
@@ -128,13 +130,13 @@ cmd_exec() {
 
 cmd_stop() {
     local cntr="scion"
-    echo "Stopping $cntr container"; docker stop "$cntr";
-    echo "Removing $cntr container"; docker rm "$cntr";
+    echo "Stopping $cntr container"; ./tools/quiet docker stop "$cntr";
+    echo "Removing $cntr container"; ./tools/quiet docker rm "$cntr";
 }
 
 setup_volumes() {
     set -e
-    for i in gen logs gen-certs gen-cache; do
+    for i in gen logs gen-certs gen-cache htmlcov; do
         mkdir -p "$SCION_MOUNT/$i"
         # Check dir exists, and is owned by the current (effective) user. If
         # it's owned by the wrong user, the docker environment won't be able to
@@ -191,6 +193,12 @@ cmd_help() {
 	    $PROGRAM build
 	    $PROGRAM run
 	        Run the Docker image.
+	    $PROGRAM start
+	        Start a Docker container.
+	    $PROGRAM exec
+	        Execute a command in a running container.
+	    $PROGRAM stop
+	        Stop the Docker container.
 	    $PROGRAM clean
 	        Remove all Docker containers and all generated images.
 	    $PROGRAM help

docker/Dockerfile.base

@@ -100,4 +100,4 @@ RUN sudo chown -R scion: $HOME
 COPY docker/docker-entrypoint.sh /
 
 CMD []
-ENTRYPOINT ["/docker-entrypoint.sh"]
+ENTRYPOINT ["/bin/bash", "-l"]

go/examples/pingpong/pp_integration/main.go

@@ -68,8 +68,8 @@ func runTests(in integration.Integration, pairs []integration.IAPair) error {
 		for i, conn := range pairs {
 			log.Info(fmt.Sprintf("Test %v: %v -> %v (%v/%v)",
 				in.Name(), conn.Src, conn.Dst, i+1, len(pairs)))
-			if err := integration.RunClient(in, conn, 2*time.Second); err != nil {
-				fmt.Fprintf(os.Stderr, "Error during client execution: %s\n", err)
+			if err := integration.RunClient(in, conn, 5*time.Second); err != nil {
+				log.Error("Error during client execution", "err", err)
 				return err
 			}
 		}

tools/ci/build

@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -c "set -eo pipefail; make -s |& tee logs/make.run"
+./docker.sh exec "set -eo pipefail; make -s |& tee logs/make.run"

tools/ci/integration

@@ -3,23 +3,12 @@
 . tools/ci/common.sh
 set -eo pipefail
 
-cntr="${1:-scion_ci}"
-
-docker container exec $cntr bash -c "set -x; ./scion.sh topology ${DOCKER:+-d}"
+# FIXME(worxli): host zk should be properly isolated from dockerized zk
+./docker.sh exec "set -x; ./scion.sh topology ${DOCKER:+zkclean -d}"
 
 if [ -n "$DOCKER" ]; then
-    # Make sure stale configs/state on the host are removed.
-    for i in gen gen-certs; do
-        [ -e "$i" ] && rm -r "$i"
-    done
-    rm -f ./logs/* ./gen-cache/*
-    mkdir -p gen-cache
-    # Copy out new topology
-    docker cp "$cntr:/home/scion/go/src/github.com/scionproto/scion/gen" .
-    docker cp "$cntr:/home/scion/go/src/github.com/scionproto/scion/gen-certs" .
-    # Run tests from host.
-    ./integration/integration_test.sh -d "$cntr" |& tee logs/integration.run
+    ./docker.sh exec "set -eo pipefail; ./integration/integration_test.sh |& tee logs/integration.run"
 else
     # Run tests inside the CI container.
-    docker container exec "$cntr" bash -c "set -eo pipefail; sudo service zookeeper start; ./integration/integration_test.sh |& tee logs/integration.run"
+    ./docker.sh exec "set -eo pipefail; sudo service zookeeper start; ./integration/integration_test.sh |& tee logs/integration.run"
 fi

tools/ci/lint

@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -c "set -eo pipefail; ./scion.sh lint |& tee logs/lint.run"
+./docker.sh exec "set -eo pipefail; ./scion.sh lint |& tee logs/lint.run"

tools/ci/local

@@ -2,10 +2,10 @@
 
 . tools/ci/common.sh
 
-cntr="${1:-scion_ci}"
+# Hardcoded in docker.sh
+cntr="scion"
 
 if [ -n "$DOCKER" ]; then
-    args="-d $cntr"
     # Stop a local scion run because we don't want to interfere
     ./scion.sh stop
     echo "Stop local zookeeper"
@@ -15,27 +15,25 @@ fi
 ./docker.sh build || exit 1
 [ -n "$DOCKER" ] && { make -sC docker/perapp || exit 1; }
 
-docker inspect "$cntr" &>/dev/null && { echo "Removing stale container"; docker rm -f "$cntr"; }
-./tools/ci/setup_container $args || exit 1
+tmpdir=$(mktemp -d /tmp/artifacts.XXXXXXX)
+SCION_MOUNT="$tmpdir" ./docker.sh start || exit 1
 
 result=0
-./tools/ci/build "$cntr"
+./tools/ci/build
 result=$((result+$?))
-./tools/ci/lint "$cntr"
+./tools/ci/lint
 result=$((result+$?))
-./tools/ci/unittest "$cntr"
+./tools/ci/unittest
 result=$((result+$?))
-./tools/ci/sphinx "$cntr"
+./tools/ci/sphinx
 result=$((result+$?))
 
-./tools/ci/integration $args "$cntr"
+./tools/ci/integration
 result=$((result+$?))
 
-tmpdir=$(mktemp -d /tmp/artifacts.XXXXXXX)
-./tools/ci/clean_up "$tmpdir" "$cntr" || exit 1
 echo "Artifacts dir: $tmpdir"
 
-[ $result -eq 0 ] && docker rm -f "$cntr"
+[ $result -eq 0 ] && ./docker.sh stop
 ./scion.sh stop
 
 if [ $result -eq 0 ]; then

tools/ci/sphinx

@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -lc "set -eo pipefail; make -f sphinx-doc/Makefile clean html |& tee logs/sphinx.run"
+./docker.sh exec "set -eo pipefail; make -f sphinx-doc/Makefile clean html |& tee logs/sphinx.run"

tools/ci/unittest

@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -c "set -eo pipefail; ./scion.sh test |& tee logs/unittests.run"
+./docker.sh exec "set -eo pipefail; ./scion.sh test |& tee logs/unittests.run"