From e56aefa2e6ae15c6bcf7ecba37697b427b688969 Mon Sep 17 00:00:00 2001
From: Lukas Bischofberger <bischofberger@anapaya.net>
Date: Fri, 19 Oct 2018 12:51:26 +0200
Subject: [PATCH 1/6] Use docker.sh on the CI

---
 .circleci/config.yml     | 13 ++++++++-----
 docker.sh                |  5 +++--
 docker/Dockerfile.base   |  2 +-
 tools/ci/build           |  2 +-
 tools/ci/clean_up        |  7 -------
 tools/ci/integration     | 17 +++--------------
 tools/ci/lint            |  2 +-
 tools/ci/local           | 24 ++++++++++++------------
 tools/ci/setup_container | 12 ------------
 tools/ci/sphinx          |  2 +-
 tools/ci/unittest        |  2 +-
 11 files changed, 31 insertions(+), 57 deletions(-)
 delete mode 100755 tools/ci/clean_up
 delete mode 100755 tools/ci/setup_container

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6b8d42ec79..e092838f09 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,7 +25,7 @@ jobs:
                 command: ./tools/ci/docker_update
             - run:
                 name: Pull and tag scion_base image
-                command: ./tools/ci/prepare_image 3e44fd367a50d6d46fc9208850fe0a2149c98fc0b54b5c77d94e9b6994783070
+                command: ./tools/ci/prepare_image 89ee7e0b01200680f8620439bba7df9b8e825a04b57258c5ee6758a274443430
                 when: always
             - run:
                 name: Build scion:latest image
@@ -38,7 +38,10 @@ jobs:
             # Build and run tests:
             - run:
                 name: Create and start container
-                command: ./tools/ci/setup_container
+                command: |
+                    set +e -x
+                    mkdir -p "/tmp/artifacts"
+                    SCION_MOUNT="/tmp/artifacts" ./docker.sh start
                 when: always
             - run:
                 name: Build
@@ -71,9 +74,9 @@ jobs:
                     BUILD=$(date +%s)
                     [ -n "$CIRCLE_BUILD_NUM" ] && BUILD="build${CIRCLE_BUILD_NUM}"
                     ARTIFACTS="circleci.${CIRCLE_PROJECT_USERNAME}.${CIRCLE_PROJECT_REPONAME}.${TARGET}.${BUILD}"
-                    mkdir -p "/tmp/artifacts/$ARTIFACTS" "/tmp/artifacts.out"
-                    ./tools/ci/clean_up /tmp/artifacts/$ARTIFACTS
-                    tar caf "/tmp/artifacts.out/$ARTIFACTS.tar.gz" -C /tmp/artifacts "$ARTIFACTS"
+                    mkdir -p "/tmp/artifacts.out"
+                    tar caf "/tmp/artifacts.out/$ARTIFACTS.tar.gz" -C /tmp artifacts
+                    ./docker.sh stop
                 when: always
             - store_artifacts:
                 path: /tmp/artifacts.out
diff --git a/docker.sh b/docker.sh
index 2f20e497b8..6aa8650c58 100755
--- a/docker.sh
+++ b/docker.sh
@@ -101,7 +101,7 @@ cmd_run() {
     SCION_MOUNT=${SCION_MOUNT:-$(mktemp -d /tmp/scion_out.XXXXXX)}
     echo "SCION_MOUNT directory: $SCION_MOUNT"
     local args=$(common_args)
-    args+=" -i -t --rm"
+    args+=" -i -t --rm --entrypoint=/docker-entrypoint.sh"
     setup_volumes
     docker run $args scion "$@"
 }
@@ -120,6 +120,7 @@ cmd_start() {
     setup_volumes
     docker container create $args scion -c "tail -f /dev/null"
     docker start "$cntr"
+    docker exec scion /docker-entrypoint.sh
 }
 
 cmd_exec() {
@@ -134,7 +135,7 @@ cmd_stop() {
 
 setup_volumes() {
     set -e
-    for i in gen logs gen-certs gen-cache; do
+    for i in gen logs gen-certs gen-cache htmlcov; do
         mkdir -p "$SCION_MOUNT/$i"
         # Check dir exists, and is owned by the current (effective) user. If
         # it's owned by the wrong user, the docker environment won't be able to
diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base
index dcfd01bc9a..41b45c0b58 100644
--- a/docker/Dockerfile.base
+++ b/docker/Dockerfile.base
@@ -100,4 +100,4 @@ RUN sudo chown -R scion: $HOME
 COPY docker/docker-entrypoint.sh /
 
 CMD []
-ENTRYPOINT ["/docker-entrypoint.sh"]
+ENTRYPOINT ["/bin/bash", "-l"]
diff --git a/tools/ci/build b/tools/ci/build
index c7195efa2d..e19538323d 100755
--- a/tools/ci/build
+++ b/tools/ci/build
@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -c "set -eo pipefail; make -s |& tee logs/make.run"
+./docker.sh exec "set -eo pipefail; make -s |& tee logs/make.run"
diff --git a/tools/ci/clean_up b/tools/ci/clean_up
deleted file mode 100755
index a694bcea9e..0000000000
--- a/tools/ci/clean_up
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-docker cp ${2:-scion_ci}:/home/scion/go/src/github.com/scionproto/scion/logs "${1:?"Directory needs to be specified!"}"
-docker cp ${2:-scion_ci}:/home/scion/go/src/github.com/scionproto/scion/gen "$1"
-cp -R logs "$1"
diff --git a/tools/ci/integration b/tools/ci/integration
index a96da9ea4a..e3efec8313 100755
--- a/tools/ci/integration
+++ b/tools/ci/integration
@@ -3,23 +3,12 @@
 . tools/ci/common.sh
 set -eo pipefail
 
-cntr="${1:-scion_ci}"
 
-docker container exec $cntr bash -c "set -x; ./scion.sh topology ${DOCKER:+-d}"
+./docker.sh exec "set -x; ./scion.sh topology ${DOCKER:+-d}"
 
 if [ -n "$DOCKER" ]; then
-    # Make sure stale configs/state on the host are removed.
-    for i in gen gen-certs; do
-        [ -e "$i" ] && rm -r "$i"
-    done
-    rm -f ./logs/* ./gen-cache/*
-    mkdir -p gen-cache
-    # Copy out new topology
-    docker cp "$cntr:/home/scion/go/src/github.com/scionproto/scion/gen" .
-    docker cp "$cntr:/home/scion/go/src/github.com/scionproto/scion/gen-certs" .
-    # Run tests from host.
-    ./integration/integration_test.sh -d "$cntr" |& tee logs/integration.run
+    ./docker.sh exec "set -eo pipefail; ./integration/integration_test.sh |& tee logs/integration.run"
 else
     # Run tests inside the CI container.
-    docker container exec "$cntr" bash -c "set -eo pipefail; sudo service zookeeper start; ./integration/integration_test.sh |& tee logs/integration.run"
+    ./docker.sh exec "set -eo pipefail; sudo service zookeeper start; ./integration/integration_test.sh |& tee logs/integration.run"
 fi
diff --git a/tools/ci/lint b/tools/ci/lint
index 35208b9d05..460b57d7df 100755
--- a/tools/ci/lint
+++ b/tools/ci/lint
@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -c "set -eo pipefail; ./scion.sh lint |& tee logs/lint.run"
+./docker.sh exec "set -eo pipefail; ./scion.sh lint |& tee logs/lint.run"
diff --git a/tools/ci/local b/tools/ci/local
index 7b727ee210..817f542b02 100755
--- a/tools/ci/local
+++ b/tools/ci/local
@@ -2,10 +2,10 @@
 
 . tools/ci/common.sh
 
-cntr="${1:-scion_ci}"
+# Hardcoded in docker.sh
+cntr="scion"
 
 if [ -n "$DOCKER" ]; then
-    args="-d $cntr"
     # Stop a local scion run because we don't want to interfere
     ./scion.sh stop
     echo "Stop local zookeeper"
@@ -15,27 +15,27 @@ fi
 ./docker.sh build || exit 1
 [ -n "$DOCKER" ] && { make -sC docker/perapp || exit 1; }
 
-docker inspect "$cntr" &>/dev/null && { echo "Removing stale container"; docker rm -f "$cntr"; }
-./tools/ci/setup_container $args || exit 1
+docker container inspect "$cntr" &>/dev/null && { echo "Removing stale container"; docker rm -f "$cntr"; }
+
+tmpdir=$(mktemp -d /tmp/artifacts.XXXXXXX)
+SCION_MOUNT="$tmpdir" ./docker.sh start || exit 1
 
 result=0
-./tools/ci/build "$cntr"
+./tools/ci/build
 result=$((result+$?))
-./tools/ci/lint "$cntr"
+./tools/ci/lint
 result=$((result+$?))
-./tools/ci/unittest "$cntr"
+./tools/ci/unittest
 result=$((result+$?))
-./tools/ci/sphinx "$cntr"
+./tools/ci/sphinx
 result=$((result+$?))
 
-./tools/ci/integration $args "$cntr"
+./tools/ci/integration
 result=$((result+$?))
 
-tmpdir=$(mktemp -d /tmp/artifacts.XXXXXXX)
-./tools/ci/clean_up "$tmpdir" "$cntr" || exit 1
 echo "Artifacts dir: $tmpdir"
 
-[ $result -eq 0 ] && docker rm -f "$cntr"
+[ $result -eq 0 ] && ./docker.sh stop
 ./scion.sh stop
 
 if [ $result -eq 0 ]; then
diff --git a/tools/ci/setup_container b/tools/ci/setup_container
deleted file mode 100755
index d38625bd6a..0000000000
--- a/tools/ci/setup_container
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-. tools/ci/common.sh
-
-cntr=${1:-scion_ci}
-
-set -e
-
-[ -n "$DOCKER" ] && mount="-v /run/shm/dispatcher:/run/shm/dispatcher:rw -v /run/shm/sciond:/run/shm/sciond:rw"
-
-docker create --name=$cntr --entrypoint= $mount scion:latest tail -f /dev/null
-docker container start $cntr
diff --git a/tools/ci/sphinx b/tools/ci/sphinx
index ff89b2b655..868c0496c5 100755
--- a/tools/ci/sphinx
+++ b/tools/ci/sphinx
@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -lc "set -eo pipefail; make -f sphinx-doc/Makefile clean html |& tee logs/sphinx.run"
+./docker.sh exec "set -eo pipefail; make -f sphinx-doc/Makefile clean html |& tee logs/sphinx.run"
diff --git a/tools/ci/unittest b/tools/ci/unittest
index bf1464a407..6d8fa3d857 100755
--- a/tools/ci/unittest
+++ b/tools/ci/unittest
@@ -2,4 +2,4 @@
 
 set -ex
 
-docker container exec ${1:-scion_ci} bash -c "set -eo pipefail; ./scion.sh test |& tee logs/unittests.run"
+./docker.sh exec "set -eo pipefail; ./scion.sh test |& tee logs/unittests.run"

From 82d2447eb3d263f4497d5727a0c129fa05cc521c Mon Sep 17 00:00:00 2001
From: Lukas Bischofberger <bischofberger@anapaya.net>
Date: Fri, 19 Oct 2018 15:56:10 +0200
Subject: [PATCH 2/6] comments

---
 .circleci/config.yml | 7 ++++---
 docker.sh            | 9 ++++++++-
 tools/ci/local       | 2 --
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e092838f09..d1706fbe65 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -40,8 +40,8 @@ jobs:
                 name: Create and start container
                 command: |
                     set +e -x
-                    mkdir -p "/tmp/artifacts"
-                    SCION_MOUNT="/tmp/artifacts" ./docker.sh start
+                    mkdir -p /tmp/artifacts/mount
+                    SCION_MOUNT=/tmp/artifacts/mount ./docker.sh start
                 when: always
             - run:
                 name: Build
@@ -75,7 +75,8 @@ jobs:
                     [ -n "$CIRCLE_BUILD_NUM" ] && BUILD="build${CIRCLE_BUILD_NUM}"
                     ARTIFACTS="circleci.${CIRCLE_PROJECT_USERNAME}.${CIRCLE_PROJECT_REPONAME}.${TARGET}.${BUILD}"
                     mkdir -p "/tmp/artifacts.out"
-                    tar caf "/tmp/artifacts.out/$ARTIFACTS.tar.gz" -C /tmp artifacts
+                    mv /tmp/artifacts/{mount,"$ARTIFACTS"}
+                    tar caf "/tmp/artifacts.out/$ARTIFACTS.tar.gz" -C /tmp/artifacts "$ARTIFACTS"
                     ./docker.sh stop
                 when: always
             - store_artifacts:
diff --git a/docker.sh b/docker.sh
index 6aa8650c58..cbe9ad8ccd 100755
--- a/docker.sh
+++ b/docker.sh
@@ -120,6 +120,7 @@ cmd_start() {
     setup_volumes
     docker container create $args scion -c "tail -f /dev/null"
     docker start "$cntr"
+    # Adjust ownership of mounted dirs
     docker exec scion /docker-entrypoint.sh
 }
 
@@ -130,7 +131,7 @@ cmd_exec() {
 cmd_stop() {
     local cntr="scion"
     echo "Stopping $cntr container"; docker stop "$cntr";
-    echo "Removing $cntr container"; docker rm "$cntr";
+    echo "Removing $cntr container"; docker rm -f "$cntr";
 }
 
 setup_volumes() {
@@ -192,6 +193,12 @@ cmd_help() {
 	    $PROGRAM build
 	    $PROGRAM run
 	        Run the Docker image.
+	    $PROGRAM start
+	        Start a Docker container.
+	    $PROGRAM exec
+	        Execute a command in a running container.
+	    $PROGRAM stop
+	        Stop the Docker container.
 	    $PROGRAM clean
 	        Remove all Docker containers and all generated images.
 	    $PROGRAM help
diff --git a/tools/ci/local b/tools/ci/local
index 817f542b02..b9960e622f 100755
--- a/tools/ci/local
+++ b/tools/ci/local
@@ -15,8 +15,6 @@ fi
 ./docker.sh build || exit 1
 [ -n "$DOCKER" ] && { make -sC docker/perapp || exit 1; }
 
-docker container inspect "$cntr" &>/dev/null && { echo "Removing stale container"; docker rm -f "$cntr"; }
-
 tmpdir=$(mktemp -d /tmp/artifacts.XXXXXXX)
 SCION_MOUNT="$tmpdir" ./docker.sh start || exit 1
 

From c3d905e3812c9cf02c5e03717de944a76f9b9e15 Mon Sep 17 00:00:00 2001
From: Lukas Bischofberger <bischofberger@anapaya.net>
Date: Fri, 19 Oct 2018 16:06:44 +0200
Subject: [PATCH 3/6] more comments

---
 docker.sh            | 3 +--
 tools/ci/integration | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/docker.sh b/docker.sh
index cbe9ad8ccd..8fdaf69a73 100755
--- a/docker.sh
+++ b/docker.sh
@@ -130,8 +130,7 @@ cmd_exec() {
 
 cmd_stop() {
     local cntr="scion"
-    echo "Stopping $cntr container"; docker stop "$cntr";
-    echo "Removing $cntr container"; docker rm -f "$cntr";
+    echo "Stopping and removing $cntr container"; docker rm -f "$cntr";
 }
 
 setup_volumes() {
diff --git a/tools/ci/integration b/tools/ci/integration
index e3efec8313..129680b2aa 100755
--- a/tools/ci/integration
+++ b/tools/ci/integration
@@ -3,8 +3,8 @@
 . tools/ci/common.sh
 set -eo pipefail
 
-
-./docker.sh exec "set -x; ./scion.sh topology ${DOCKER:+-d}"
+# FIXME(worxli): host zk should be properly isolated from dockerized zk
+./docker.sh exec "set -x; ./scion.sh topology ${DOCKER:+zkclean -d}"
 
 if [ -n "$DOCKER" ]; then
     ./docker.sh exec "set -eo pipefail; ./integration/integration_test.sh |& tee logs/integration.run"

From ba465a393d437c3187e58da66bedd6d948345e97 Mon Sep 17 00:00:00 2001
From: Lukas Bischofberger <bischofberger@anapaya.net>
Date: Fri, 19 Oct 2018 16:16:40 +0200
Subject: [PATCH 4/6] quiet

---
 docker.sh                                   | 11 ++++++-----
 go/examples/pingpong/pp_integration/main.go |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/docker.sh b/docker.sh
index 8fdaf69a73..9225623e3c 100755
--- a/docker.sh
+++ b/docker.sh
@@ -112,14 +112,14 @@ cmd_start() {
     echo "SCION_MOUNT directory: $SCION_MOUNT"
     local cntr="scion"
     if docker container inspect "$cntr" &>/dev/null; then
-        echo "Removing stale container"
-        docker rm -f "$cntr"
+        echo "Removing stale $cntr container"
+        ./tools/quiet docker rm -f "$cntr"
     fi
     local args=$(common_args)
     args+=" --name $cntr"
     setup_volumes
-    docker container create $args scion -c "tail -f /dev/null"
-    docker start "$cntr"
+    ./tools/quiet docker container create $args scion -c "tail -f /dev/null"
+    ./tools/quiet docker start "$cntr"
     # Adjust ownership of mounted dirs
     docker exec scion /docker-entrypoint.sh
 }
@@ -130,7 +130,8 @@ cmd_exec() {
 
 cmd_stop() {
     local cntr="scion"
-    echo "Stopping and removing $cntr container"; docker rm -f "$cntr";
+    echo "Stopping $cntr container"; ./tools/quiet docker stop "$cntr";
+    echo "Removing $cntr container"; ./tools/quiet docker rm "$cntr";
 }
 
 setup_volumes() {
diff --git a/go/examples/pingpong/pp_integration/main.go b/go/examples/pingpong/pp_integration/main.go
index 1b8d7768b0..02eaea9bf4 100644
--- a/go/examples/pingpong/pp_integration/main.go
+++ b/go/examples/pingpong/pp_integration/main.go
@@ -68,7 +68,7 @@ func runTests(in integration.Integration, pairs []integration.IAPair) error {
 		for i, conn := range pairs {
 			log.Info(fmt.Sprintf("Test %v: %v -> %v (%v/%v)",
 				in.Name(), conn.Src, conn.Dst, i+1, len(pairs)))
-			if err := integration.RunClient(in, conn, 2*time.Second); err != nil {
+			if err := integration.RunClient(in, conn, 5*time.Second); err != nil {
 				fmt.Fprintf(os.Stderr, "Error during client execution: %s\n", err)
 				return err
 			}

From 7e3aaae1fe364b57ebed3915d8dc355661a5fb88 Mon Sep 17 00:00:00 2001
From: Lukas Bischofberger <bischofberger@anapaya.net>
Date: Mon, 22 Oct 2018 10:08:27 +0200
Subject: [PATCH 5/6] log error

---
 go/examples/pingpong/pp_integration/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go/examples/pingpong/pp_integration/main.go b/go/examples/pingpong/pp_integration/main.go
index 02eaea9bf4..93aef14609 100644
--- a/go/examples/pingpong/pp_integration/main.go
+++ b/go/examples/pingpong/pp_integration/main.go
@@ -69,7 +69,7 @@ func runTests(in integration.Integration, pairs []integration.IAPair) error {
 			log.Info(fmt.Sprintf("Test %v: %v -> %v (%v/%v)",
 				in.Name(), conn.Src, conn.Dst, i+1, len(pairs)))
 			if err := integration.RunClient(in, conn, 5*time.Second); err != nil {
-				fmt.Fprintf(os.Stderr, "Error during client execution: %s\n", err)
+				log.Error("Error during client execution", "err", err)
 				return err
 			}
 		}

From 441d33560d49d3c338c81e314657052c7c4d0e69 Mon Sep 17 00:00:00 2001
From: Lukas Bischofberger <bischofberger@anapaya.net>
Date: Mon, 22 Oct 2018 11:27:13 +0200
Subject: [PATCH 6/6] increase memory

---
 docker.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker.sh b/docker.sh
index 9225623e3c..0e67a4b3f9 100755
--- a/docker.sh
+++ b/docker.sh
@@ -78,8 +78,8 @@ cmd_clean() {
 }
 
 common_args() {
-    # Limit to 4G of ram, don't allow swapping.
-    local args="-h scion -m 4096M --memory-swap=4096M --shm-size=1024M $DOCKER_ARGS"
+    # Limit to 6G of ram, don't allow swapping.
+    local args="-h scion -m 6GB --memory-swap=6GB --shm-size=1024M $DOCKER_ARGS"
     args+=" -v /var/run/docker.sock:/var/run/docker.sock"
     args+=" -v $SCION_MOUNT/gen:/home/scion/go/src/github.com/scionproto/scion/gen"
     args+=" -v $SCION_MOUNT/logs:/home/scion/go/src/github.com/scionproto/scion/logs"