Acceptance test environment

Jenkinsfile

@@ -69,20 +69,12 @@ def stage_rpmbuild(label) {
   }
 }
 
-def stage_integration(label) {
+def stage_acceptance(label) {
   node("multibox") {
+    stage "Acceptance Test ${label}"
     checkout_and_merge()
-    stage "Build Integration Environment"
     write_build_env(label)
-
-    try {
-      sh "cd ci/multibox/ ; ./build.sh"
-      stage "Run Tntegration Test"
-      // This is where the integration test will be run
-    } finally {
-      stage "Cleanup Environment"
-      sh "cd ci/multibox/ ; ./destroy_leaving_cache.sh"
-    }
+    sh "./ci/acceptance-test/build_and_run_in_docker.sh ./build.env"
   }
 }
 
@@ -112,5 +104,5 @@ if( buildParams.BUILD_OS != "all" ){
 for( label in build_nodes) {
   stage_unit_test(label)
   stage_rpmbuild(label)
-  stage_integration(label)
+  stage_acceptance(label)
 }

ci/acceptance-test/.dockerignore

@@ -0,0 +1,3 @@
+tests
+illustrations
+build_and_run_in_docker.sh

ci/acceptance-test/Dockerfile

@@ -0,0 +1,29 @@
+FROM centos:7
+
+MAINTAINER "Axsh Co. LTD"
+
+VOLUME /data
+
+RUN ["yum", "install", "-y", "epel-release"]
+RUN ["yum", "install", "-y", "rsync", "bridge-utils", "qemu-kvm", "qemu-system-x86", "parted", "sudo", "openssh-clients", "nmap-ncat"]
+
+RUN groupadd -r axsh && useradd -d /home/axsh -g axsh axsh
+WORKDIR /home/axsh
+
+ARG BRANCH
+ARG RELEASE_SUFFIX
+ARG REBUILD
+
+# Set the ARGs to ENV because otherwise they're not visible to the run_tests.sh script
+ENV BRANCH=${BRANCH:-master} RELEASE_SUFFIX=${RELEASE_SUFFIX:-current} REBUILD=${REBUILD:-false}
+
+LABEL "jp.axsh.vendor"="Axsh Co. LTD"  \
+      "jp.axsh.project"="OpenVDC" \
+      "jp.axsh.task"="acceptance test" \
+      "jp.axsh.branch"="$BRANCH" \
+      "jp.axsh.release_suffix"="$RELEASE_SUFFIX"
+
+COPY ["multibox", "/multibox"]
+COPY ["run_tests.sh", "run_tests.sh"]
+
+ENTRYPOINT ["./run_tests.sh", "RUN"]

ci/acceptance-test/README.md

@@ -0,0 +1,141 @@
+# OpenVDC Acceptance Test
+
+## Quick start
+
+![Test environment drawing](illustrations/drawing.svg)
+
+This is the environment used on the OpenVDC CI to run the integration tests. To run this environment locally first make a file containing the following environment variables.
+
+```
+# The following two lines make up the yum repository from which we'll download OpenVDC packages to test
+# "https://ci.openvdc.org/repos/${BRANCH}/${RELEASE_SUFFIX}/"
+BRANCH="master"
+RELEASE_SUFFIX="current"
+
+# Set to "1" if you don't want to remove the Docker container after running
+REBUILD="0"
+
+# This variable holds the directory on the *host* that will be bind mounted
+# into the Docker container to place the cache for KVM images.
+# If not provided, the default value is "/data2"
+DATA_DIR=/data2
+```
+
+Let's save this file as `build.env`. Now kick off the `build_and_run_in_docker.sh` script, passing in that file as an argument.
+
+```
+./build_and_run_in_docker.sh build.env
+```
+
+That's it. This should build the environment and run the tests
+
+## The nitty gritty
+
+Here's everything you need to know about how this acceptance test works exactly.
+
+### The actual test code
+
+The actual test code is in the `tests` directory. This is written in go using the [Testing](https://golang.org/pkg/testing/) package. During the CI's rpmbuild stage, these tests are compiled as a binary and packaged in an rpm called `openvdc-acceptance-test`. This way we don't have to worry about installing go and setting up a GOPATH on the test environment.
+
+### The Docker container
+
+The test environment consists of multiple KVM machines and we've decided to put them all together in a Docker container. That way we can test multiple branches in parallel and cleanup becomes simple. We just remove the container when we're done.
+
+The command we run inside of the Docker container is `run_tests.sh`. This script will first write the correct OpenVDC yum repository to `/etc/yum.repos.d/openvdc.repo`. The environment variables described in [Quick start](#quick-start) decide which repository is used.
+
+Next the `run_tests.sh` script installs the `openvdc-acceptance-test` package through yum which depends on `openvdc-cli`.
+
+Now we build the KVM environment.
+
+### The KVM environment
+
+The code that builds the KVM machines running inside of the Docker container is in the `multibox` directory.
+
+If you want to run this KVM environment locally without Docker, you can do so like this.
+
+```
+BRANCH=master RELEASE_SUFFIX=current REBUILD=false ./build.sh
+```
+
+This environment uses a caching system to avoid having to constantly rebuild the images every time we test a new branch or commit. The system works as follows.
+
+#### First time ever run
+
+* Download seed image containing a minimal Centos 7. All images built are based on this.
+
+#### First time run on a branch
+
+* Build images in .raw format **without** openvdc installed. We use .raw so we can loopback mount and install packages using chroot.
+
+* Convert images to .qcow format so we can use copy-on-write.
+
+* Store images in cache directory.
+
+#### Every time we run on a branch
+
+* Copy-on-write cached images for branch to the directory we will run KVM from.
+
+* Run KVM.
+
+* Install OpenVDC packages on KVM VMs for the branch/commit we're testing
+
+The cache directory is bind mounted into the Docker container so the same cache is re-used every time the tests run.
+
+### The actual tests
+
+Once the KVM environment is in place we can run the actual tests. This is the `run_tests.sh` script's final action.
+
+To make sure that the `openvdc` command doesn't require root previliges, we run it as an unpreviliged `axsh` user.
+
+### Cleanup
+
+The `build_and_run_in_docker.sh` script traps EXIT and calls a cleanup function that will remove the Docker container unless `LEAVE_CONTAINER` is set.
+
+The leftover KVM cache and Docker images are cleaned up periodically by a garbage collection job on Jenkins.
+
+## FAQ
+
+Keep in mind that all answers are true at time of writing (January 2017) and might no longer be accurate.
+
+#### What OS has it run on successfully?
+
+Fedora 23 and Arch Linux.
+
+#### What Docker versions has it run on successfully?
+
+1.10.3 on Fedora and 1.12.6 on Arch.
+
+#### How much disk space is required?
+
+609 MB for the KVM centos 7 seed image.
+
+```
+> du -hs openvdc-ci/boxes
+
+609M    openvdc-ci/boxes
+```
+
+About 3.9 GB KVM cache per branch
+
+```
+> du -hs openvdc-ci/branches/*
+
+3.9G    openvdc-ci/branches/acceptance-test
+3.8G    openvdc-ci/branches/master
+```
+
+About 470 MB per Docker image plus about 190 MB for the base Centos 7 image.
+
+```
+> sudo docker images
+
+REPOSITORY                TAG                       IMAGE ID            CREATED             SIZE
+openvdc/acceptance-test   acceptance-test.current   8f53091d43bb        29 minutes ago      470.8 MB
+centos                    7                         67591570dd29        5 weeks ago         191.8 MB
+```
+
+You do the math. :p
+
+#### How much memory is required?
+
+We've got 5 KVM machines with 1 GB of memory assigned to each one.

ci/acceptance-test/build_and_run_in_docker.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -ex -o pipefail
+
+whereami="$(cd "$(dirname $(readlink -f "$0"))" && pwd -P)"
+
+BUILD_ENV_PATH=${1:?"ERROR: env file is not given."}
+if [[ -n "${BUILD_ENV_PATH}" && ! -f "${BUILD_ENV_PATH}" ]]; then
+  echo "ERROR: Can't find the file: ${BUILD_ENV_PATH}" >&2
+  exit 1
+fi
+
+set -a
+. ${BUILD_ENV_PATH}
+set +a
+
+DATA_DIR="${DATA_DIR:-/data2}"
+repo_and_tag="openvdc/acceptance-test:${BRANCH}.${RELEASE_SUFFIX}"
+
+function cleanup() {
+  if [[ -z "${LEAVE_CONTAINER}" || "${LEAVE_CONTAINER}" == "0" ]]; then
+    # Clean up containers
+    # Images don't need to be cleaned up. Removing them immediately would slow down
+    # builds and they can be garbage collected later.
+    for CID in $(sudo docker ps -af ancestor="${repo_and_tag}" --format "{{.ID}}"); do
+      sudo docker rm "${CID}"
+    done
+  else
+    echo "LEAVE_CONTAINER was set and not 0. Skip container cleanup."
+  fi
+}
+trap "cleanup" EXIT
+
+sudo docker build -t "${repo_and_tag}" --build-arg BRANCH="${BRANCH}" \
+                                  --build-arg RELEASE_SUFFIX="${RELEASE_SUFFIX}" \
+                                  --build-arg REBUILD="${REBUILD}" \
+                                  "${whereami}"
+
+sudo docker run --privileged -v "${DATA_DIR}":/data "${repo_and_tag}"