scionproto · worxli · Dec 14, 2018 · Dec 13, 2018 · Dec 14, 2018 · Dec 14, 2018
diff --git a/.buildkite/steps/acceptance b/.buildkite/steps/acceptance
@@ -20,7 +20,7 @@ for test in ./acceptance/*_acceptance; do
         echo "  - $BASE/steps/host_acceptance $test"
     else
         echo "  env:"
-        echo "    DOCKER_ARGS: \"-e ACCEPTANCE_ARTIFACTS=logs/acceptance\""
+        echo "    DOCKER_ARGS: \"-e ACCEPTANCE_ARTIFACTS=logs/acceptance --network=host\""
         echo "  command:"
         echo "  - $BASE/scripts/all_images pull"
         echo "  - $BASE/run_step run_acceptance $test"

diff --git a/acceptance/README.md b/acceptance/README.md
@@ -49,5 +49,10 @@ acceptance/ctl gsetup
 ```
 This will also rebuild the docker images, taking new code into account.
 
+To run the `setup`, `run` and `teardown` phases of a single test (without gsetup):
+```
+acceptance/ctl grun TESTNAME
+```
+
 Note that `acceptance/ctl` will not save artifacts on its own, and all output
 is dumped on the console.
diff --git a/acceptance/common.sh b/acceptance/common.sh
@@ -0,0 +1,56 @@
+# Acceptance tests common functions
+
+#######################################
+# Converts ISD-AS representation to its file format
+# Arguments:
+#   ISD-AS
+#######################################
+ia_file() {
+    echo ${1:?} | sed -e "s/:/_/g"
+}
+
+#######################################
+# Converts ISD-AS representation to AS file format, i.e. removes the ISD
+# Arguments:
+#   ISD-AS
+#######################################
+as_file() {
+    ia_file ${1:?} | cut -d '-' -f 2
+}
+
+#######################################
+# Collects metrics from services that can be found in the topology file
+# Metrics are saved in METRICS_DIR/
+# Arguments:
+#   Topology path
+#   METRICS_DIR
+#######################################
+collect_metrics() {
+    TOPOLOGY=${1:?}
+    echo "Reading topology: $TOPOLOGY"
+    METRICS_DIR=${2:?}
+    echo "Saving metrics in $METRICS_DIR"
+    mkdir -p "$METRICS_DIR"
+
+    collect_elem_metrics 'BorderRouters' '30442' 'InternalAddrs.IPv4.PublicOverlay.Addr'
+    collect_elem_metrics 'SIG' '30456'
+    collect_elem_metrics 'BeaconService' '30452'
+    collect_elem_metrics 'CertificateService' '30454'
+    collect_elem_metrics 'PathService' '30453'
+}
+
+#######################################
+# Collect metrics for one element type
+# Arguments:
+#   Service Type
+#   Port
+#   IP Addr Key (optional)
+#######################################
+collect_elem_metrics() {
+    local elems=$(jq -r ".${1:?} | keys | .[]" $TOPOLOGY)
+    local addr_key=${3:-Addrs.IPv4.Public.Addr}
+    for elem in $elems; do
+        local ip="$(jq -r .$1[\"$elem\"].$addr_key $TOPOLOGY)"
+        curl "$ip:${2:?}/metrics" -o "$METRICS_DIR/$elem" -s -S --connect-timeout 2 || true
+    done
+}
diff --git a/acceptance/ctl b/acceptance/ctl
@@ -32,7 +32,9 @@ case "$COMMAND" in
     gsetup)
         global_setup ;;
     grun)
-        global_run_single "$1" ;;
+        artifacts_dir; stats_failed=0
+        global_run_single "$1"
+        [ $stats_failed -eq 0 ] ;;
     gteardown)
         global_teardown ;;
     *) cmd_help; exit 1 ;;

diff --git a/acceptance/lib.sh b/acceptance/lib.sh
@@ -42,7 +42,13 @@ build_docker_perapp() {
     make -C docker/perapp
 }
 
+artifacts_dir() {
+    export ACCEPTANCE_ARTIFACTS="${ACCEPTANCE_ARTIFACTS:-$(mktemp -d /tmp/acceptance-artifacts-$(date +"%Y%m%d-%H%M%S").XXXXXXX)}"
+    echo "Acceptance artifacts saved to $ACCEPTANCE_ARTIFACTS"
+}
+
 global_setup() {
+    artifacts_dir
     local out_dir="$ACCEPTANCE_ARTIFACTS"
     set -e
     print_green "[==========]"
@@ -89,9 +95,9 @@ test_run_wrapper() {
 }
 
 save_logs() {
-    local out="$1"
-    mkdir -p "$out/$TEST_NAME/logs"
-    mv logs/* "$out/$TEST_NAME/logs"
+    local out_dir="${ACCEPTANCE_ARTIFACTS:?}"
+    mkdir -p "$out_dir/$TEST_NAME/logs"
+    mv logs/* "$out_dir/$TEST_NAME/logs"
 }
 
 test_teardown_wrapper() {
@@ -114,7 +120,7 @@ global_run() {
         if [[ "$TEST_NAME" =~ $regex_matcher ]]; then
             global_run_single "$TEST_PROGRAM"
             local fatal_teardown=$?
-            save_logs "$out"
+            save_logs
             if [ $fatal_teardown -ne 0 ]; then
                 print_red "[  FATAL   ]" "Teardown failed, stopping test suite"
                 exit 1
@@ -127,13 +133,13 @@ global_run() {
 }
 
 global_run_single() {
-    local out="${ACCEPTANCE_ARTIFACTS:?}"
+    local out_dir="${ACCEPTANCE_ARTIFACTS:?}"
     TEST_PROGRAM="${1:?}"
     TEST_NAME=$($TEST_PROGRAM name)
-    mkdir -p "$out/$TEST_NAME"
-    SETUP_FILE="$out/$TEST_NAME/setup.out"
-    RUN_FILE="$out/$TEST_NAME/run.out"
-    TEARDOWN_FILE="$out/$TEST_NAME/teardown.out"
+    mkdir -p "$out_dir/$TEST_NAME"
+    SETUP_FILE="$out_dir/$TEST_NAME/setup.out"
+    RUN_FILE="$out_dir/$TEST_NAME/run.out"
+    TEARDOWN_FILE="$out_dir/$TEST_NAME/teardown.out"
     test_setup_wrapper "$SETUP_FILE" && \
         test_run_wrapper "$RUN_FILE"
     test_teardown_wrapper "$TEARDOWN_FILE"

diff --git a/acceptance/run b/acceptance/run
@@ -2,9 +2,6 @@
 
 . acceptance/lib.sh
 
-export ACCEPTANCE_ARTIFACTS=$(mktemp -d /tmp/acceptance-artifacts-$(date +"%Y%m%d-%H%M%S").XXXXXXX)
-echo "Acceptance artifacts saved to $ACCEPTANCE_ARTIFACTS"
-
 stats_passed=0
 stats_failed=0
 stats_skipped=0

diff --git a/acceptance/sig_acceptance/test b/acceptance/sig_acceptance/test
@@ -3,9 +3,11 @@
 TEST_NAME="sig"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
+. acceptance/common.sh
+
 test_setup() {
-    set -ex
-    ./scion.sh topology zkclean -c $TEST_TOPOLOGY -d --sig -n 242.0.0.0/16
+    set -e
+    ./scion.sh topology zkclean -c $TEST_TOPOLOGY -d --sig -n 242.254.0.0/16
     ./scion.sh run nobuild
     ./tools/dc start 'tester*'
     sleep 10
@@ -16,6 +18,11 @@ test_run() {
 }
 
 test_teardown() {
+    set -e
+    METRICS="${ACCEPTANCE_ARTIFACTS:?}/$TEST_NAME/metrics"
+    collect_metrics "gen/ISD1/ASff00_0_110/br1-ff00_0_110-1/topology.json" "$METRICS"
+    collect_metrics "gen/ISD1/ASff00_0_111/br1-ff00_0_111-1/topology.json" "$METRICS"
+    collect_metrics "gen/ISD1/ASff00_0_112/br1-ff00_0_112-1/topology.json" "$METRICS"
     ./tools/dc down
 }
 

diff --git a/acceptance/topo_ps_reloads_br_acceptance/test b/acceptance/topo_ps_reloads_br_acceptance/test
@@ -5,12 +5,14 @@
 # use the new internal address as next-hop when doing path segment
 # synchronization with other core PSes.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_ps_reloads_br"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:111}
 
 test_setup() {

diff --git a/acceptance/topo_ps_reloads_cs_acceptance/test b/acceptance/topo_ps_reloads_cs_acceptance/test
@@ -4,12 +4,14 @@
 # the PS process receives a SIGHUP it will reload the config and push future
 # crypto to the new address.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_ps_reloads_cs"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 IA=${IA:-1-ff00:0:112}
-IA_FILE=$(echo $IA | sed -e "s/:/_/g")
-AS_FILE=$(echo $IA_FILE | cut -d '-' -f 2)
+IA_FILE="$(ia_file $IA)"
+AS_FILE="$(as_file $IA)"
 
 test_setup() {
     set -e

diff --git a/acceptance/topo_sd_reloads_br_acceptance/test b/acceptance/topo_sd_reloads_br_acceptance/test
@@ -5,12 +5,14 @@
 # use the new internal address as next-hop in the replies to path requests from
 # clients.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_sd_reloads_br"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {

diff --git a/acceptance/topo_sd_reloads_cs_acceptance/test b/acceptance/topo_sd_reloads_cs_acceptance/test
@@ -4,12 +4,14 @@
 # the SD process receives a SIGHUP it will reload the config and get future
 # certificate chains from the new address.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_sd_reloads_cs"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {

diff --git a/acceptance/topo_sd_reloads_ps_acceptance/test b/acceptance/topo_sd_reloads_ps_acceptance/test
@@ -4,12 +4,14 @@
 # the SD process receives a SIGHUP it will reload the config and service future
 # client requests for paths by querying the new PS addresses.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_reloading"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {

diff --git a/python/topology/common.py b/python/topology/common.py
@@ -89,16 +89,16 @@ def __repr__(self):
         return "<TopoID: %s>" % self
 
 
-def prom_addr_br(br_id, br_ele, port_gen):
+def prom_addr_br(br_id, br_ele, port):
     """Get the prometheus address for a border router"""
     pub = get_pub(br_ele['InternalAddrs'])
-    return "[%s]:%s" % (pub['PublicOverlay']['Addr'].ip, port_gen.register(br_id + "prom"))
+    return "[%s]:%s" % (pub['PublicOverlay']['Addr'].ip, port)
 
 
-def prom_addr_infra(infra_id, infra_ele, port_gen):
+def prom_addr_infra(infra_id, infra_ele, port):
     """Get the prometheus address for an infrastructure element."""
     pub = get_pub(infra_ele['Addrs'])
-    return "[%s]:%s" % (pub['Public']['Addr'].ip, port_gen.register(infra_id + "prom"))
+    return "[%s]:%s" % (pub['Public']['Addr'].ip, port)
 
 
 def get_pub(topo_addr):

diff --git a/python/topology/docker.py b/python/topology/docker.py
@@ -30,11 +30,15 @@
     ArgsTopoDicts,
     docker_image,
     DOCKER_USR_VOL,
-    prom_addr_infra,
     sciond_name,
     sciond_svc_name
 )
 from topology.docker_utils import DockerUtilsGenArgs, DockerUtilsGenerator
+from topology.prometheus import (
+    BS_PROM_PORT,
+    CS_PROM_PORT,
+    PS_PROM_PORT
+)
 from topology.sig import SIGGenArgs, SIGGenerator
 
 DOCKER_CONF = 'scion-dc.yml'
@@ -159,6 +163,7 @@ def _cs_conf(self, topo_id, topo, base):
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
             },
+            'network_mode': 'service:scion_disp_%s' % topo_id.file_fmt(),
             'volumes': self._std_vol(topo_id),
             'command': []
         }
@@ -169,7 +174,7 @@ def _cs_conf(self, topo_id, topo, base):
             if self.args.cert_server == 'py':
                 sciond = get_default_sciond_path(ISD_AS(topo["ISD_AS"]))
                 entry['command'].append('--spki_cache_dir=cache')
-                entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
+                entry['command'].append('--prom=[0.0.0.0]:%s' % CS_PROM_PORT)
                 entry['command'].append('--sciond_path=%s' % sciond)
                 entry['command'].append(k)
                 entry['command'].append('conf')
@@ -185,6 +190,7 @@ def _bs_conf(self, topo_id, topo, base):
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
             },
+            'network_mode': 'service:scion_disp_%s' % topo_id.file_fmt(),
             'volumes': self._std_vol(topo_id),
             'command': [
                 '--spki_cache_dir=cache'
@@ -195,7 +201,7 @@ def _bs_conf(self, topo_id, topo, base):
             name = self.prefix + k
             entry['container_name'] = name
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
-            entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
+            entry['command'].append('--prom=[0.0.0.0]:%s' % BS_PROM_PORT)
             entry['command'].append('--sciond_path=%s' %
                                     get_default_sciond_path(ISD_AS(topo["ISD_AS"])))
             entry['command'].append(k)
@@ -213,6 +219,7 @@ def _ps_conf(self, topo_id, topo, base):
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
             },
+            'network_mode': 'service:scion_disp_%s' % topo_id.file_fmt(),
             'volumes': self._std_vol(topo_id),
             'command': [],
         }
@@ -223,7 +230,7 @@ def _ps_conf(self, topo_id, topo, base):
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
             if self.args.path_server == 'py':
                 entry['command'].append('--spki_cache_dir=cache')
-                entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
+                entry['command'].append('--prom=[0.0.0.0]:%s' % PS_PROM_PORT)
                 entry['command'].append('--sciond_path=%s' %
                                         get_default_sciond_path(ISD_AS(topo["ISD_AS"])))
                 entry['command'].append(k)

diff --git a/python/topology/go.py b/python/topology/go.py
@@ -34,6 +34,7 @@
     prom_addr_br,
     PS_CONFIG_NAME,
 )
+from topology.prometheus import DEFAULT_BR_PROM_PORT
 
 
 class GoGenArgs(ArgsTopoDicts):
@@ -66,7 +67,7 @@ def _build_br_conf(self, topo_id, ia, base, name, v):
             },
             'logging': self._log_entry(name),
             'metrics': {
-                'Prometheus': prom_addr_br(name, v, self.args.port_gen),
+                'Prometheus': prom_addr_br(name, v, DEFAULT_BR_PROM_PORT),
             },
             'br': {
                 'Profile': False,