Acceptance tests: Collect and logs metrics

.buildkite/pipeline.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+export RUN_ALL_TESTS=y
+
 export BASE=".buildkite"
 STEPS="$BASE/steps"
 

acceptance/README.md

@@ -49,5 +49,10 @@ acceptance/ctl gsetup
 ```
 This will also rebuild the docker images, taking new code into account.
 
+To run the `setup`, `run` and `teardown` phases of a single test (without gsetup):
+```
+acceptance/ctl grun TESTNAME
+```
+
 Note that `acceptance/ctl` will not save artifacts on its own, and all output
 is dumped on the console.

acceptance/common.sh

@@ -0,0 +1,32 @@
+# Acceptance tests common functions
+
+ia_file() {
+    echo ${1:?} | sed -e "s/:/_/g"
+}
+
+as_file() {
+    ia_file ${1:?} | cut -d '-' -f 2
+}
+
+collect_metrics() {
+    echo "Reading topology: ${1:?}"
+    METRICS_DIR=${2:?}
+    echo "Saving metrics in $METRICS_DIR"
+    mkdir -p "$METRICS_DIR"
+
+    local elems=$(jq '.BorderRouters | to_entries[] | .key' $1)
+    for elem in $elems; do
+        local ip="$(jq .BorderRouters[$elem].InternalAddrs.IPv4.PublicOverlay.Addr $1 | sed -e 's/^"//' -e 's/"$//')"
+        curl "$ip:30442/metrics" -o "$METRICS_DIR/$(remove_quotes $elem)" -s -S
+    done
+
+    local elems=$(jq '.SIG | to_entries[] | .key' $1)
+    for elem in $elems; do
+        local ip="$(jq .SIG[$elem].Addrs.IPv4.Public.Addr $1 | sed -e 's/^"//' -e 's/"$//')"
+        curl "$ip:30456/metrics" -o "$METRICS_DIR/$(remove_quotes $elem)" -s -S
+    done
+}
+
+remove_quotes() {
+    echo "${1:?}" | sed -e 's/^"//' -e 's/"$//'
+}

acceptance/ctl

@@ -32,7 +32,9 @@ case "$COMMAND" in
     gsetup)
         global_setup ;;
     grun)
-        global_run_single "$1" ;;
+        artifacts_dir; stats_failed=0
+        global_run_single "$1"
+        [ $stats_failed -eq 0 ] ;;
     gteardown)
         global_teardown ;;
     *) cmd_help; exit 1 ;;

acceptance/lib.sh

@@ -42,7 +42,13 @@ build_docker_perapp() {
     make -C docker/perapp
 }
 
+artifacts_dir() {
+    export ACCEPTANCE_ARTIFACTS="${ACCEPTANCE_ARTIFACTS:-$(mktemp -d /tmp/acceptance-artifacts-$(date +"%Y%m%d-%H%M%S").XXXXXXX)}"
+    echo "Acceptance artifacts saved to $ACCEPTANCE_ARTIFACTS"
+}
+
 global_setup() {
+    artifacts_dir
     local out_dir="$ACCEPTANCE_ARTIFACTS"
     set -e
     print_green "[==========]"
@@ -89,9 +95,9 @@ test_run_wrapper() {
 }
 
 save_logs() {
-    local out="$1"
-    mkdir -p "$out/$TEST_NAME/logs"
-    mv logs/* "$out/$TEST_NAME/logs"
+    local out_dir="${ACCEPTANCE_ARTIFACTS:?}"
+    mkdir -p "$out_dir/$TEST_NAME/logs"
+    mv logs/* "$out_dir/$TEST_NAME/logs"
 }
 
 test_teardown_wrapper() {
@@ -114,7 +120,7 @@ global_run() {
         if [[ "$TEST_NAME" =~ $regex_matcher ]]; then
             global_run_single "$TEST_PROGRAM"
             local fatal_teardown=$?
-            save_logs "$out"
+            save_logs
             if [ $fatal_teardown -ne 0 ]; then
                 print_red "[  FATAL   ]" "Teardown failed, stopping test suite"
                 exit 1
@@ -127,13 +133,13 @@ global_run() {
 }
 
 global_run_single() {
-    local out="${ACCEPTANCE_ARTIFACTS:?}"
+    local out_dir="${ACCEPTANCE_ARTIFACTS:?}"
     TEST_PROGRAM="${1:?}"
     TEST_NAME=$($TEST_PROGRAM name)
-    mkdir -p "$out/$TEST_NAME"
-    SETUP_FILE="$out/$TEST_NAME/setup.out"
-    RUN_FILE="$out/$TEST_NAME/run.out"
-    TEARDOWN_FILE="$out/$TEST_NAME/teardown.out"
+    mkdir -p "$out_dir/$TEST_NAME"
+    SETUP_FILE="$out_dir/$TEST_NAME/setup.out"
+    RUN_FILE="$out_dir/$TEST_NAME/run.out"
+    TEARDOWN_FILE="$out_dir/$TEST_NAME/teardown.out"
     test_setup_wrapper "$SETUP_FILE" && \
         test_run_wrapper "$RUN_FILE"
     test_teardown_wrapper "$TEARDOWN_FILE"

acceptance/run

@@ -2,9 +2,6 @@
 
 . acceptance/lib.sh
 
-export ACCEPTANCE_ARTIFACTS=$(mktemp -d /tmp/acceptance-artifacts-$(date +"%Y%m%d-%H%M%S").XXXXXXX)
-echo "Acceptance artifacts saved to $ACCEPTANCE_ARTIFACTS"
-
 stats_passed=0
 stats_failed=0
 stats_skipped=0

acceptance/sig_acceptance/test

@@ -4,8 +4,8 @@ TEST_NAME="sig"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 test_setup() {
-    set -ex
-    ./scion.sh topology zkclean -c $TEST_TOPOLOGY -d --sig -n 242.0.0.0/16
+    set -e
+    ./scion.sh topology zkclean -c $TEST_TOPOLOGY -d --sig -n 242.254.0.0/16
     ./scion.sh run nobuild
     ./tools/dc start 'tester*'
     sleep 10
@@ -16,6 +16,11 @@ test_run() {
 }
 
 test_teardown() {
+    set -e
+    METRICS="${ACCEPTANCE_ARTIFACTS:?}/$TEST_NAME/metrics"
+    collect_metrics "gen/ISD1/ASff00_0_110/br1-ff00_0_110-1/topology.json" "$METRICS"
+    collect_metrics "gen/ISD1/ASff00_0_111/br1-ff00_0_111-1/topology.json" "$METRICS"
+    collect_metrics "gen/ISD1/ASff00_0_112/br1-ff00_0_112-1/topology.json" "$METRICS"
     ./tools/dc down
 }
 

acceptance/topo_ps_reloads_br_acceptance/test

@@ -5,12 +5,14 @@
 # use the new internal address as next-hop when doing path segment
 # synchronization with other core PSes.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_ps_reloads_br"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:111}
 
 test_setup() {

acceptance/topo_ps_reloads_cs_acceptance/test

@@ -4,12 +4,14 @@
 # the PS process receives a SIGHUP it will reload the config and push future
 # crypto to the new address.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_ps_reloads_cs"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 IA=${IA:-1-ff00:0:112}
-IA_FILE=$(echo $IA | sed -e "s/:/_/g")
-AS_FILE=$(echo $IA_FILE | cut -d '-' -f 2)
+IA_FILE="$(ia_file $SRC_IA)"
+AS_FILE="$(as_file $SRC_IA)"
 
 test_setup() {
     set -e

acceptance/topo_sd_reloads_br_acceptance/test

@@ -5,12 +5,14 @@
 # use the new internal address as next-hop in the replies to path requests from
 # clients.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_sd_reloads_br"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {

acceptance/topo_sd_reloads_cs_acceptance/test

@@ -4,12 +4,14 @@
 # the SD process receives a SIGHUP it will reload the config and get future
 # certificate chains from the new address.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_sd_reloads_cs"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {

acceptance/topo_sd_reloads_ps_acceptance/test

@@ -4,12 +4,14 @@
 # the SD process receives a SIGHUP it will reload the config and service future
 # client requests for paths by querying the new PS addresses.
 
+. acceptance/common.sh
+
 TEST_NAME="topo_reloading"
 TEST_TOPOLOGY="topology/Tiny.topo"
 
 SRC_IA=${SRC_IA:-1-ff00:0:112}
-SRC_IA_FILE=$(echo $SRC_IA | sed -e "s/:/_/g")
-SRC_AS_FILE=$(echo $SRC_IA_FILE | cut -d '-' -f 2)
+SRC_IA_FILE="$(ia_file $SRC_IA)"
+SRC_AS_FILE="$(as_file $SRC_IA)"
 DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {

python/topology/common.py

@@ -89,16 +89,16 @@ def __repr__(self):
         return "<TopoID: %s>" % self
 
 
-def prom_addr_br(br_id, br_ele, port_gen):
+def prom_addr_br(br_id, br_ele, port):
     """Get the prometheus address for a border router"""
     pub = get_pub(br_ele['InternalAddrs'])
-    return "[%s]:%s" % (pub['PublicOverlay']['Addr'].ip, port_gen.register(br_id + "prom"))
+    return "[%s]:%s" % (pub['PublicOverlay']['Addr'].ip, port)
 
 
-def prom_addr_infra(infra_id, infra_ele, port_gen):
+def prom_addr_infra(infra_id, infra_ele, port):
     """Get the prometheus address for an infrastructure element."""
     pub = get_pub(infra_ele['Addrs'])
-    return "[%s]:%s" % (pub['Public']['Addr'].ip, port_gen.register(infra_id + "prom"))
+    return "[%s]:%s" % (pub['Public']['Addr'].ip, port)
 
 
 def get_pub(topo_addr):

python/topology/docker.py

@@ -30,11 +30,15 @@
     ArgsTopoDicts,
     docker_image,
     DOCKER_USR_VOL,
-    prom_addr_infra,
     sciond_name,
     sciond_svc_name
 )
 from topology.docker_utils import DockerUtilsGenArgs, DockerUtilsGenerator
+from topology.prometheus import (
+    BS_PROM_PORT,
+    CS_PROM_PORT,
+    PS_PROM_PORT
+)
 from topology.sig import SIGGenArgs, SIGGenerator
 
 DOCKER_CONF = 'scion-dc.yml'
@@ -169,7 +173,7 @@ def _cs_conf(self, topo_id, topo, base):
             if self.args.cert_server == 'py':
                 sciond = get_default_sciond_path(ISD_AS(topo["ISD_AS"]))
                 entry['command'].append('--spki_cache_dir=cache')
-                entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
+                entry['command'].append('--prom=[0.0.0.0]:%s' % CS_PROM_PORT)
                 entry['command'].append('--sciond_path=%s' % sciond)
                 entry['command'].append(k)
                 entry['command'].append('conf')
@@ -195,7 +199,7 @@ def _bs_conf(self, topo_id, topo, base):
             name = self.prefix + k
             entry['container_name'] = name
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
-            entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
+            entry['command'].append('--prom=[0.0.0.0]:%s' % BS_PROM_PORT)
             entry['command'].append('--sciond_path=%s' %
                                     get_default_sciond_path(ISD_AS(topo["ISD_AS"])))
             entry['command'].append(k)
@@ -223,7 +227,7 @@ def _ps_conf(self, topo_id, topo, base):
             entry['volumes'].append('%s:/share/conf:ro' % os.path.join(base, k))
             if self.args.path_server == 'py':
                 entry['command'].append('--spki_cache_dir=cache')
-                entry['command'].append('--prom=%s' % prom_addr_infra(k, v, self.args.port_gen))
+                entry['command'].append('--prom=[0.0.0.0]:%s' % PS_PROM_PORT)
                 entry['command'].append('--sciond_path=%s' %
                                         get_default_sciond_path(ISD_AS(topo["ISD_AS"])))
                 entry['command'].append(k)

python/topology/go.py

@@ -34,6 +34,7 @@
     prom_addr_br,
     PS_CONFIG_NAME,
 )
+from topology.prometheus import DEFAULT_BR_PROM_PORT
 
 
 class GoGenArgs(ArgsTopoDicts):
@@ -66,7 +67,7 @@ def _build_br_conf(self, topo_id, ia, base, name, v):
             },
             'logging': self._log_entry(name),
             'metrics': {
-                'Prometheus': prom_addr_br(name, v, self.args.port_gen),
+                'Prometheus': prom_addr_br(name, v, DEFAULT_BR_PROM_PORT),
             },
             'br': {
                 'Profile': False,

python/topology/prometheus.py

@@ -28,6 +28,12 @@
 from lib.util import write_file
 from topology.common import prom_addr_br, prom_addr_infra, ArgsTopoDicts
 
+PS_PROM_PORT = 30453
+BS_PROM_PORT = 30452
+CS_PROM_PORT = 30454
+SIG_PROM_PORT = 30456
+DEFAULT_BR_PROM_PORT = 30442
+
 
 class PrometheusGenArgs(ArgsTopoDicts):
     pass
@@ -59,10 +65,13 @@ def generate(self):
         for topo_id, as_topo in self.args.topo_dicts.items():
             ele_dict = defaultdict(list)
             for br_id, br_ele in as_topo["BorderRouters"].items():
-                ele_dict["BorderRouters"].append(prom_addr_br(br_id, br_ele, self.args.port_gen))
-            for svc_type in ["BeaconService", "PathService", "CertificateService"]:
-                for elem_id, elem in as_topo[svc_type].items():
-                    ele_dict[svc_type].append(prom_addr_infra(elem_id, elem, self.args.port_gen))
+                ele_dict["BorderRouters"].append(prom_addr_br(br_id, br_ele, DEFAULT_BR_PROM_PORT))
+            for elem_id, elem in as_topo["BeaconService"].items():
+                ele_dict["BeaconService"].append(prom_addr_infra(elem_id, elem, BS_PROM_PORT))
+            for elem_id, elem in as_topo["PathService"].items():
+                ele_dict["PathService"].append(prom_addr_infra(elem_id, elem, PS_PROM_PORT))
+            for elem_id, elem in as_topo["CertificateService"].items():
+                ele_dict["CertificateService"].append(prom_addr_infra(elem_id, elem, CS_PROM_PORT))
             config_dict[topo_id] = ele_dict
         self._write_config_files(config_dict)
 

python/topology/sig.py

@@ -29,6 +29,7 @@
     remote_nets,
     sciond_svc_name
 )
+from topology.prometheus import SIG_PROM_PORT
 
 
 class SIGGenArgs(ArgsBase):
@@ -156,6 +157,9 @@ def _sig_toml(self, topo_id, topo, base):
                 'console': {
                     'Level': 'error',
                 }
+            },
+            'metrics': {
+                'Prometheus': '0.0.0.0:%s' % SIG_PROM_PORT
             }
         }
         write_file(os.path.join(base, name, "sig.toml"), toml.dumps(sig_conf))