Replace host ZK with container (#2019)

.buildkite/clean_env.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 cntrs="$(docker ps -aq)"
-[ -n "$cntrs" ] && { echo "Remove leftover containers: $cntrs"; docker rm -f $cntrs; }
+[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
 
 echo "Remove leftover networks"
 docker network prune -f

README.md

@@ -64,12 +64,6 @@ Necessary steps in order to run SCION:
     ```
     This requires a go version >= 1.9.4 to be available on the system. If no sufficiently up-to-date version is available in the system repositories refer to [golang.org/dl](https://golang.org/dl) for installation instructions.
 
-1. Configure the host Zookeeper instance. At a minimum, add `maxClientCnxns=0`
-   to `/etc/zookeeper/conf/zoo.cfg`, but replacing it with `docker/zoo.cfg` is
-   recommended. This has the standard parameters set, as well as using a ram
-   disk for the data log, which greatly improves ZK performance (at the cost of
-   reliability, so it should only be done in a testing environment).
-
 1. Create the topology and configuration files (according to
    `topology/Default.topo`):
 

docker.sh

@@ -87,8 +87,6 @@ common_args() {
     args+=" -v $SCION_MOUNT/gen-certs:/home/scion/go/src/github.com/scionproto/scion/gen-certs"
     args+=" -v $SCION_MOUNT/gen-cache:/home/scion/go/src/github.com/scionproto/scion/gen-cache"
     args+=" -v $SCION_MOUNT/htmlcov:/home/scion/go/src/github.com/scionproto/scion/python/htmlcov"
-    args+=" -v /run/shm/dispatcher:/run/shm/dispatcher"
-    args+=" -v /run/shm/sciond:/run/shm/sciond"
     args+=" -e SCION_OUTPUT_BASE=$SCION_MOUNT"
     args+=" -e SCION_UID=$(id -u)"
     args+=" -e SCION_GID=$(id -g)"

docker/Dockerfile.base

@@ -90,9 +90,6 @@ RUN set -e; mkdir su-exec; \
 COPY docker/profile $HOME/.profile
 # Install basic screen config
 COPY docker/screenrc $HOME/.screenrc
-# Install ZK config
-COPY docker/zoo.cfg /etc/zookeeper/conf/
-RUN sudo sh -c "echo 'JAVA_OPTS=-Xmx150m # Added by Dockerfile.base' >> /etc/zookeeper/conf/environment"
 
 # Fix ownership one last time:
 RUN sudo chown -R scion: $HOME

env/debian/pkgs.txt

@@ -49,4 +49,3 @@ sqlite3
 sudo
 uthash-dev
 vim-common
-zookeeperd

python/topology/common.py

@@ -14,7 +14,8 @@
 
 # Stdlib
 import os
-
+import subprocess
+import sys
 # SCION
 from lib.packet.scion_addr import ISD_AS
 
@@ -82,17 +83,17 @@ def __repr__(self):
 
 def prom_addr_br(br_id, br_ele, port_gen):
     """Get the prometheus address for a border router"""
-    pub = _get_pub(br_ele['InternalAddrs'])
+    pub = get_pub(br_ele['InternalAddrs'])
     return "[%s]:%s" % (pub['PublicOverlay']['Addr'].ip, port_gen.register(br_id + "prom"))
 
 
 def prom_addr_infra(infra_id, infra_ele, port_gen):
     """Get the prometheus address for an infrastructure element."""
-    pub = _get_pub(infra_ele['Addrs'])
+    pub = get_pub(infra_ele['Addrs'])
     return "[%s]:%s" % (pub['Public']['Addr'].ip, port_gen.register(infra_id + "prom"))
 
 
-def _get_pub(topo_addr):
+def get_pub(topo_addr):
     pub = topo_addr.get('IPv6')
     if pub is not None:
         return pub
@@ -117,3 +118,21 @@ def docker_image(args, image):
     if args.image_tag:
         image = '%s:%s' % (image, args.image_tag)
     return image
+
+
+def docker_host(in_docker, docker, addr=None):
+    if in_docker:
+        # If in-docker we need to know the DOCKER0 IP
+        addr = os.getenv('DOCKER0', None)
+        if not addr:
+            print('DOCKER0 env variable required! Exiting!')
+            sys.exit(1)
+    elif docker or not addr:
+        # Using docker topology or there is no default addr,
+        # we directly get the DOCKER0 IP
+        addr = docker_ip()
+    return addr
+
+
+def docker_ip():
+    return subprocess.check_output(['tools/docker-ip']).decode("utf-8").strip()

python/topology/config.py

@@ -65,10 +65,10 @@
 from topology.prometheus import PrometheusGenArgs, PrometheusGenerator
 from topology.supervisor import SupervisorGenArgs, SupervisorGenerator
 from topology.topo import TopoGenArgs, TopoGenerator
+from topology.zk import ZKGenArgs, ZKGenerator
 
 DEFAULT_TOPOLOGY_FILE = "topology/Default.topo"
 DEFAULT_PATH_POLICY_FILE = "topology/PathPolicy.yml"
-DEFAULT_ZK_CONFIG = "topology/Zookeeper.yml"
 
 DEFAULT_CERTIFICATE_SERVER = "go"
 DEFAULT_SCIOND = "go"
@@ -93,7 +93,6 @@ def __init__(self, args):
         """
         self.args = args
         self.topo_config = load_yaml_file(self.args.topo_config)
-        self.zk_config = load_yaml_file(self.args.zk_config)
         self.default_mtu = None
         self._read_defaults(self.args.network)
         self.port_gen = PortGenerator()
@@ -115,8 +114,11 @@ def _read_defaults(self, network):
             priv_net = DEFAULT6_PRIV_NETWORK
         else:
             priv_net = DEFAULT_PRIV_NETWORK
-        self.subnet_gen = SubnetGenerator(def_network, self.args.docker)
-        self.prvnet_gen = SubnetGenerator(priv_net, self.args.docker)
+        self.subnet_gen = SubnetGenerator(def_network, self.args.docker, self.args.in_docker)
+        self.prvnet_gen = SubnetGenerator(priv_net, self.args.docker, self.args.in_docker)
+        if "zookeepers" not in defaults:
+            logging.critical("No zookeeper configured in the topology!")
+            sys.exit(1)
         self.default_mtu = defaults.get("mtu", DEFAULT_MTU)
 
     def generate_all(self):
@@ -154,6 +156,7 @@ def _generate_with_topo(self, topo_dicts):
             self._generate_docker(topo_dicts)
         else:
             self._generate_supervisor(topo_dicts)
+        self._generate_zk(topo_dicts)
         self._generate_prom_conf(topo_dicts)
 
     def _generate_cas(self):
@@ -188,7 +191,7 @@ def _generate_topology(self):
         return topo_gen.generate()
 
     def _topo_args(self):
-        return TopoGenArgs(self.args, self.topo_config, self.zk_config, self.subnet_gen,
+        return TopoGenArgs(self.args, self.topo_config, self.subnet_gen,
                            self.prvnet_gen, self.default_mtu, self.port_gen)
 
     def _generate_supervisor(self, topo_dicts):
@@ -207,6 +210,10 @@ def _generate_docker(self, topo_dicts):
     def _docker_args(self, topo_dicts):
         return DockerGenArgs(self.args, topo_dicts, self.networks, self.port_gen)
 
+    def _generate_zk(self, topo_dicts):
+        zk_gen = ZKGenerator(ZKGenArgs(self.args, topo_dicts))
+        zk_gen.generate()
+
     def _generate_prom_conf(self, topo_dicts):
         args = self._prometheus_args(topo_dicts)
         prom_gen = PrometheusGenerator(args)

python/topology/docker.py

@@ -15,7 +15,6 @@
 # Stdlib
 import copy
 import os
-from shutil import copyfile
 from string import Template
 # External packages
 import yaml
@@ -31,7 +30,6 @@
 from topology.docker_utils import DockerUtilsGenArgs, DockerUtilsGenerator
 
 DOCKER_CONF = 'scion-dc.yml'
-DEFAULT_DOCKER_NETWORK = "172.18.0.0/24"
 
 
 class DockerGenArgs(ArgsTopoDicts):
@@ -61,7 +59,6 @@ def __init__(self, args):
 
     def generate(self):
         self._create_networks()
-        self._zookeeper_conf()
         for topo_id, topo in self.args.topo_dicts.items():
             base = os.path.join(self.output_base, topo_id.base_dir(self.args.output_dir))
             self._gen_topo(topo_id, topo, base)
@@ -90,8 +87,6 @@ def _vol_conf(self, topo_id):
         self.dc_conf['volumes']['vol_%ssciond_%s' % (self.prefix, topo_id.file_fmt())] = None
 
     def _create_networks(self):
-        default_net = {'ipam': {'config': [{'subnet': DEFAULT_DOCKER_NETWORK}]}}
-        self.dc_conf['networks']['default'] = default_net
         for network in self.args.networks:
             for elem in self.args.networks[network]:
                 if elem not in self.elem_networks:
@@ -149,7 +144,6 @@ def _cs_conf(self, topo_id, topo, base):
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
-                'zookeeper'
             ],
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
@@ -176,7 +170,6 @@ def _bs_conf(self, topo_id, topo, base):
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
-                'zookeeper'
             ],
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
@@ -205,7 +198,6 @@ def _ps_conf(self, topo_id, topo, base):
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
-                'zookeeper'
             ],
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
@@ -227,32 +219,6 @@ def _ps_conf(self, topo_id, topo, base):
                 entry['command'].append('conf')
             self.dc_conf['services']['scion_%s' % k] = entry
 
-    def _zookeeper_conf(self):
-        cfg_file = 'docker/zoo-container.cfg'
-        entry = {
-            'image': 'zookeeper:latest',
-            'container_name': 'zookeeper',
-            'environment': {
-                'ZOO_USER': self.user_spec,
-                'ZOO_DATA_DIR': '/var/lib/zookeeper',
-                'ZOO_DATA_LOG_DIR': '/dev/shm/zookeeper'
-            },
-            'volumes': [
-                *self._usr_vol(),
-                os.path.join(
-                    self.output_base, self.args.output_dir, cfg_file) + ':/conf/zoo.cfg:rw',
-                '/var/lib/docker-zk:/var/lib/zookeeper:rw',
-                '/run/shm/docker-zk:/dev/shm/zookeeper:rw'
-            ],
-            'ports': [
-                '2181:2181'
-            ]
-        }
-        self.dc_conf['services']['zookeeper'] = entry
-        cfg_path = os.path.join(self.args.output_dir, cfg_file)
-        os.makedirs(os.path.dirname(cfg_path))
-        copyfile(os.path.join(os.environ['PWD'], cfg_file), cfg_path)
-
     def _dispatcher_conf(self, topo_id, topo, base):
         # Create dispatcher config
         entry = {

python/topology/generator.py

@@ -33,7 +33,6 @@
     DEFAULT_PATH_SERVER,
     DEFAULT_SCIOND,
     DEFAULT_TOPOLOGY_FILE,
-    DEFAULT_ZK_CONFIG,
     GENERATE_BIND_ADDRESS,
 )
 
@@ -51,8 +50,6 @@ def add_arguments(parser):
                         help='Network to create subnets in (E.g. "127.0.0.0/8"')
     parser.add_argument('-o', '--output-dir', default=GEN_PATH,
                         help='Output directory')
-    parser.add_argument('-z', '--zk-config', default=DEFAULT_ZK_CONFIG,
-                        help='Zookeeper configuration file')
     parser.add_argument('-b', '--bind-addr', default=GENERATE_BIND_ADDRESS,
                         help='Generate bind addresses (E.g. "192.168.0.0/16"')
     parser.add_argument('-t', '--trace', action='store_true',

python/topology/net.py

@@ -31,14 +31,19 @@
 
 DEFAULT_NETWORK = "127.0.0.0/8"
 DEFAULT_PRIV_NETWORK = "192.168.0.0/16"
-DEFAULT_SCN_DC_NETWORK = "172.20.0.0/16"
+DEFAULT_MININET_NETWORK = "100.64.0.0/10"
+DEFAULT_SCN_DC_NETWORK = "172.20.0.0/20"
+DEFAULT_SCN_IN_D_NETWORK = "172.20.16.0/20"
 
 
 class SubnetGenerator(object):
-    def __init__(self, network, docker):
+    def __init__(self, network, docker, in_docker):
         self.docker = docker
         if self.docker and network == DEFAULT_NETWORK:
-            network = DEFAULT_SCN_DC_NETWORK
+            if in_docker:
+                network = DEFAULT_SCN_IN_D_NETWORK
+            else:
+                network = DEFAULT_SCN_DC_NETWORK
         if "/" not in network:
             logging.critical("No prefix length specified for network '%s'",
                              network)