Only use zookeeper in docker

.buildkite/clean_env.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 cntrs="$(docker ps -aq)"
-[ -n "$cntrs" ] && { echo "Remove leftover containers: $cntrs"; docker rm -f "$cntrs"; }
+[ -n "$cntrs" ] && { echo "Remove leftover containers: $cntrs"; docker rm -f $cntrs; }
 
 echo "Remove leftover networks"
 docker network prune -f

README.md

@@ -64,12 +64,6 @@ Necessary steps in order to run SCION:
     ```
     This requires a go version >= 1.9.4 to be available on the system. If no sufficiently up-to-date version is available in the system repositories refer to [golang.org/dl](https://golang.org/dl) for installation instructions.
 
-1. Configure the host Zookeeper instance. At a minimum, add `maxClientCnxns=0`
-   to `/etc/zookeeper/conf/zoo.cfg`, but replacing it with `docker/zoo.cfg` is
-   recommended. This has the standard parameters set, as well as using a ram
-   disk for the data log, which greatly improves ZK performance (at the cost of
-   reliability, so it should only be done in a testing environment).
-
 1. Create the topology and configuration files (according to
    `topology/Default.topo`):
 

docker/Dockerfile.base

@@ -90,9 +90,6 @@ RUN set -e; mkdir su-exec; \
 COPY docker/profile $HOME/.profile
 # Install basic screen config
 COPY docker/screenrc $HOME/.screenrc
-# Install ZK config
-COPY docker/zoo.cfg /etc/zookeeper/conf/
-RUN sudo sh -c "echo 'JAVA_OPTS=-Xmx150m # Added by Dockerfile.base' >> /etc/zookeeper/conf/environment"
 
 # Fix ownership one last time:
 RUN sudo chown -R scion: $HOME

env/debian/pkgs.txt

@@ -49,4 +49,3 @@ sqlite3
 sudo
 uthash-dev
 vim-common
-zookeeperd

python/topology/common.py

@@ -14,6 +14,11 @@
 
 # Stdlib
 import os
+import subprocess
+import sys
+
+# External packages
+from external.ipaddress import ip_address
 
 # SCION
 from lib.packet.scion_addr import ISD_AS
@@ -28,6 +33,9 @@
     "DiscoveryService",
 )
 
+DEFAULT_DOCKER_ZK_PORT = 2182
+DEFAULT_ZK_PORT = 2181
+
 
 class ArgsBase:
     def __init__(self, args):
@@ -115,3 +123,30 @@ def _srv_iter(topo_dicts, out_dir, common=False):
                 yield topo_id, as_topo, os.path.join(base, elem)
         if common:
             yield topo_id, as_topo, os.path.join(base, COMMON_DIR)
+
+
+def _gen_zk_entry(addr, port, in_docker, docker):
+    if in_docker:
+        port = DEFAULT_DOCKER_ZK_PORT
+    if not port:
+        port = DEFAULT_ZK_PORT
+
+    if in_docker:
+        addr = os.getenv('DOCKER0', None)
+        if not addr:
+            print('DOCKER0 env variable required! Exiting!')
+            sys.exit(1)
+    elif docker:
+        addr = _docker_ip()
+    elif not addr:
+        addr = _docker_ip()
+    else:
+        addr = str(ip_address(addr))
+    return {
+        'Addr': addr,
+        'L4Port': port
+    }
+
+
+def _docker_ip():
+    return subprocess.check_output(['tools/docker-ip']).decode("utf-8").strip()

python/topology/config.py

@@ -66,10 +66,10 @@
 from topology.prometheus import PrometheusGenArgs, PrometheusGenerator
 from topology.supervisor import SupervisorGenArgs, SupervisorGenerator
 from topology.topo import TopoGenArgs, TopoGenerator
+from topology.zk import ZKGenArgs, ZKGenerator
 
 DEFAULT_TOPOLOGY_FILE = "topology/Default.topo"
 DEFAULT_PATH_POLICY_FILE = "topology/PathPolicy.yml"
-DEFAULT_ZK_CONFIG = "topology/Zookeeper.yml"
 
 DEFAULT_CERTIFICATE_SERVER = "go"
 DEFAULT_SCIOND = "go"
@@ -94,7 +94,6 @@ def __init__(self, args):
         """
         self.args = args
         self.topo_config = load_yaml_file(self.args.topo_config)
-        self.zk_config = load_yaml_file(self.args.zk_config)
         if self.args.docker and self.args.mininet:
             logging.critical("Cannot use mininet with docker!")
             sys.exit(1)
@@ -122,8 +121,8 @@ def _read_defaults(self, network):
             priv_net = DEFAULT6_PRIV_NETWORK
         else:
             priv_net = DEFAULT_PRIV_NETWORK
-        self.subnet_gen = SubnetGenerator(def_network, self.args.docker)
-        self.prvnet_gen = SubnetGenerator(priv_net, self.args.docker)
+        self.subnet_gen = SubnetGenerator(def_network, self.args.docker, self.args.in_docker)
+        self.prvnet_gen = SubnetGenerator(priv_net, self.args.docker, self.args.in_docker)
         for key, val in defaults.get("zookeepers", {}).items():
             if self.args.mininet and val['addr'] == "127.0.0.1":
                 val['addr'] = "169.254.0.1"
@@ -164,6 +163,7 @@ def _generate_with_topo(self, topo_dicts):
             self._generate_docker(topo_dicts)
         else:
             self._generate_supervisor(topo_dicts)
+        self._generate_zk()
         self._generate_prom_conf(topo_dicts)
 
     def _generate_cas(self):
@@ -198,7 +198,7 @@ def _generate_topology(self):
         return topo_gen.generate()
 
     def _topo_args(self):
-        return TopoGenArgs(self.args, self.topo_config, self.zk_config, self.subnet_gen,
+        return TopoGenArgs(self.args, self.topo_config, self.subnet_gen,
                            self.prvnet_gen, self.default_mtu, self.port_gen)
 
     def _generate_supervisor(self, topo_dicts):
@@ -217,6 +217,10 @@ def _generate_docker(self, topo_dicts):
     def _docker_args(self, topo_dicts):
         return DockerGenArgs(self.args, topo_dicts, self.networks, self.port_gen)
 
+    def _generate_zk(self):
+        zk_gen = ZKGenerator(ZKGenArgs(self.args, self.topo_config))
+        zk_gen.generate()
+
     def _generate_prom_conf(self, topo_dicts):
         args = self._prometheus_args(topo_dicts)
         prom_gen = PrometheusGenerator(args)

python/topology/docker.py

@@ -15,7 +15,6 @@
 # Stdlib
 import copy
 import os
-from shutil import copyfile
 from string import Template
 # External packages
 import yaml
@@ -31,7 +30,6 @@
 from topology.docker_utils import DockerUtilsGenArgs, DockerUtilsGenerator
 
 DOCKER_CONF = 'scion-dc.yml'
-DEFAULT_DOCKER_NETWORK = "172.18.0.0/24"
 
 
 class DockerGenArgs(ArgsTopoDicts):
@@ -61,7 +59,6 @@ def __init__(self, args):
 
     def generate(self):
         self._create_networks()
-        self._zookeeper_conf()
         for topo_id, topo in self.args.topo_dicts.items():
             base = os.path.join(self.output_base, topo_id.base_dir(self.args.output_dir))
             self._gen_topo(topo_id, topo, base)
@@ -90,8 +87,6 @@ def _vol_conf(self, topo_id):
         self.dc_conf['volumes']['vol_%ssciond_%s' % (self.prefix, topo_id.file_fmt())] = None
 
     def _create_networks(self):
-        default_net = {'ipam': {'config': [{'subnet': DEFAULT_DOCKER_NETWORK}]}}
-        self.dc_conf['networks']['default'] = default_net
         for network in self.args.networks:
             for elem in self.args.networks[network]:
                 if elem not in self.elem_networks:
@@ -149,7 +144,6 @@ def _cs_conf(self, topo_id, topo, base):
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
-                'zookeeper'
             ],
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
@@ -176,7 +170,6 @@ def _bs_conf(self, topo_id, topo, base):
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
-                'zookeeper'
             ],
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
@@ -205,7 +198,6 @@ def _ps_conf(self, topo_id, topo, base):
             'depends_on': [
                 self._sciond_name(topo_id),
                 'scion_disp_%s' % topo_id.file_fmt(),
-                'zookeeper'
             ],
             'environment': {
                 'SU_EXEC_USERSPEC': self.user_spec,
@@ -227,32 +219,6 @@ def _ps_conf(self, topo_id, topo, base):
                 entry['command'].append('conf')
             self.dc_conf['services']['scion_%s' % k] = entry
 
-    def _zookeeper_conf(self):
-        cfg_file = 'docker/zoo-container.cfg'
-        entry = {
-            'image': 'zookeeper:latest',
-            'container_name': 'zookeeper',
-            'environment': {
-                'ZOO_USER': self.user_spec,
-                'ZOO_DATA_DIR': '/var/lib/zookeeper',
-                'ZOO_DATA_LOG_DIR': '/dev/shm/zookeeper'
-            },
-            'volumes': [
-                *self._usr_vol(),
-                os.path.join(
-                    self.output_base, self.args.output_dir, cfg_file) + ':/conf/zoo.cfg:rw',
-                '/var/lib/docker-zk:/var/lib/zookeeper:rw',
-                '/run/shm/docker-zk:/dev/shm/zookeeper:rw'
-            ],
-            'ports': [
-                '2181:2181'
-            ]
-        }
-        self.dc_conf['services']['zookeeper'] = entry
-        cfg_path = os.path.join(self.args.output_dir, cfg_file)
-        os.makedirs(os.path.dirname(cfg_path))
-        copyfile(os.path.join(os.environ['PWD'], cfg_file), cfg_path)
-
     def _dispatcher_conf(self, topo_id, topo, base):
         # Create dispatcher config
         entry = {

python/topology/generator.py

@@ -33,7 +33,6 @@
     DEFAULT_PATH_SERVER,
     DEFAULT_SCIOND,
     DEFAULT_TOPOLOGY_FILE,
-    DEFAULT_ZK_CONFIG,
     GENERATE_BIND_ADDRESS,
 )
 
@@ -53,8 +52,6 @@ def add_arguments(parser):
                         help='Network to create subnets in (E.g. "127.0.0.0/8"')
     parser.add_argument('-o', '--output-dir', default=GEN_PATH,
                         help='Output directory')
-    parser.add_argument('-z', '--zk-config', default=DEFAULT_ZK_CONFIG,
-                        help='Zookeeper configuration file')
     parser.add_argument('-b', '--bind-addr', default=GENERATE_BIND_ADDRESS,
                         help='Generate bind addresses (E.g. "192.168.0.0/16"')
     parser.add_argument('-t', '--trace', action='store_true',

python/topology/net.py

@@ -32,14 +32,18 @@
 DEFAULT_NETWORK = "127.0.0.0/8"
 DEFAULT_PRIV_NETWORK = "192.168.0.0/16"
 DEFAULT_MININET_NETWORK = "100.64.0.0/10"
-DEFAULT_SCN_DC_NETWORK = "172.20.0.0/16"
+DEFAULT_SCN_DC_NETWORK = "172.40.0.0/16"
+DEFAULT_SCN_IN_D_NETWORK = "172.50.0.0/16"
 
 
 class SubnetGenerator(object):
-    def __init__(self, network, docker):
+    def __init__(self, network, docker, in_docker):
         self.docker = docker
         if self.docker and network == DEFAULT_NETWORK:
-            network = DEFAULT_SCN_DC_NETWORK
+            if in_docker:
+                network = DEFAULT_SCN_IN_D_NETWORK
+            else:
+                network = DEFAULT_SCN_DC_NETWORK
         if "/" not in network:
             logging.critical("No prefix length specified for network '%s'",
                              network)

python/topology/topo.py

@@ -21,11 +21,11 @@
 import logging
 import os
 import random
+import sys
 from collections import defaultdict
 
 # External packages
 import yaml
-from external.ipaddress import ip_address
 
 # SCION
 from lib.defines import (
@@ -40,7 +40,7 @@
 from lib.topology import Topology
 from lib.types import LinkType
 from lib.util import write_file
-from topology.common import _srv_iter, ArgsBase, TopoID, SCION_SERVICE_NAMES
+from topology.common import _srv_iter, ArgsBase, TopoID, SCION_SERVICE_NAMES, _gen_zk_entry
 from topology.net import AddressProxy
 
 DEFAULT_LINK_BW = 1000
@@ -55,20 +55,17 @@
 
 
 class TopoGenArgs(ArgsBase):
-    def __init__(self, args, topo_config, zk_config, subnet_gen,
-                 privnet_gen, default_mtu, port_gen):
+    def __init__(self, args, topo_config, subnet_gen, privnet_gen, default_mtu, port_gen):
         """
         :param ArgsBase args: Contains the passed command line arguments.
         :param dict topo_config: The parsed topology config.
-        :param dict zk_config: The parsed zookeeper config.
         :param SubnetGenerator subnet_gen: The default network generator.
         :param SubnetGenerator privnet_gen: The private network generator.
         :param dict default_mtu: The default mtu.
         :param PortGenerator port_gen: The port generator
         """
         super().__init__(args)
         self.topo_config_dict = topo_config
-        self.zk_config_dict = zk_config
         self.subnet_gen = subnet_gen
         self.privnet_gen = privnet_gen
         self.default_mtu = default_mtu
@@ -288,18 +285,14 @@ def _gen_zk_entries(self, topo_id, as_conf):
         zk_conf = {}
         if "zookeepers" in self.args.topo_config_dict.get("defaults", {}):
             zk_conf = self.args.topo_config_dict["defaults"]["zookeepers"]
-        if self.args.docker:
-            zk_conf[1] = {'addr': ZOOKEEPER_ADDR}
+        if len(zk_conf) > 1:
+            logging.critical("Only one zk instance is allowed!")
+            sys.exit(1)
         for key, val in zk_conf.items():
-            self._gen_zk_entry(topo_id, key, val)
-
-    def _gen_zk_entry(self, topo_id, zk_id, zk_conf):
-        zk = ZKTopo(zk_conf, self.args.zk_config_dict)
-        addr = str(zk.addr)
-        self.topo_dicts[topo_id]["ZookeeperService"][zk_id] = {
-            'Addr': addr,
-            'L4Port': zk.clientPort
-        }
+            addr = val.get("addr", None)
+            port = val.get("port", None)
+            zk_entry = _gen_zk_entry(addr, port, self.args.in_docker, self.args.docker)
+            self.topo_dicts[topo_id]["ZookeeperService"][key] = zk_entry
 
     def _generate_as_list(self, topo_id, as_conf):
         if as_conf.get('core', False):
@@ -353,18 +346,6 @@ def br_name(self):
         return None
 
 
-class ZKTopo(object):
-    def __init__(self, topo_config, zk_config):
-        self.addr = None
-        self.topo_config = topo_config
-        self.zk_config = zk_config
-        self.addr = ip_address(self.topo_config["addr"])
-        self.clientPort = self._get_def("clientPort")
-
-    def _get_def(self, key):
-        return self.topo_config.get(key, self.zk_config["Default"][key])
-
-
 class IFIDGenerator(object):
     """Generates unique interface IDs"""
     def __init__(self):