salt, kubernetes: remove apiServer from BootstrapConfiguration

We no longer need this since we provide in-cluster HA for `kube-apiserver` access. If this is desired for out-of-cluster access, we can provide this using a `LoadBalancer` `Service` once we have the infrastructure to support this in place. This also removed the optional deployment of `keepalived`. See: #2103 See: #1788
scality · Dec 12, 2019 · b6acef7 · b6acef7
1 parent 84882ad
commit b6acef7
Show file tree

Hide file tree

Showing 16 changed files with 17 additions and 296 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,15 @@
 
 ## Release 2.4.2 (in development)
 
+### Breaking changes
+- If `apiServer.host` is configured in `BootstrapConfiguration`, this is no
+longer used (and must no longer be defined).
+- If `apiServer.keepalived` is configured in `BootstrapConfiguration`, this is
+no longer used, and Keepalived is no longer deployed at all.
+- Generated `admin.conf` `KubeConfig` files point to the control-plane IP of the
+host on which they are generated. You can override this when using them using
+`kubectl`s `-s`/`--server` argument to point to another address.
+
 ### Features added
 
 - [#2103](https://github.com/scality/metalk8s/issues/2103) - Add a host-local

diff --git a/Vagrantfile b/Vagrantfile
@@ -36,9 +36,7 @@ CONTROL_PLANE_NETWORK = {
   # and
   # https://github.com/hashicorp/vagrant/blob/1e1c398de565ed0aab9631cfad2db6e1dac82d7f/plugins/providers/virtualbox/action/network.rb#L317
   # or https://github.com/hashicorp/vagrant/pull/7699
-  # Also, we leave off the 'last' IP in the control-plane network, which is
-  # reserved as the VIP of the API server.
-  :dhcp_upper => IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(3).first.to_s,
+  :dhcp_upper => IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(2).first.to_s,
 }
 
 # Bigger network for the workload plane. However, we only allow DHCP to allocate
@@ -136,10 +134,6 @@ networks:
   workloadPlane: #{WORKLOAD_PLANE_IP}/#{prefixlen(WORKLOAD_PLANE_NETMASK)}
 ca:
   minion: bootstrap
-apiServer:
-  host: #{IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(2).first.to_s}
-  keepalived:
-    enabled: true
 archives:
   - /srv/scality/metalk8s-$VERSION
 EOF

diff --git a/buildchain/buildchain/image.py b/buildchain/buildchain/image.py
@@ -218,21 +218,6 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage:
         name='salt-master',
         build_args={'SALT_VERSION': versions.SALT_VERSION},
     ),
-    _local_image(
-        name='keepalived',
-        build_args={
-            'KEEPALIVED_IMAGE': versions.CENTOS_BASE_IMAGE,
-            'KEEPALIVED_IMAGE_SHA256': versions.CENTOS_BASE_IMAGE_SHA256,
-            'KEEPALIVED_VERSION': versions.KEEPALIVED_VERSION,
-            'BUILD_DATE': datetime.datetime.now(datetime.timezone.utc)
-                            .astimezone()
-                            .isoformat(),
-            'VCS_REF': constants.GIT_REF or '<unknown>',
-            'VERSION': versions.CONTAINER_IMAGES_MAP['keepalived'].version,
-            'METALK8S_VERSION': versions.VERSION,
-        },
-        file_dep=[constants.ROOT/'images'/'keepalived'/'entrypoint.sh'],
-    ),
     _local_image(
         name='metalk8s-ui',
         dockerfile=constants.ROOT/'ui'/'Dockerfile',

diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py
@@ -19,7 +19,6 @@
 
 CALICO_VERSION     : str = '3.8.2'
 K8S_VERSION        : str = '1.15.5'
-KEEPALIVED_VERSION : str = '1.3.5-16.el7'
 SALT_VERSION       : str = '2018.3.4'
 
 def load_version_information() -> None:
@@ -67,7 +66,6 @@ def load_version_information() -> None:
 # This should be reset to 1 when the service exposed by the container changes
 # version.
 SALT_MASTER_BUILD_ID = 1
-KEEPALIVED_BUILD_ID  = 1
 
 
 def _version_prefix(version: str, prefix: str = 'v') -> str:
@@ -184,13 +182,6 @@ def _version_prefix(version: str, prefix: str = 'v') -> str:
         digest='sha256:ed3ec0597c2d5b7102a7f62c661a23d8e4b34d910693fc23fd40bfb1d9404dcf',
     ),
     # Local images
-    Image(
-        name='keepalived',
-        version='{version}-{build_id}'.format(
-            version=KEEPALIVED_VERSION, build_id=KEEPALIVED_BUILD_ID
-        ),
-        digest=None,
-    ),
     Image(
         name='metalk8s-ui',
         version=VERSION,

diff --git a/docs/developer/architecture/deployment.rst b/docs/developer/architecture/deployment.rst
@@ -57,19 +57,6 @@ which can vary from one installation to another:
 
   Default: ``10.96.0.0/12``
 
-- VIP for the ``kube-apiserver`` and ``keepalived`` toggle
-
-  Used as the address of ``kube-apiserver`` where required. This can either be
-  a VIP managed by custom load-balancing/high-availability infrastructure, in
-  which case the ``keepalived`` toggle must be off, or one which our platform
-  will manage using ``keepalived``.
-
-  If ``keepalived`` is enabled, this VIP must sit in a control plane CIDR
-  shared by all control plane nodes.
-
-  Note: we run ``keepalived`` in unicast mode, which is an extension of classic
-  VRRP, but removes the need for multicast support on the network.
-
 Firewall
 ^^^^^^^^
 
@@ -81,7 +68,6 @@ We assume SSH access is not blocked by the host-based firewall.
 
 These services include:
 
-- VRRP if ``keepalived`` is enabled on control-plane nodes
 - HTTPS on the bootstrap node, for ``nginx`` fronting the OCI registry and
   serving the yum repository
 - ``salt-master`` on the bootstrap node

diff --git a/docs/developer/running/ui.rst b/docs/developer/running/ui.rst
@@ -30,8 +30,12 @@ Procedure
        'salt-call', 'pillar.get', 'metalk8s', '--out', 'json'
    ])
    pillar = json.loads(output)['local']
+   output = subprocess.check_output([
+       'salt-call', 'grains.get', 'metalk8s:control_plane_ip', '--out', 'json'
+   ])
+   control_plane_ip = json.loads(output)['local']
    ui_conf = {
-       'url': 'https://{}:6443'.format(pillar['api_server']['host']),
+       'url': 'https://{}:6443'.format(control_plane_ip),
        'url_salt': 'https://{salt[ip]}:{salt[ports][api]}'.format(
            salt=pillar['endpoints']['salt-master']
        ),

diff --git a/docs/quickstart/bootstrap.rst b/docs/quickstart/bootstrap.rst
@@ -38,8 +38,6 @@ Configuration
         workloadPlane: <CIDR-notation>
       ca:
         minion: <hostname-of-the-bootstrap-node>
-      apiServer:
-        host: <IP-of-the-bootstrap-node>
       archives:
         - <path-to-metalk8s-iso>
 
@@ -50,7 +48,6 @@ system is configured to re-mount them automatically after a reboot.
 .. todo::
 
    - Explain the role of this config file and its values
-   - Add a note about setting HA for ``apiServer``
 
 
 .. _quickstart-bootstrap-ssh:

diff --git a/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh b/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh
@@ -17,8 +17,6 @@ networks:
   workloadPlane: 10.100.0.0/16
 ca:
   minion: $(cat /etc/salt/minion_id)
-apiServer:
-  host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }')
 archives:
   - /var/tmp/metalk8s
 EOF

diff --git a/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh b/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh
@@ -17,8 +17,6 @@ networks:
   workloadPlane: 10.100.0.0/16
 ca:
   minion: $(cat /etc/salt/minion_id)
-apiServer:
-  host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }')
 archives:
   - /var/tmp/metalk8s
 EOF

diff --git a/images/keepalived/Dockerfile b/images/keepalived/Dockerfile
diff --git a/images/keepalived/entrypoint.sh b/images/keepalived/entrypoint.sh
diff --git a/salt/_pillar/metalk8s.py b/salt/_pillar/metalk8s.py
@@ -70,39 +70,6 @@ def _load_ca(config_data):
     }
 
 
-def _load_apiserver(config_data):
-    errors = __utils__['pillar_utils.assert_keys'](config_data, ['apiServer'])
-    if errors:
-        return __utils__['pillar_utils.errors_to_dict'](errors)
-
-    as_data = config_data['apiServer']
-
-    result = {
-        'host': None,
-        'keepalived': {
-            'enabled': False,
-            'virtualRouterId': 1,
-            'authPassword': 'MeTaLk8s',
-        },
-        'kubeconfig': '/etc/kubernetes/admin.conf'
-    }
-
-    errors = __utils__['pillar_utils.assert_keys'](as_data, ['host'])
-    if errors:
-        return __utils__['pillar_utils.errors_to_dict'](errors)
-
-    result['host'] = as_data['host']
-
-    if 'keepalived' in as_data:
-        k_data = as_data['keepalived']
-        k_result = result['keepalived']
-
-        for (key, default) in k_result.items():
-            k_result[key] = k_data.get(key, default)
-
-    return result
-
-
 def _load_iso_path(config_data):
     """Load iso path from BootstrapConfiguration
 
@@ -133,7 +100,6 @@ def ext_pillar(minion_id, pillar, bootstrap_config):
         metal_data = {
             'archives': _load_iso_path(config),
             'ca': _load_ca(config),
-            'api_server': _load_apiserver(config)
         }
 
     result = {
@@ -144,7 +110,7 @@ def ext_pillar(minion_id, pillar, bootstrap_config):
     if not isinstance(metal_data['archives'], list):
         # Special case for archives in pillar
         __utils__['pillar_utils.promote_errors'](metal_data, 'archives')
-    for key in ['ca', 'api_server']:
+    for key in ['ca',]:
         __utils__['pillar_utils.promote_errors'](metal_data, key)
     for key in ['networks', 'metalk8s']:
         __utils__['pillar_utils.promote_errors'](result, key)

diff --git a/salt/metalk8s/kubernetes/apiserver/certs/server.sls b/salt/metalk8s/kubernetes/apiserver/certs/server.sls
@@ -26,7 +26,6 @@ Create kube-apiserver private key:
     'kubernetes.default.svc.cluster.local',
     kubernetes_service_ip,
     grains['metalk8s']['control_plane_ip'],
-    pillar['metalk8s']['api_server']['host'],
     '127.0.0.1',
 ]
 %}