From 7c93b7cf052245aef80ab39ac106be53f681406e Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Fri, 6 Dec 2019 21:05:18 +0100 Subject: [PATCH] salt, kubernetes: remove `apiServer` from `BootstrapConfiguration` We no longer need this since we provide in-cluster HA for `kube-apiserver` access. If this is desired for out-of-cluster access, we can provide this using a `LoadBalancer` `Service` once we have the infrastructure to support this in place. This also removed the optional deployment of `keepalived`. See: https://github.com/scality/metalk8s/issues/2103 See: https://github.com/scality/metalk8s/issues/1788 --- Vagrantfile | 4 - buildchain/buildchain/image.py | 15 --- buildchain/buildchain/versions.py | 9 -- docs/developer/architecture/deployment.rst | 14 --- docs/developer/architecture/requirements.rst | 3 - docs/developer/running/ui.rst | 6 +- docs/quickstart/bootstrap.rst | 3 - eve/main.yml | 2 - .../terraform/scripts/bootstrap-config.sh | 2 - .../terraform/scripts/bootstrap-config.sh | 2 - images/keepalived/Dockerfile | 68 ---------- images/keepalived/entrypoint.sh | 14 --- salt/_pillar/metalk8s.py | 36 +----- .../kubernetes/apiserver/certs/server.sls | 1 - .../kubernetes/apiserver/installed.sls | 118 ------------------ salt/metalk8s/node/grains.sls | 3 - 16 files changed, 6 insertions(+), 294 deletions(-) delete mode 100644 images/keepalived/Dockerfile delete mode 100755 images/keepalived/entrypoint.sh diff --git a/Vagrantfile b/Vagrantfile index 6257310f36..12ccc6e3e0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -136,10 +136,6 @@ networks: workloadPlane: #{WORKLOAD_PLANE_IP}/#{prefixlen(WORKLOAD_PLANE_NETMASK)} ca: minion: bootstrap -apiServer: - host: #{IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(2).first.to_s} - keepalived: - enabled: true archives: - /srv/scality/metalk8s-$VERSION EOF diff --git a/buildchain/buildchain/image.py b/buildchain/buildchain/image.py index 0e50e02c69..2daa113918 100644 --- a/buildchain/buildchain/image.py +++ b/buildchain/buildchain/image.py @@ -222,21 +222,6 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage: name='salt-master', build_args={'SALT_VERSION': versions.SALT_VERSION}, ), - _local_image( - name='keepalived', - build_args={ - 'KEEPALIVED_IMAGE': versions.CENTOS_BASE_IMAGE, - 'KEEPALIVED_IMAGE_SHA256': versions.CENTOS_BASE_IMAGE_SHA256, - 'KEEPALIVED_VERSION': versions.KEEPALIVED_VERSION, - 'BUILD_DATE': datetime.datetime.now(datetime.timezone.utc) - .astimezone() - .isoformat(), - 'VCS_REF': constants.GIT_REF or '', - 'VERSION': versions.CONTAINER_IMAGES_MAP['keepalived'].version, - 'METALK8S_VERSION': versions.VERSION, - }, - file_dep=[constants.ROOT/'images'/'keepalived'/'entrypoint.sh'], - ), _local_image( name='metalk8s-ui', dockerfile=constants.ROOT/'ui'/'Dockerfile', diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py index b41134ddbb..8cd5844cd3 100644 --- a/buildchain/buildchain/versions.py +++ b/buildchain/buildchain/versions.py @@ -19,7 +19,6 @@ CALICO_VERSION : str = '3.8.2' K8S_VERSION : str = '1.15.5' -KEEPALIVED_VERSION : str = '1.3.5-16.el7' SALT_VERSION : str = '2018.3.4' def load_version_information() -> None: @@ -67,7 +66,6 @@ def load_version_information() -> None: # This should be reset to 1 when the service exposed by the container changes # version. SALT_MASTER_BUILD_ID = 1 -KEEPALIVED_BUILD_ID = 1 def _version_prefix(version: str, prefix: str = 'v') -> str: @@ -184,13 +182,6 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: digest='sha256:ed3ec0597c2d5b7102a7f62c661a23d8e4b34d910693fc23fd40bfb1d9404dcf', ), # Local images - Image( - name='keepalived', - version='{version}-{build_id}'.format( - version=KEEPALIVED_VERSION, build_id=KEEPALIVED_BUILD_ID - ), - digest=None, - ), Image( name='metalk8s-ui', version=VERSION, diff --git a/docs/developer/architecture/deployment.rst b/docs/developer/architecture/deployment.rst index 008effd683..c8dd4ab724 100644 --- a/docs/developer/architecture/deployment.rst +++ b/docs/developer/architecture/deployment.rst @@ -57,19 +57,6 @@ which can vary from one installation to another: Default: ``10.96.0.0/12`` -- VIP for the ``kube-apiserver`` and ``keepalived`` toggle - - Used as the address of ``kube-apiserver`` where required. This can either be - a VIP managed by custom load-balancing/high-availability infrastructure, in - which case the ``keepalived`` toggle must be off, or one which our platform - will manage using ``keepalived``. - - If ``keepalived`` is enabled, this VIP must sit in a control plane CIDR - shared by all control plane nodes. - - Note: we run ``keepalived`` in unicast mode, which is an extension of classic - VRRP, but removes the need for multicast support on the network. - Firewall ^^^^^^^^ @@ -81,7 +68,6 @@ We assume SSH access is not blocked by the host-based firewall. These services include: -- VRRP if ``keepalived`` is enabled on control-plane nodes - HTTPS on the bootstrap node, for ``nginx`` fronting the OCI registry and serving the yum repository - ``salt-master`` on the bootstrap node diff --git a/docs/developer/architecture/requirements.rst b/docs/developer/architecture/requirements.rst index 6647770354..14dd447338 100644 --- a/docs/developer/architecture/requirements.rst +++ b/docs/developer/architecture/requirements.rst @@ -204,9 +204,6 @@ discovered through an updated DNS entry), it must be possible to reconfigure the deployment accordingly, with as little impact as possible (i.e., requiring as little changes as possible). This related to the `DNS` section above. -For some services, e.g. `keepalived` configuration, IP addresses are mandatory, -so these are permitted. - Multi-Homed Servers ------------------- A deployment can specify subnet CIDRs for various purposes, e.g. control-plane, diff --git a/docs/developer/running/ui.rst b/docs/developer/running/ui.rst index 1628c0df3e..36a9da413e 100644 --- a/docs/developer/running/ui.rst +++ b/docs/developer/running/ui.rst @@ -30,8 +30,12 @@ Procedure 'salt-call', 'pillar.get', 'metalk8s', '--out', 'json' ]) pillar = json.loads(output)['local'] + output = subprocess.check_output([ + 'salt-call', 'grains.get', 'metalk8s:control_plane_ip', '--out', 'json' + ]) + control_plane_ip = json.loads(output)['local'] ui_conf = { - 'url': 'https://{}:6443'.format(pillar['api_server']['host']), + 'url': 'https://{}:6443'.format(control_plane_ip), 'url_salt': 'https://{salt[ip]}:{salt[ports][api]}'.format( salt=pillar['endpoints']['salt-master'] ), diff --git a/docs/quickstart/bootstrap.rst b/docs/quickstart/bootstrap.rst index b574a64b45..280d36f6d4 100644 --- a/docs/quickstart/bootstrap.rst +++ b/docs/quickstart/bootstrap.rst @@ -38,8 +38,6 @@ Configuration workloadPlane: ca: minion: - apiServer: - host: archives: - @@ -50,7 +48,6 @@ system is configured to re-mount them automatically after a reboot. .. todo:: - Explain the role of this config file and its values - - Add a note about setting HA for ``apiServer`` .. _quickstart-bootstrap-ssh: diff --git a/eve/main.yml b/eve/main.yml index f1082f7cea..d26d85b51b 100644 --- a/eve/main.yml +++ b/eve/main.yml @@ -213,8 +213,6 @@ models: workloadPlane: 10.100.0.0/16 ca: minion: $(hostname) - apiServer: - host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }') archives: - "/srv/scality/metalk8s-${PRODUCT_VERSION}" END diff --git a/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh b/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh index 1a2eb86801..8365951580 100644 --- a/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh +++ b/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh @@ -17,8 +17,6 @@ networks: workloadPlane: 10.100.0.0/16 ca: minion: $(cat /etc/salt/minion_id) -apiServer: - host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }') archives: - /var/tmp/metalk8s EOF diff --git a/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh b/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh index 1a2eb86801..8365951580 100644 --- a/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh +++ b/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh @@ -17,8 +17,6 @@ networks: workloadPlane: 10.100.0.0/16 ca: minion: $(cat /etc/salt/minion_id) -apiServer: - host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }') archives: - /var/tmp/metalk8s EOF diff --git a/images/keepalived/Dockerfile b/images/keepalived/Dockerfile deleted file mode 100644 index 732fabd740..0000000000 --- a/images/keepalived/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# SHA256 digest of the base image -ARG KEEPALIVED_IMAGE_SHA256 -ARG KEEPALIVED_IMAGE=docker.io/centos - -FROM $KEEPALIVED_IMAGE@sha256:$KEEPALIVED_IMAGE_SHA256 - -ARG KEEPALIVED_VERSION - -# Timestamp of the build, formatted as RFC3339 -ARG BUILD_DATE -# Git revision o the tree at build time -ARG VCS_REF -# Version of the image -ARG VERSION -# Version of the project, e.g. `git describe --always --long --dirty --broken` -ARG METALK8S_VERSION - -# Create user early on, since this can be a shared layer for many versions of this image -RUN useradd \ - --user-group \ - --no-create-home \ - --system \ - keepalived_script - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["--dont-fork", "--dont-respawn", "--dump-conf", "--address-monitoring", "--log-console", "--log-detail"] - -# The entrypoint script is less likely to change than KEEPALIVED_VERSION -COPY entrypoint.sh /entrypoint.sh - -# These contain BUILD_DATE so should come 'late' for layer caching -LABEL maintainer="moonshot-platform@scality.com" \ - # http://label-schema.org/rc1/ - org.label-schema.build-date="$BUILD_DATE" \ - org.label-schema.name="keepalived" \ - org.label-schema.description="Load balancer and high availability service, part of MetalK8s" \ - org.label-schema.license="GPLv2+" \ - org.label-schema.url="https://github.com/scality/metalk8s/" \ - org.label-schema.vcs-url="https://github.com/scality/metalk8s.git" \ - org.label-schema.vcs-ref="$VCS_REF" \ - org.label-schema.vendor="Scality" \ - org.label-schema.version="$VERSION" \ - org.label-schema.schema-version="1.0" \ - org.label-schema.docker.cmd="docker run --net host -v /keepalived.conf:/etc/keepalived/keepalived.conf:ro keepalived:$VERSION" \ - # https://github.com/opencontainers/image-spec/blob/master/annotations.md - org.opencontainers.image.created="$BUILD_DATE" \ - org.opencontainers.image.authors="moonshot-platform@scality.com" \ - org.opencontainers.image.url="https://github.com/scality/metalk8s/" \ - org.opencontainers.image.source="https://github.com/scality/metalk8s.git" \ - org.opencontainers.image.version="$VERSION" \ - org.opencontainers.image.revision="$VCS_REF" \ - org.opencontainers.image.vendor="Scality" \ - org.opencontainers.image.title="keepalived" \ - org.opencontainers.image.description="Load balancer and high availability service, part of MetalK8s" \ - # https://docs.openshift.org/latest/creating_images/metadata.html - io.openshift.tags="metalk8s,keepalived,ha" \ - io.k8s.description="Load balancer and high availability service, part of MetalK8s" \ - io.openshift.non-scalable="true" \ - # Various - com.scality.metalk8s.keepalived.version="$KEEPALIVED_VERSION" \ - com.scality.metalk8s.version="$METALK8S_VERSION" - -# Final layer installing keepalived-$KEEPALIVED_VERSION -RUN yum install -y \ - iproute \ - keepalived-${KEEPALIVED_VERSION} \ - && \ - yum clean all diff --git a/images/keepalived/entrypoint.sh b/images/keepalived/entrypoint.sh deleted file mode 100755 index 95eee93a28..0000000000 --- a/images/keepalived/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -xue -o pipefail - -if test -x /etc/keepalived/keepalived.conf.sh; then - KEEPALIVED_CONF=/var/run/keepalived.conf - /etc/keepalived/keepalived.conf.sh "${KEEPALIVED_CONF}" -else - KEEPALIVED_CONF=/etc/keepalived/keepalived.conf -fi - -test -f "${KEEPALIVED_CONF}" - -exec keepalived --use-file "${KEEPALIVED_CONF}" "$@" diff --git a/salt/_pillar/metalk8s.py b/salt/_pillar/metalk8s.py index 7fd0267a93..b99a2dd7d2 100644 --- a/salt/_pillar/metalk8s.py +++ b/salt/_pillar/metalk8s.py @@ -70,39 +70,6 @@ def _load_ca(config_data): } -def _load_apiserver(config_data): - errors = __utils__['pillar_utils.assert_keys'](config_data, ['apiServer']) - if errors: - return __utils__['pillar_utils.errors_to_dict'](errors) - - as_data = config_data['apiServer'] - - result = { - 'host': None, - 'keepalived': { - 'enabled': False, - 'virtualRouterId': 1, - 'authPassword': 'MeTaLk8s', - }, - 'kubeconfig': '/etc/kubernetes/admin.conf' - } - - errors = __utils__['pillar_utils.assert_keys'](as_data, ['host']) - if errors: - return __utils__['pillar_utils.errors_to_dict'](errors) - - result['host'] = as_data['host'] - - if 'keepalived' in as_data: - k_data = as_data['keepalived'] - k_result = result['keepalived'] - - for (key, default) in k_result.items(): - k_result[key] = k_data.get(key, default) - - return result - - def _load_iso_path(config_data): """Load iso path from BootstrapConfiguration @@ -133,7 +100,6 @@ def ext_pillar(minion_id, pillar, bootstrap_config): metal_data = { 'archives': _load_iso_path(config), 'ca': _load_ca(config), - 'api_server': _load_apiserver(config) } result = { @@ -144,7 +110,7 @@ def ext_pillar(minion_id, pillar, bootstrap_config): if not isinstance(metal_data['archives'], list): # Special case for archives in pillar __utils__['pillar_utils.promote_errors'](metal_data, 'archives') - for key in ['ca', 'api_server']: + for key in ['ca',]: __utils__['pillar_utils.promote_errors'](metal_data, key) for key in ['networks', 'metalk8s']: __utils__['pillar_utils.promote_errors'](result, key) diff --git a/salt/metalk8s/kubernetes/apiserver/certs/server.sls b/salt/metalk8s/kubernetes/apiserver/certs/server.sls index 90f2cbc017..56a1453e79 100644 --- a/salt/metalk8s/kubernetes/apiserver/certs/server.sls +++ b/salt/metalk8s/kubernetes/apiserver/certs/server.sls @@ -26,7 +26,6 @@ Create kube-apiserver private key: 'kubernetes.default.svc.cluster.local', kubernetes_service_ip, grains['metalk8s']['control_plane_ip'], - pillar['metalk8s']['api_server']['host'], '127.0.0.1', ] %} diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index be3eec4fba..6a85a6045e 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -9,62 +9,6 @@ include: - metalk8s.kubernetes.sa.advertised - .certs -{%- if pillar.metalk8s.api_server.keepalived.enabled %} -Create keepalived check script: - file.managed: - - name: /etc/keepalived/check-apiserver.sh - - mode: 0555 - - makedirs: true - - dir_mode: 0755 - - contents: | - #!/bin/bash - set -ue -o pipefail - test $(curl -k https://127.0.0.1:6443/healthz) = 'ok' - -Create keepalived configuration file generator: - file.managed: - - name: /etc/keepalived/keepalived.conf.sh - - mode: 0555 - - makedirs: true - - dir_mode: 0755 - - contents: | - #!/bin/bash - set -xue -o pipefail - - IP=${IP:-ip} - AWK=${AWK:-awk} - - INTERFACE=${INTERFACE:-$(${IP} route get ${INTERFACE_ADDRESS} | ${AWK} '/dev / { print $4 }')} - - cat > "$1" << EOF - global_defs { - enable_script_security - } - - vrrp_script check_apiserver { - script "/etc/keepalived/check-apiserver.sh" - interval 2 - weight 2 - } - - vrrp_instance VI_1 { - state ${VRRP_STATE:-BACKUP} - virtual_router_id ${VRRP_VIRTUAL_ROUTER_ID:-1} - interface ${INTERFACE} - authentication { - auth_type PASS - auth_pass ${VRRP_PASSWORD} - } - virtual_ipaddress { - ${VIP} - } - track_script { - check_apiserver - } - } - EOF -{%- endif %} - Set up default basic auth htpasswd: file.managed: - name: {{ htpasswd_path }} @@ -96,10 +40,6 @@ Create kube-apiserver Pod manifest: - /etc/kubernetes/pki/front-proxy-client.key - /etc/kubernetes/pki/sa.pub - {{ htpasswd_path }} -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - /etc/keepalived/check-apiserver.sh - - /etc/keepalived/keepalived.conf.sh -{%- endif %} - context: name: kube-apiserver host: {{ host }} @@ -153,72 +93,14 @@ Create kube-apiserver Pod manifest: - path: {{ htpasswd_path }} type: File name: htpasswd -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - path: /etc/keepalived - name: keepalived-config -{%- endif %} sidecars: -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - name: keepalived - image: {{ build_image_name("keepalived") }} - args: - - --dont-fork - - --dump-conf - - --address-monitoring - - --log-console - - --log-detail - - --vrrp - env: - - name: INTERFACE_ADDRESS - value: {{ networks.control_plane.split('/')[0] }} - - name: VRRP_PASSWORD - value: {{ pillar.metalk8s.api_server.keepalived.authPassword }} - - name: VIP - value: {{ pillar.metalk8s.api_server.host }} - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: - - NET_ADMIN - - NET_BROADCAST - - NET_RAW - - SETGID - - SETUID - resources: - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - name: keepalived-config - mountPath: /etc/keepalived - readOnly: true - - name: keepalived-varrun - mountPath: /var/run - - name: keepalived-tmp - mountPath: /tmp -{%- endif %} extra_volumes: -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - name: keepalived-varrun - emptyDir: - medium: Memory - - name: keepalived-tmp - emptyDir: - medium: Memory -{%- endif %} - require: - file: Ensure kubernetes CA cert is present - file: Ensure etcd CA cert is present - file: Ensure front-proxy CA cert is present - file: Ensure SA pub key is present - file: Set up default basic auth htpasswd -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - file: Create keepalived check script - - file: Create keepalived configuration file generator -{%- endif %} Make sure kube-apiserver container is up: module.wait: diff --git a/salt/metalk8s/node/grains.sls b/salt/metalk8s/node/grains.sls index 24c7e7fd67..6b8c5b9d3f 100644 --- a/salt/metalk8s/node/grains.sls +++ b/salt/metalk8s/node/grains.sls @@ -1,7 +1,4 @@ {% set control_plane_ips = salt['network.ip_addrs'](cidr=salt['pillar.get']('networks:control_plane')) %} -{% if pillar.metalk8s.api_server.keepalived.enabled %} -{% set control_plane_ips = control_plane_ips | difference([pillar.metalk8s.api_server.host]) | list | sort %} -{% endif %} {% if control_plane_ips %} {% if 'metalk8s' not in grains