From b6acef779284c840a9da6c1e5f277f9f03ed1ade Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Fri, 6 Dec 2019 21:05:18 +0100 Subject: [PATCH] salt, kubernetes: remove `apiServer` from `BootstrapConfiguration` We no longer need this since we provide in-cluster HA for `kube-apiserver` access. If this is desired for out-of-cluster access, we can provide this using a `LoadBalancer` `Service` once we have the infrastructure to support this in place. This also removed the optional deployment of `keepalived`. See: https://github.com/scality/metalk8s/issues/2103 See: https://github.com/scality/metalk8s/issues/1788 --- CHANGELOG.md | 9 ++ Vagrantfile | 8 +- buildchain/buildchain/image.py | 15 --- buildchain/buildchain/versions.py | 9 -- docs/developer/architecture/deployment.rst | 14 --- docs/developer/running/ui.rst | 6 +- docs/quickstart/bootstrap.rst | 3 - .../terraform/scripts/bootstrap-config.sh | 2 - .../terraform/scripts/bootstrap-config.sh | 2 - images/keepalived/Dockerfile | 68 ---------- images/keepalived/entrypoint.sh | 14 --- salt/_pillar/metalk8s.py | 36 +----- .../kubernetes/apiserver/certs/server.sls | 1 - .../kubernetes/apiserver/installed.sls | 118 ------------------ salt/metalk8s/node/grains.sls | 3 - salt/metalk8s/salt/master/configured.sls | 5 +- 16 files changed, 17 insertions(+), 296 deletions(-) delete mode 100644 images/keepalived/Dockerfile delete mode 100755 images/keepalived/entrypoint.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 6edbd63274..8184c1670e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Release 2.4.2 (in development) +### Breaking changes +- If `apiServer.host` is configured in `BootstrapConfiguration`, this is no +longer used (and must no longer be defined). +- If `apiServer.keepalived` is configured in `BootstrapConfiguration`, this is +no longer used, and Keepalived is no longer deployed at all. +- Generated `admin.conf` `KubeConfig` files point to the control-plane IP of the +host on which they are generated. You can override this when using them using +`kubectl`s `-s`/`--server` argument to point to another address. + ### Features added - [#2103](https://github.com/scality/metalk8s/issues/2103) - Add a host-local diff --git a/Vagrantfile b/Vagrantfile index 6257310f36..d5db6a92aa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -36,9 +36,7 @@ CONTROL_PLANE_NETWORK = { # and # https://github.com/hashicorp/vagrant/blob/1e1c398de565ed0aab9631cfad2db6e1dac82d7f/plugins/providers/virtualbox/action/network.rb#L317 # or https://github.com/hashicorp/vagrant/pull/7699 - # Also, we leave off the 'last' IP in the control-plane network, which is - # reserved as the VIP of the API server. - :dhcp_upper => IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(3).first.to_s, + :dhcp_upper => IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(2).first.to_s, } # Bigger network for the workload plane. However, we only allow DHCP to allocate @@ -136,10 +134,6 @@ networks: workloadPlane: #{WORKLOAD_PLANE_IP}/#{prefixlen(WORKLOAD_PLANE_NETMASK)} ca: minion: bootstrap -apiServer: - host: #{IPAddr.new(CONTROL_PLANE_IP).mask(CONTROL_PLANE_NETMASK).to_range.last(2).first.to_s} - keepalived: - enabled: true archives: - /srv/scality/metalk8s-$VERSION EOF diff --git a/buildchain/buildchain/image.py b/buildchain/buildchain/image.py index 564d492545..d3ae4e30b4 100644 --- a/buildchain/buildchain/image.py +++ b/buildchain/buildchain/image.py @@ -218,21 +218,6 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage: name='salt-master', build_args={'SALT_VERSION': versions.SALT_VERSION}, ), - _local_image( - name='keepalived', - build_args={ - 'KEEPALIVED_IMAGE': versions.CENTOS_BASE_IMAGE, - 'KEEPALIVED_IMAGE_SHA256': versions.CENTOS_BASE_IMAGE_SHA256, - 'KEEPALIVED_VERSION': versions.KEEPALIVED_VERSION, - 'BUILD_DATE': datetime.datetime.now(datetime.timezone.utc) - .astimezone() - .isoformat(), - 'VCS_REF': constants.GIT_REF or '', - 'VERSION': versions.CONTAINER_IMAGES_MAP['keepalived'].version, - 'METALK8S_VERSION': versions.VERSION, - }, - file_dep=[constants.ROOT/'images'/'keepalived'/'entrypoint.sh'], - ), _local_image( name='metalk8s-ui', dockerfile=constants.ROOT/'ui'/'Dockerfile', diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py index b41134ddbb..8cd5844cd3 100644 --- a/buildchain/buildchain/versions.py +++ b/buildchain/buildchain/versions.py @@ -19,7 +19,6 @@ CALICO_VERSION : str = '3.8.2' K8S_VERSION : str = '1.15.5' -KEEPALIVED_VERSION : str = '1.3.5-16.el7' SALT_VERSION : str = '2018.3.4' def load_version_information() -> None: @@ -67,7 +66,6 @@ def load_version_information() -> None: # This should be reset to 1 when the service exposed by the container changes # version. SALT_MASTER_BUILD_ID = 1 -KEEPALIVED_BUILD_ID = 1 def _version_prefix(version: str, prefix: str = 'v') -> str: @@ -184,13 +182,6 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: digest='sha256:ed3ec0597c2d5b7102a7f62c661a23d8e4b34d910693fc23fd40bfb1d9404dcf', ), # Local images - Image( - name='keepalived', - version='{version}-{build_id}'.format( - version=KEEPALIVED_VERSION, build_id=KEEPALIVED_BUILD_ID - ), - digest=None, - ), Image( name='metalk8s-ui', version=VERSION, diff --git a/docs/developer/architecture/deployment.rst b/docs/developer/architecture/deployment.rst index 008effd683..c8dd4ab724 100644 --- a/docs/developer/architecture/deployment.rst +++ b/docs/developer/architecture/deployment.rst @@ -57,19 +57,6 @@ which can vary from one installation to another: Default: ``10.96.0.0/12`` -- VIP for the ``kube-apiserver`` and ``keepalived`` toggle - - Used as the address of ``kube-apiserver`` where required. This can either be - a VIP managed by custom load-balancing/high-availability infrastructure, in - which case the ``keepalived`` toggle must be off, or one which our platform - will manage using ``keepalived``. - - If ``keepalived`` is enabled, this VIP must sit in a control plane CIDR - shared by all control plane nodes. - - Note: we run ``keepalived`` in unicast mode, which is an extension of classic - VRRP, but removes the need for multicast support on the network. - Firewall ^^^^^^^^ @@ -81,7 +68,6 @@ We assume SSH access is not blocked by the host-based firewall. These services include: -- VRRP if ``keepalived`` is enabled on control-plane nodes - HTTPS on the bootstrap node, for ``nginx`` fronting the OCI registry and serving the yum repository - ``salt-master`` on the bootstrap node diff --git a/docs/developer/running/ui.rst b/docs/developer/running/ui.rst index 1628c0df3e..36a9da413e 100644 --- a/docs/developer/running/ui.rst +++ b/docs/developer/running/ui.rst @@ -30,8 +30,12 @@ Procedure 'salt-call', 'pillar.get', 'metalk8s', '--out', 'json' ]) pillar = json.loads(output)['local'] + output = subprocess.check_output([ + 'salt-call', 'grains.get', 'metalk8s:control_plane_ip', '--out', 'json' + ]) + control_plane_ip = json.loads(output)['local'] ui_conf = { - 'url': 'https://{}:6443'.format(pillar['api_server']['host']), + 'url': 'https://{}:6443'.format(control_plane_ip), 'url_salt': 'https://{salt[ip]}:{salt[ports][api]}'.format( salt=pillar['endpoints']['salt-master'] ), diff --git a/docs/quickstart/bootstrap.rst b/docs/quickstart/bootstrap.rst index b574a64b45..280d36f6d4 100644 --- a/docs/quickstart/bootstrap.rst +++ b/docs/quickstart/bootstrap.rst @@ -38,8 +38,6 @@ Configuration workloadPlane: ca: minion: - apiServer: - host: archives: - @@ -50,7 +48,6 @@ system is configured to re-mount them automatically after a reboot. .. todo:: - Explain the role of this config file and its values - - Add a note about setting HA for ``apiServer`` .. _quickstart-bootstrap-ssh: diff --git a/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh b/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh index 1a2eb86801..8365951580 100644 --- a/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh +++ b/eve/workers/openstack-multiple-nodes/terraform/scripts/bootstrap-config.sh @@ -17,8 +17,6 @@ networks: workloadPlane: 10.100.0.0/16 ca: minion: $(cat /etc/salt/minion_id) -apiServer: - host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }') archives: - /var/tmp/metalk8s EOF diff --git a/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh b/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh index 1a2eb86801..8365951580 100644 --- a/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh +++ b/eve/workers/openstack-single-node-rhel/terraform/scripts/bootstrap-config.sh @@ -17,8 +17,6 @@ networks: workloadPlane: 10.100.0.0/16 ca: minion: $(cat /etc/salt/minion_id) -apiServer: - host: $(ip route get 10.100.0.0 | awk '/10.100.0.0/{ print $6 }') archives: - /var/tmp/metalk8s EOF diff --git a/images/keepalived/Dockerfile b/images/keepalived/Dockerfile deleted file mode 100644 index 732fabd740..0000000000 --- a/images/keepalived/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# SHA256 digest of the base image -ARG KEEPALIVED_IMAGE_SHA256 -ARG KEEPALIVED_IMAGE=docker.io/centos - -FROM $KEEPALIVED_IMAGE@sha256:$KEEPALIVED_IMAGE_SHA256 - -ARG KEEPALIVED_VERSION - -# Timestamp of the build, formatted as RFC3339 -ARG BUILD_DATE -# Git revision o the tree at build time -ARG VCS_REF -# Version of the image -ARG VERSION -# Version of the project, e.g. `git describe --always --long --dirty --broken` -ARG METALK8S_VERSION - -# Create user early on, since this can be a shared layer for many versions of this image -RUN useradd \ - --user-group \ - --no-create-home \ - --system \ - keepalived_script - -ENTRYPOINT ["/entrypoint.sh"] -CMD ["--dont-fork", "--dont-respawn", "--dump-conf", "--address-monitoring", "--log-console", "--log-detail"] - -# The entrypoint script is less likely to change than KEEPALIVED_VERSION -COPY entrypoint.sh /entrypoint.sh - -# These contain BUILD_DATE so should come 'late' for layer caching -LABEL maintainer="moonshot-platform@scality.com" \ - # http://label-schema.org/rc1/ - org.label-schema.build-date="$BUILD_DATE" \ - org.label-schema.name="keepalived" \ - org.label-schema.description="Load balancer and high availability service, part of MetalK8s" \ - org.label-schema.license="GPLv2+" \ - org.label-schema.url="https://github.com/scality/metalk8s/" \ - org.label-schema.vcs-url="https://github.com/scality/metalk8s.git" \ - org.label-schema.vcs-ref="$VCS_REF" \ - org.label-schema.vendor="Scality" \ - org.label-schema.version="$VERSION" \ - org.label-schema.schema-version="1.0" \ - org.label-schema.docker.cmd="docker run --net host -v /keepalived.conf:/etc/keepalived/keepalived.conf:ro keepalived:$VERSION" \ - # https://github.com/opencontainers/image-spec/blob/master/annotations.md - org.opencontainers.image.created="$BUILD_DATE" \ - org.opencontainers.image.authors="moonshot-platform@scality.com" \ - org.opencontainers.image.url="https://github.com/scality/metalk8s/" \ - org.opencontainers.image.source="https://github.com/scality/metalk8s.git" \ - org.opencontainers.image.version="$VERSION" \ - org.opencontainers.image.revision="$VCS_REF" \ - org.opencontainers.image.vendor="Scality" \ - org.opencontainers.image.title="keepalived" \ - org.opencontainers.image.description="Load balancer and high availability service, part of MetalK8s" \ - # https://docs.openshift.org/latest/creating_images/metadata.html - io.openshift.tags="metalk8s,keepalived,ha" \ - io.k8s.description="Load balancer and high availability service, part of MetalK8s" \ - io.openshift.non-scalable="true" \ - # Various - com.scality.metalk8s.keepalived.version="$KEEPALIVED_VERSION" \ - com.scality.metalk8s.version="$METALK8S_VERSION" - -# Final layer installing keepalived-$KEEPALIVED_VERSION -RUN yum install -y \ - iproute \ - keepalived-${KEEPALIVED_VERSION} \ - && \ - yum clean all diff --git a/images/keepalived/entrypoint.sh b/images/keepalived/entrypoint.sh deleted file mode 100755 index 95eee93a28..0000000000 --- a/images/keepalived/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -xue -o pipefail - -if test -x /etc/keepalived/keepalived.conf.sh; then - KEEPALIVED_CONF=/var/run/keepalived.conf - /etc/keepalived/keepalived.conf.sh "${KEEPALIVED_CONF}" -else - KEEPALIVED_CONF=/etc/keepalived/keepalived.conf -fi - -test -f "${KEEPALIVED_CONF}" - -exec keepalived --use-file "${KEEPALIVED_CONF}" "$@" diff --git a/salt/_pillar/metalk8s.py b/salt/_pillar/metalk8s.py index 7fd0267a93..b99a2dd7d2 100644 --- a/salt/_pillar/metalk8s.py +++ b/salt/_pillar/metalk8s.py @@ -70,39 +70,6 @@ def _load_ca(config_data): } -def _load_apiserver(config_data): - errors = __utils__['pillar_utils.assert_keys'](config_data, ['apiServer']) - if errors: - return __utils__['pillar_utils.errors_to_dict'](errors) - - as_data = config_data['apiServer'] - - result = { - 'host': None, - 'keepalived': { - 'enabled': False, - 'virtualRouterId': 1, - 'authPassword': 'MeTaLk8s', - }, - 'kubeconfig': '/etc/kubernetes/admin.conf' - } - - errors = __utils__['pillar_utils.assert_keys'](as_data, ['host']) - if errors: - return __utils__['pillar_utils.errors_to_dict'](errors) - - result['host'] = as_data['host'] - - if 'keepalived' in as_data: - k_data = as_data['keepalived'] - k_result = result['keepalived'] - - for (key, default) in k_result.items(): - k_result[key] = k_data.get(key, default) - - return result - - def _load_iso_path(config_data): """Load iso path from BootstrapConfiguration @@ -133,7 +100,6 @@ def ext_pillar(minion_id, pillar, bootstrap_config): metal_data = { 'archives': _load_iso_path(config), 'ca': _load_ca(config), - 'api_server': _load_apiserver(config) } result = { @@ -144,7 +110,7 @@ def ext_pillar(minion_id, pillar, bootstrap_config): if not isinstance(metal_data['archives'], list): # Special case for archives in pillar __utils__['pillar_utils.promote_errors'](metal_data, 'archives') - for key in ['ca', 'api_server']: + for key in ['ca',]: __utils__['pillar_utils.promote_errors'](metal_data, key) for key in ['networks', 'metalk8s']: __utils__['pillar_utils.promote_errors'](result, key) diff --git a/salt/metalk8s/kubernetes/apiserver/certs/server.sls b/salt/metalk8s/kubernetes/apiserver/certs/server.sls index 90f2cbc017..56a1453e79 100644 --- a/salt/metalk8s/kubernetes/apiserver/certs/server.sls +++ b/salt/metalk8s/kubernetes/apiserver/certs/server.sls @@ -26,7 +26,6 @@ Create kube-apiserver private key: 'kubernetes.default.svc.cluster.local', kubernetes_service_ip, grains['metalk8s']['control_plane_ip'], - pillar['metalk8s']['api_server']['host'], '127.0.0.1', ] %} diff --git a/salt/metalk8s/kubernetes/apiserver/installed.sls b/salt/metalk8s/kubernetes/apiserver/installed.sls index be3eec4fba..6a85a6045e 100644 --- a/salt/metalk8s/kubernetes/apiserver/installed.sls +++ b/salt/metalk8s/kubernetes/apiserver/installed.sls @@ -9,62 +9,6 @@ include: - metalk8s.kubernetes.sa.advertised - .certs -{%- if pillar.metalk8s.api_server.keepalived.enabled %} -Create keepalived check script: - file.managed: - - name: /etc/keepalived/check-apiserver.sh - - mode: 0555 - - makedirs: true - - dir_mode: 0755 - - contents: | - #!/bin/bash - set -ue -o pipefail - test $(curl -k https://127.0.0.1:6443/healthz) = 'ok' - -Create keepalived configuration file generator: - file.managed: - - name: /etc/keepalived/keepalived.conf.sh - - mode: 0555 - - makedirs: true - - dir_mode: 0755 - - contents: | - #!/bin/bash - set -xue -o pipefail - - IP=${IP:-ip} - AWK=${AWK:-awk} - - INTERFACE=${INTERFACE:-$(${IP} route get ${INTERFACE_ADDRESS} | ${AWK} '/dev / { print $4 }')} - - cat > "$1" << EOF - global_defs { - enable_script_security - } - - vrrp_script check_apiserver { - script "/etc/keepalived/check-apiserver.sh" - interval 2 - weight 2 - } - - vrrp_instance VI_1 { - state ${VRRP_STATE:-BACKUP} - virtual_router_id ${VRRP_VIRTUAL_ROUTER_ID:-1} - interface ${INTERFACE} - authentication { - auth_type PASS - auth_pass ${VRRP_PASSWORD} - } - virtual_ipaddress { - ${VIP} - } - track_script { - check_apiserver - } - } - EOF -{%- endif %} - Set up default basic auth htpasswd: file.managed: - name: {{ htpasswd_path }} @@ -96,10 +40,6 @@ Create kube-apiserver Pod manifest: - /etc/kubernetes/pki/front-proxy-client.key - /etc/kubernetes/pki/sa.pub - {{ htpasswd_path }} -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - /etc/keepalived/check-apiserver.sh - - /etc/keepalived/keepalived.conf.sh -{%- endif %} - context: name: kube-apiserver host: {{ host }} @@ -153,72 +93,14 @@ Create kube-apiserver Pod manifest: - path: {{ htpasswd_path }} type: File name: htpasswd -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - path: /etc/keepalived - name: keepalived-config -{%- endif %} sidecars: -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - name: keepalived - image: {{ build_image_name("keepalived") }} - args: - - --dont-fork - - --dump-conf - - --address-monitoring - - --log-console - - --log-detail - - --vrrp - env: - - name: INTERFACE_ADDRESS - value: {{ networks.control_plane.split('/')[0] }} - - name: VRRP_PASSWORD - value: {{ pillar.metalk8s.api_server.keepalived.authPassword }} - - name: VIP - value: {{ pillar.metalk8s.api_server.host }} - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - add: - - NET_ADMIN - - NET_BROADCAST - - NET_RAW - - SETGID - - SETUID - resources: - requests: - cpu: 100m - memory: 64Mi - volumeMounts: - - name: keepalived-config - mountPath: /etc/keepalived - readOnly: true - - name: keepalived-varrun - mountPath: /var/run - - name: keepalived-tmp - mountPath: /tmp -{%- endif %} extra_volumes: -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - name: keepalived-varrun - emptyDir: - medium: Memory - - name: keepalived-tmp - emptyDir: - medium: Memory -{%- endif %} - require: - file: Ensure kubernetes CA cert is present - file: Ensure etcd CA cert is present - file: Ensure front-proxy CA cert is present - file: Ensure SA pub key is present - file: Set up default basic auth htpasswd -{%- if pillar.metalk8s.api_server.keepalived.enabled %} - - file: Create keepalived check script - - file: Create keepalived configuration file generator -{%- endif %} Make sure kube-apiserver container is up: module.wait: diff --git a/salt/metalk8s/node/grains.sls b/salt/metalk8s/node/grains.sls index 24c7e7fd67..6b8c5b9d3f 100644 --- a/salt/metalk8s/node/grains.sls +++ b/salt/metalk8s/node/grains.sls @@ -1,7 +1,4 @@ {% set control_plane_ips = salt['network.ip_addrs'](cidr=salt['pillar.get']('networks:control_plane')) %} -{% if pillar.metalk8s.api_server.keepalived.enabled %} -{% set control_plane_ips = control_plane_ips | difference([pillar.metalk8s.api_server.host]) | list | sort %} -{% endif %} {% if control_plane_ips %} {% if 'metalk8s' not in grains diff --git a/salt/metalk8s/salt/master/configured.sls b/salt/metalk8s/salt/master/configured.sls index f2e55e3fee..464034f97b 100644 --- a/salt/metalk8s/salt/master/configured.sls +++ b/salt/metalk8s/salt/master/configured.sls @@ -13,10 +13,7 @@ Configure salt master: - template: jinja - defaults: salt_ip: "{{ salt_ip }}" - kubeconfig: "{{ pillar['metalk8s']['api_server']['kubeconfig'] }}" - {%- if pillar['metalk8s']['api_server'].get('context') %} - kubecontext: "{{ pillar['metalk8s']['api_server']['context'] }}" - {%- endif %} + kubeconfig: "/etc/kubernetes/admin.conf" Configure salt master roots paths: file.serialize: