diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 449512d18dd..180166c2422 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ stages: - deploy-special variables: - KUBESPRAY_VERSION: v2.23.0 + KUBESPRAY_VERSION: v2.23.1 FAILFASTCI_NAMESPACE: 'kargo-ci' GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray' ANSIBLE_FORCE_COLOR: "true" diff --git a/README.md b/README.md index e3a0f3fdea9..f0dbc6b4999 100644 --- a/README.md +++ b/README.md @@ -75,11 +75,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou to access the inventory and SSH key in the container, like this: ```ShellSession -git checkout v2.23.0 -docker pull quay.io/kubespray/kubespray:v2.23.0 +git checkout v2.23.1 +docker pull quay.io/kubespray/kubespray:v2.23.1 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \ --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \ - quay.io/kubespray/kubespray:v2.23.0 bash + quay.io/kubespray/kubespray:v2.23.1 bash # Inside the container you may now run the kubespray playbooks: ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml ``` @@ -164,7 +164,7 @@ Note: Upstart/SysV init based OS types are not supported. - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.3 - [etcd](https://github.com/etcd-io/etcd) v3.5.9 - [docker](https://www.docker.com/) v20.10 (see note) - - [containerd](https://containerd.io/) v1.7.7 + - [containerd](https://containerd.io/) v1.7.8 - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0 @@ -172,14 +172,14 @@ Note: Upstart/SysV init based OS types are not supported. - [cilium](https://github.com/cilium/cilium) v1.13.4 - [flannel](https://github.com/flannel-io/flannel) v0.22.0 - [kube-ovn](https://github.com/alauda/kube-ovn) v1.11.5 - - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.6.0 + - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0 - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8 - [weave](https://github.com/weaveworks/weave) v2.8.1 - [kube-vip](https://github.com/kube-vip/kube-vip) v0.5.12 - Application - [cert-manager](https://github.com/jetstack/cert-manager) v1.12.5 - [coredns](https://github.com/coredns/coredns) v1.10.1 - - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.3 + - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.9.4 - [krew](https://github.com/kubernetes-sigs/krew) v0.4.4 - [argocd](https://argoproj.github.io/) v2.8.4 - [helm](https://helm.sh/) v3.13.1 diff --git a/docs/ansible_collection.md b/docs/ansible_collection.md index 6b559ccdc8b..cbe1ad2c5e5 100644 --- a/docs/ansible_collection.md +++ b/docs/ansible_collection.md @@ -15,7 +15,7 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a collections: - name: https://github.com/kubernetes-sigs/kubespray type: git - version: v2.23.0 + version: v2.23.1 ``` 2. Install your collection diff --git a/docs/vars.md b/docs/vars.md index 36dd3621da9..b3239da948f 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -186,6 +186,8 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m * *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin. [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overridden in inventory vars. +* *crio_criu_support_enabled* - When set to `true`, enables the container checkpoint/restore in CRI-O. It's required to install [CRIU](https://criu.org/Installation) on the host when dumping/restoring checkpoints. And it's recommended to enable the feature gate `ContainerCheckpoint` so that the kubelet get a higher level API to simplify the operations (**Note**: It's still in experimental stage, just for container analytics so far). You can follow the [documentation](https://kubernetes.io/blog/2022/12/05/forensic-container-checkpointing-alpha/). + * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames that correspond to each node. diff --git a/galaxy.yml b/galaxy.yml index a0df3882c4d..7b487ca50dc 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,7 +2,7 @@ namespace: kubernetes_sigs description: Deploy a production ready Kubernetes cluster name: kubespray -version: 2.23.0 +version: 2.23.1 readme: README.md authors: - luksi1 diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml index 144b381029a..21947a99d88 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml @@ -1,4 +1,10 @@ -# See roles/network_plugin/kube-router//defaults/main.yml +# See roles/network_plugin/kube-router/defaults/main.yml + +# Kube router version +# Default to v2 +# kube_router_version: "v2.0.0" +# Uncomment to use v1 (Deprecated) +# kube_router_version: "v1.6.0" # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP # kube_router_run_router: true diff --git a/roles/container-engine/containerd/handlers/main.yml b/roles/container-engine/containerd/handlers/main.yml index 3c132bdf0d0..1959dc99171 100644 --- a/roles/container-engine/containerd/handlers/main.yml +++ b/roles/container-engine/containerd/handlers/main.yml @@ -1,10 +1,4 @@ --- -- name: Restart containerd - command: /bin/true - notify: - - Containerd | restart containerd - - Containerd | wait for containerd - - name: Containerd | restart containerd systemd: name: containerd @@ -12,6 +6,7 @@ enabled: yes daemon-reload: yes masked: no + listen: Restart containerd - name: Containerd | wait for containerd command: "{{ containerd_bin_dir }}/ctr images ls -q" @@ -19,3 +14,4 @@ retries: 8 delay: 4 until: containerd_ready.rc == 0 + listen: Restart containerd diff --git a/roles/container-engine/cri-dockerd/handlers/main.yml b/roles/container-engine/cri-dockerd/handlers/main.yml index 3990d3397c7..3a249791669 100644 --- a/roles/container-engine/cri-dockerd/handlers/main.yml +++ b/roles/container-engine/cri-dockerd/handlers/main.yml @@ -1,35 +1,31 @@ --- -- name: Restart and enable cri-dockerd - command: /bin/true - notify: - - Cri-dockerd | reload systemd - - Cri-dockerd | restart docker.service - - Cri-dockerd | reload cri-dockerd.socket - - Cri-dockerd | reload cri-dockerd.service - - Cri-dockerd | enable cri-dockerd service - - name: Cri-dockerd | reload systemd systemd: name: cri-dockerd daemon_reload: true masked: no + listen: Restart and enable cri-dockerd - name: Cri-dockerd | restart docker.service service: name: docker.service state: restarted + listen: Restart and enable cri-dockerd - name: Cri-dockerd | reload cri-dockerd.socket service: name: cri-dockerd.socket state: restarted + listen: Restart and enable cri-dockerd - name: Cri-dockerd | reload cri-dockerd.service service: name: cri-dockerd.service state: restarted + listen: Restart and enable cri-dockerd - name: Cri-dockerd | enable cri-dockerd service service: name: cri-dockerd.service enabled: yes + listen: Restart and enable cri-dockerd diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml index 949ed69ed53..21de17aeb9c 100644 --- a/roles/container-engine/cri-o/defaults/main.yml +++ b/roles/container-engine/cri-o/defaults/main.yml @@ -97,3 +97,6 @@ crio_man_files: 8: - crio - crio-status + +# If set to true, it will enable the CRIU support in cri-o +crio_criu_support_enabled: false diff --git a/roles/container-engine/cri-o/handlers/main.yml b/roles/container-engine/cri-o/handlers/main.yml index 763f4b558b6..b0c5951fda8 100644 --- a/roles/container-engine/cri-o/handlers/main.yml +++ b/roles/container-engine/cri-o/handlers/main.yml @@ -1,16 +1,12 @@ --- -- name: Restart crio - command: /bin/true - notify: - - CRI-O | reload systemd - - CRI-O | reload crio - - name: CRI-O | reload systemd systemd: daemon_reload: true + listen: Restart crio - name: CRI-O | reload crio service: name: crio state: restarted enabled: yes + listen: Restart crio diff --git a/roles/container-engine/cri-o/tasks/cleanup.yaml b/roles/container-engine/cri-o/tasks/cleanup.yaml index 2b8251c4e46..1675a5b2fad 100644 --- a/roles/container-engine/cri-o/tasks/cleanup.yaml +++ b/roles/container-engine/cri-o/tasks/cleanup.yaml @@ -118,3 +118,8 @@ - cri-o - cri-o-runc - oci-systemd-hook + +- name: CRI-O | Remove CRI-O package configuration files + file: + name: /etc/crio/crio.conf.d/01-crio-runc.conf + state: absent diff --git a/roles/container-engine/cri-o/tasks/reset.yml b/roles/container-engine/cri-o/tasks/reset.yml index 65ee0026a64..1bae013acb1 100644 --- a/roles/container-engine/cri-o/tasks/reset.yml +++ b/roles/container-engine/cri-o/tasks/reset.yml @@ -17,7 +17,7 @@ - name: CRI-O | Remove cri-o apt repo apt_repository: repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /" - state: present + state: absent filename: devel-kubic-libcontainers-stable-cri-o when: crio_kubic_debian_repo_name is defined tags: diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index f0455d0939b..81d5a421e0b 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -273,6 +273,11 @@ pinns_path = "" pinns_path = "{{ bin_dir }}/pinns" {% endif %} +{% if crio_criu_support_enabled %} +# Enable CRIU integration, requires that the criu binary is available in $PATH. +enable_criu_support = true +{% endif %} + # The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. # The runtime to use is picked based on the runtime_handler provided by the CRI. # If no runtime_handler is provided, the runtime will be picked based on the level diff --git a/roles/container-engine/docker/handlers/main.yml b/roles/container-engine/docker/handlers/main.yml index 14a7b3973d3..4a8a63948bd 100644 --- a/roles/container-engine/docker/handlers/main.yml +++ b/roles/container-engine/docker/handlers/main.yml @@ -1,28 +1,25 @@ --- -- name: Restart docker - command: /bin/true - notify: - - Docker | reload systemd - - Docker | reload docker.socket - - Docker | reload docker - - Docker | wait for docker - - name: Docker | reload systemd systemd: name: docker daemon_reload: true masked: no + listen: Restart docker - name: Docker | reload docker.socket service: name: docker.socket state: restarted when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos + listen: Restart docker + - name: Docker | reload docker service: name: docker state: restarted + listen: Restart docker + - name: Docker | wait for docker command: "{{ docker_bin_dir }}/docker images" @@ -30,3 +27,4 @@ retries: 20 delay: 1 until: docker_ready.rc == 0 + listen: Restart docker diff --git a/roles/download/defaults/main/checksums.yml b/roles/download/defaults/main/checksums.yml index 4c638d1e44e..062e1e62e27 100644 --- a/roles/download/defaults/main/checksums.yml +++ b/roles/download/defaults/main/checksums.yml @@ -1045,6 +1045,7 @@ nerdctl_archive_checksums: 1.0.0: 2fb02e629a4be16b194bbfc64819132a72ede1f52596bd8e1ec2beaf7c28c117 containerd_archive_checksums: arm: + 1.7.8: 0 1.7.7: 0 1.7.6: 0 1.7.5: 0 @@ -1084,6 +1085,7 @@ containerd_archive_checksums: 1.5.15: 0 1.5.14: 0 arm64: + 1.7.8: 3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1 1.7.7: 0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3 1.7.6: d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce 1.7.5: 98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9 @@ -1123,6 +1125,7 @@ containerd_archive_checksums: 1.5.15: 0 1.5.14: 0 amd64: + 1.7.8: 5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9 1.7.7: 371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba 1.7.6: 58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe 1.7.5: 33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25 @@ -1162,6 +1165,7 @@ containerd_archive_checksums: 1.5.15: 0d09043be08dcf6bf136aa78bfd719e836cf9f9679afa4db0b6e4d478e396528 1.5.14: 8513ead11aca164b7e70bcea0429b4e51dad836b6383b806322e128821aaebbd ppc64le: + 1.7.8: 2b563df9e1bddc96a99a023963c99b5faf3066d3fcbc23ff44ba24229e939444 1.7.7: 0335e7447ed84757489337686a709e95ffa379a8780f238725abb10facaeaa7f 1.7.6: 956fadb01b35c3214f2b6f82abc0dda3e1b754cb223cd24e818334b08cb09fb2 1.7.5: 2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc diff --git a/roles/download/defaults/main/main.yml b/roles/download/defaults/main/main.yml index 4f00beb0421..0868270761f 100644 --- a/roles/download/defaults/main/main.yml +++ b/roles/download/defaults/main/main.yml @@ -79,7 +79,7 @@ runc_version: v1.1.9 kata_containers_version: 3.1.3 youki_version: 0.1.0 gvisor_version: 20230807 -containerd_version: 1.7.7 +containerd_version: 1.7.8 cri_dockerd_version: 0.3.4 # this is relevant when container_manager == 'docker' @@ -122,7 +122,7 @@ cilium_enable_hubble: false kube_ovn_version: "v1.11.5" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" -kube_router_version: "v1.6.0" +kube_router_version: "v2.0.0" multus_version: "v3.8" helm_version: "v3.13.1" nerdctl_version: "1.6.0" @@ -306,7 +306,7 @@ rbd_provisioner_image_tag: "{{ rbd_provisioner_version }}" local_path_provisioner_version: "v0.0.24" local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner" local_path_provisioner_image_tag: "{{ local_path_provisioner_version }}" -ingress_nginx_version: "v1.9.3" +ingress_nginx_version: "v1.9.4" ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller" ingress_nginx_controller_image_tag: "{{ ingress_nginx_version }}" ingress_nginx_kube_webhook_certgen_image_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen" diff --git a/roles/etcd/handlers/backup.yml b/roles/etcd/handlers/backup.yml index 2c5577862b7..b79dd014865 100644 --- a/roles/etcd/handlers/backup.yml +++ b/roles/etcd/handlers/backup.yml @@ -1,22 +1,14 @@ --- -- name: Backup etcd data - command: /bin/true - notify: - - Refresh Time Fact - - Set Backup Directory - - Create Backup Directory - - Stat etcd v2 data directory - - Backup etcd v2 data - - Backup etcd v3 data - when: etcd_cluster_is_healthy.rc == 0 - - name: Refresh Time Fact setup: filter: ansible_date_time + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 - name: Set Backup Directory set_fact: etcd_backup_directory: "{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}" + listen: Restart etcd - name: Create Backup Directory file: @@ -25,6 +17,8 @@ owner: root group: root mode: 0600 + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 - name: Stat etcd v2 data directory stat: @@ -33,9 +27,13 @@ get_checksum: no get_mime: no register: etcd_data_dir_member + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 - name: Backup etcd v2 data - when: etcd_data_dir_member.stat.exists + when: + - etcd_data_dir_member.stat.exists + - etcd_cluster_is_healthy.rc == 0 command: >- {{ bin_dir }}/etcdctl backup --data-dir {{ etcd_data_dir }} @@ -46,6 +44,7 @@ register: backup_v2_command until: backup_v2_command.rc == 0 delay: "{{ retry_stagger | random + 3 }}" + listen: Restart etcd - name: Backup etcd v3 data command: >- @@ -61,3 +60,5 @@ register: etcd_backup_v3_command until: etcd_backup_v3_command.rc == 0 delay: "{{ retry_stagger | random + 3 }}" + listen: Restart etcd + when: etcd_cluster_is_healthy.rc == 0 diff --git a/roles/etcd/handlers/backup_cleanup.yml b/roles/etcd/handlers/backup_cleanup.yml index 63dcf41918f..85b8d19cf68 100644 --- a/roles/etcd/handlers/backup_cleanup.yml +++ b/roles/etcd/handlers/backup_cleanup.yml @@ -1,10 +1,4 @@ --- -- name: Cleanup etcd backups - command: /bin/true - notify: - - Find old etcd backups - - Remove old etcd backups - - name: Find old etcd backups ansible.builtin.find: file_type: directory @@ -13,6 +7,7 @@ patterns: "etcd-*" register: _etcd_backups when: etcd_backup_retention_count >= 0 + listen: Restart etcd - name: Remove old etcd backups ansible.builtin.file: @@ -20,3 +15,4 @@ path: "{{ item }}" loop: "{{ (_etcd_backups.files | sort(attribute='ctime', reverse=True))[etcd_backup_retention_count:] | map(attribute='path') }}" when: etcd_backup_retention_count >= 0 + listen: Restart etcd diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml index f09789c25f0..33890617aec 100644 --- a/roles/etcd/handlers/main.yml +++ b/roles/etcd/handlers/main.yml @@ -1,38 +1,27 @@ --- -- name: Restart etcd - command: /bin/true - notify: - - Backup etcd data - - Etcd | reload systemd - - Reload etcd - - Wait for etcd up - - Cleanup etcd backups - -- name: Restart etcd-events - command: /bin/true - notify: - - Etcd | reload systemd - - Reload etcd-events - - Wait for etcd-events up - - name: Backup etcd import_tasks: backup.yml - name: Etcd | reload systemd systemd: daemon_reload: true + listen: + - Restart etcd + - Restart etcd-events - name: Reload etcd service: name: etcd state: restarted when: is_etcd_master + listen: Restart etcd - name: Reload etcd-events service: name: etcd-events state: restarted when: is_etcd_master + listen: Restart etcd-events - name: Wait for etcd up uri: @@ -44,6 +33,7 @@ until: result.status is defined and result.status == 200 retries: 60 delay: 1 + listen: Restart etcd - name: Cleanup etcd backups import_tasks: backup_cleanup.yml @@ -58,6 +48,7 @@ until: result.status is defined and result.status == 200 retries: 60 delay: 1 + listen: Restart etcd-events - name: Set etcd_secret_changed set_fact: diff --git a/roles/kubernetes-apps/metallb/defaults/main.yml b/roles/kubernetes-apps/metallb/defaults/main.yml index 22e2935b25f..02f4e3cae94 100644 --- a/roles/kubernetes-apps/metallb/defaults/main.yml +++ b/roles/kubernetes-apps/metallb/defaults/main.yml @@ -13,3 +13,4 @@ metallb_speaker_tolerations: key: node-role.kubernetes.io/control-plane operator: Exists metallb_controller_tolerations: [] +metallb_loadbalancer_class: "" \ No newline at end of file diff --git a/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 b/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 index eab386ff8f8..608ad31cda4 100644 --- a/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 +++ b/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2 @@ -1724,6 +1724,9 @@ spec: - args: - --port={{ metallb_port }} - --log-level={{ metallb_log_level }} +{% if metallb_loadbalancer_class != "" %} + - --lb-class={{ metallb_loadbalancer_class }} +{% endif %} env: - name: METALLB_ML_SECRET_NAME value: memberlist @@ -1814,6 +1817,9 @@ spec: - args: - --port={{ metallb_port }} - --log-level={{ metallb_log_level }} +{% if metallb_loadbalancer_class != "" %} + - --lb-class={{ metallb_loadbalancer_class }} +{% endif %} env: - name: METALLB_NODE_NAME valueFrom: diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 8f76ad1bf79..16731d9e766 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -224,8 +224,7 @@ event_ttl_duration: "1h0m0s" ## Automatically renew K8S control plane certificates on first Monday of each month auto_renew_certificates: false # First Monday of each month -auto_renew_certificates_systemd_calendar: "{{ 'Mon *-*-1,2,3,4,5,6,7 03:' ~ - groups['kube_control_plane'].index(inventory_hostname) ~ '0:00' }}" +auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00" # kubeadm renews all the certificates during control plane upgrade. # If we have requirement like without renewing certs upgrade the cluster, # we can opt out from the default behavior by setting kubeadm_upgrade_auto_cert_renewal to false diff --git a/roles/kubernetes/control-plane/handlers/main.yml b/roles/kubernetes/control-plane/handlers/main.yml index d5f17963ffd..1ee64f23069 100644 --- a/roles/kubernetes/control-plane/handlers/main.yml +++ b/roles/kubernetes/control-plane/handlers/main.yml @@ -1,47 +1,14 @@ --- -- name: Master | restart kubelet - command: /bin/true - notify: - - Master | reload systemd - - Master | reload kubelet - - Master | wait for master static pods - -- name: Master | wait for master static pods - command: /bin/true - notify: - - Master | wait for the apiserver to be running - - Master | wait for kube-scheduler - - Master | wait for kube-controller-manager - -- name: Master | Restart apiserver - command: /bin/true - notify: - - Master | Remove apiserver container docker - - Master | Remove apiserver container containerd/crio - - Master | wait for the apiserver to be running - -- name: Master | Restart kube-scheduler - command: /bin/true - notify: - - Master | Remove scheduler container docker - - Master | Remove scheduler container containerd/crio - - Master | wait for kube-scheduler - -- name: Master | Restart kube-controller-manager - command: /bin/true - notify: - - Master | Remove controller manager container docker - - Master | Remove controller manager container containerd/crio - - Master | wait for kube-controller-manager - - name: Master | reload systemd systemd: daemon_reload: true + listen: Master | restart kubelet - name: Master | reload kubelet service: name: kubelet state: restarted + listen: Master | restart kubelet - name: Master | Remove apiserver container docker shell: "set -o pipefail && docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f" @@ -52,6 +19,7 @@ until: remove_apiserver_container.rc == 0 delay: 1 when: container_manager == "docker" + listen: Master | Restart apiserver - name: Master | Remove apiserver container containerd/crio shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" @@ -62,6 +30,7 @@ until: remove_apiserver_container.rc == 0 delay: 1 when: container_manager in ['containerd', 'crio'] + listen: Master | Restart apiserver - name: Master | Remove scheduler container docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" @@ -72,6 +41,7 @@ until: remove_scheduler_container.rc == 0 delay: 1 when: container_manager == "docker" + listen: Master | Restart kube-scheduler - name: Master | Remove scheduler container containerd/crio shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-scheduler* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" @@ -82,6 +52,7 @@ until: remove_scheduler_container.rc == 0 delay: 1 when: container_manager in ['containerd', 'crio'] + listen: Master | Restart kube-scheduler - name: Master | Remove controller manager container docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" @@ -92,6 +63,7 @@ until: remove_cm_container.rc == 0 delay: 1 when: container_manager == "docker" + listen: Master | Restart kube-controller-manager - name: Master | Remove controller manager container containerd/crio shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" @@ -102,6 +74,7 @@ until: remove_cm_container.rc == 0 delay: 1 when: container_manager in ['containerd', 'crio'] + listen: Master | Restart kube-controller-manager - name: Master | wait for kube-scheduler vars: @@ -113,6 +86,9 @@ until: scheduler_result.status == 200 retries: 60 delay: 1 + listen: + - Master | restart kubelet + - Master | Restart kube-scheduler - name: Master | wait for kube-controller-manager vars: @@ -124,6 +100,9 @@ until: controller_manager_result.status == 200 retries: 60 delay: 1 + listen: + - Master | restart kubelet + - Master | Restart kube-controller-manager - name: Master | wait for the apiserver to be running uri: @@ -133,3 +112,6 @@ until: result.status == 200 retries: 60 delay: 1 + listen: + - Master | restart kubelet + - Master | Restart apiserver diff --git a/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 b/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 index 904f0073cf4..cca5aca3e72 100644 --- a/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 +++ b/roles/kubernetes/control-plane/templates/k8s-certs-renew.timer.j2 @@ -3,6 +3,9 @@ Description=Timer to renew K8S control plane certificates [Timer] OnCalendar={{ auto_renew_certificates_systemd_calendar }} +RandomizedDelaySec={{ 10 * (groups['kube_control_plane'] | length) }}min +FixedRandomDelay=yes +Persistent=yes [Install] WantedBy=multi-user.target diff --git a/roles/kubernetes/kubeadm/handlers/main.yml b/roles/kubernetes/kubeadm/handlers/main.yml index 4c2b1257c1e..9f6d4318b6c 100644 --- a/roles/kubernetes/kubeadm/handlers/main.yml +++ b/roles/kubernetes/kubeadm/handlers/main.yml @@ -1,15 +1,11 @@ --- -- name: Kubeadm | restart kubelet - command: /bin/true - notify: - - Kubeadm | reload systemd - - Kubeadm | reload kubelet - - name: Kubeadm | reload systemd systemd: daemon_reload: true + listen: Kubeadm | restart kubelet - name: Kubeadm | reload kubelet service: name: kubelet state: restarted + listen: Kubeadm | restart kubelet diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index f5dbf38ab0c..9d21d50147c 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -240,3 +240,11 @@ azure_cloud: AzurePublicCloud # - TLS_RSA_WITH_AES_256_CBC_SHA # - TLS_RSA_WITH_AES_256_GCM_SHA384 # - TLS_RSA_WITH_RC4_128_SHA + +kube_proxy_ipvs_modules: + - ip_vs + - ip_vs_rr + - ip_vs_wrr + - ip_vs_sh + - ip_vs_wlc + - ip_vs_lc diff --git a/roles/kubernetes/node/handlers/main.yml b/roles/kubernetes/node/handlers/main.yml index 512b4e8d401..8195b7d06ba 100644 --- a/roles/kubernetes/node/handlers/main.yml +++ b/roles/kubernetes/node/handlers/main.yml @@ -1,15 +1,11 @@ --- -- name: Node | restart kubelet - command: /bin/true - notify: - - Kubelet | reload systemd - - Kubelet | restart kubelet - - name: Kubelet | reload systemd systemd: daemon_reload: true + listen: Node | restart kubelet - name: Kubelet | restart kubelet service: name: kubelet state: restarted + listen: Node | restart kubelet diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index f89e03e2df8..7eb5b2e597d 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -112,11 +112,7 @@ community.general.modprobe: name: "{{ item }}" state: present - with_items: - - ip_vs - - ip_vs_rr - - ip_vs_wrr - - ip_vs_sh + loop: "{{ kube_proxy_ipvs_modules }}" when: kube_proxy_mode == 'ipvs' tags: - kube-proxy @@ -132,18 +128,18 @@ tags: - kube-proxy +- name: Add nf_conntrack_ipv4 kube-proxy ipvs module list + set_fact: + kube_proxy_ipvs_modules: "{{ kube_proxy_ipvs_modules + ['nf_conntrack_ipv4'] }}" + when: modprobe_nf_conntrack_ipv4 is success + tags: + - kube-proxy + - name: Persist ip_vs modules copy: dest: /etc/modules-load.d/kube_proxy-ipvs.conf mode: 0644 - content: | - ip_vs - ip_vs_rr - ip_vs_wrr - ip_vs_sh - {% if modprobe_nf_conntrack_ipv4 is success -%} - nf_conntrack_ipv4 - {%- endif -%} + content: "{{ kube_proxy_ipvs_modules | join('\n') }}" when: kube_proxy_mode == 'ipvs' tags: - kube-proxy diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index f7670318fb3..c18209e7ceb 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -23,6 +23,9 @@ common_required_pkgs: # GCE docker repository disable_ipv6_dns: false +# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). +remove_default_searchdomains: false + kube_owner: kube kube_cert_group: kube-cert kube_config_dir: /etc/kubernetes diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index 8ae931f267d..35140ab42fc 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -1,38 +1,14 @@ --- -- name: Preinstall | propagate resolvconf to k8s components - command: /bin/true - notify: - - Preinstall | reload kubelet - - Preinstall | kube-controller configured - - Preinstall | kube-apiserver configured - - Preinstall | restart kube-controller-manager docker - - Preinstall | restart kube-controller-manager crio/containerd - - Preinstall | restart kube-apiserver docker - - Preinstall | restart kube-apiserver crio/containerd - - Preinstall | wait for the apiserver to be running - when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos - -- name: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk - command: /bin/true - notify: - - Preinstall | apply resolvconf cloud-init - - Preinstall | reload kubelet - when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - - name: Preinstall | apply resolvconf cloud-init command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - -- name: Preinstall | update resolvconf for networkmanager - command: /bin/true - notify: - - Preinstall | reload NetworkManager - - Preinstall | reload kubelet + listen: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk - name: Preinstall | reload NetworkManager service: name: NetworkManager.service state: restarted + listen: Preinstall | update resolvconf for networkmanager - name: Preinstall | reload kubelet service: @@ -46,6 +22,10 @@ - Preinstall | restart kube-apiserver docker - Preinstall | restart kube-apiserver crio/containerd when: not dns_early | bool + listen: + - Preinstall | propagate resolvconf to k8s components + - Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk + - Preinstall | update resolvconf for networkmanager # FIXME(mattymo): Also restart for kubeadm mode - name: Preinstall | kube-apiserver configured @@ -56,6 +36,7 @@ get_mime: no register: kube_apiserver_set when: inventory_hostname in groups['kube_control_plane'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' + listen: Preinstall | propagate resolvconf to k8s components # FIXME(mattymo): Also restart for kubeadm mode - name: Preinstall | kube-controller configured @@ -66,6 +47,7 @@ get_mime: no register: kube_controller_set when: inventory_hostname in groups['kube_control_plane'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' + listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-controller-manager docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" @@ -77,6 +59,7 @@ - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_controller_set.stat.exists + listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-controller-manager crio/containerd shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" @@ -92,6 +75,7 @@ - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_controller_set.stat.exists + listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-apiserver docker shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" @@ -103,6 +87,7 @@ - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_apiserver_set.stat.exists + listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | restart kube-apiserver crio/containerd shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" @@ -118,6 +103,7 @@ - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - kube_apiserver_set.stat.exists + listen: Preinstall | propagate resolvconf to k8s components # When running this as the last phase ensure we wait for kube-apiserver to come up - name: Preinstall | wait for the apiserver to be running @@ -133,6 +119,8 @@ - inventory_hostname in groups['kube_control_plane'] - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' + - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos + listen: Preinstall | propagate resolvconf to k8s components - name: Preinstall | Restart systemd-resolved service: diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index 3d41050f5b5..e68f568372d 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -91,6 +91,7 @@ - configured_nameservers is defined - not (upstream_dns_servers is defined and upstream_dns_servers | length > 0) - not (disable_host_nameservers | default(false)) + - dns_mode in ['coredns', 'coredns_dual'] - name: NetworkManager | Check if host has NetworkManager # noqa command-instead-of-module - Should we use service_facts for this? @@ -220,7 +221,7 @@ nameserverentries: |- {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }} supersede_nameserver: - supersede domain-name-servers {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([])) | unique | join(', ') }}; + supersede domain-name-servers {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(', ') }}; when: not dns_early or dns_late # This task should run instead of the above task when cluster/nodelocal DNS hasn't diff --git a/roles/network_plugin/calico/handlers/main.yml b/roles/network_plugin/calico/handlers/main.yml index 7f998dba382..f5f5dc29ebc 100644 --- a/roles/network_plugin/calico/handlers/main.yml +++ b/roles/network_plugin/calico/handlers/main.yml @@ -1,16 +1,10 @@ --- -- name: Reset_calico_cni - command: /bin/true - when: calico_cni_config is defined - notify: - - Delete 10-calico.conflist - - Calico | delete calico-node docker containers - - Calico | delete calico-node crio/containerd containers - - name: Delete 10-calico.conflist file: path: /etc/cni/net.d/10-calico.conflist state: absent + listen: Reset_calico_cni + when: calico_cni_config is defined - name: Calico | delete calico-node docker containers shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" @@ -19,7 +13,10 @@ register: docker_calico_node_remove until: docker_calico_node_remove is succeeded retries: 5 - when: container_manager in ["docker"] + when: + - container_manager in ["docker"] + - calico_cni_config is defined + listen: Reset_calico_cni - name: Calico | delete calico-node crio/containerd containers shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name calico-node-* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"' @@ -28,4 +25,7 @@ register: crictl_calico_node_remove until: crictl_calico_node_remove is succeeded retries: 5 - when: container_manager in ["crio", "containerd"] + when: + - container_manager in ["crio", "containerd"] + - calico_cni_config is defined + listen: Reset_calico_cni diff --git a/roles/network_plugin/kube-router/handlers/main.yml b/roles/network_plugin/kube-router/handlers/main.yml index 0723dfd8ab0..ad5eb21401d 100644 --- a/roles/network_plugin/kube-router/handlers/main.yml +++ b/roles/network_plugin/kube-router/handlers/main.yml @@ -1,10 +1,4 @@ --- -- name: Reset_kube_router - command: /bin/true - notify: - - Kube-router | delete kube-router docker containers - - Kube-router | delete kube-router crio/containerd containers - - name: Kube-router | delete kube-router docker containers shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f" args: @@ -13,6 +7,7 @@ until: docker_kube_router_remove is succeeded retries: 5 when: container_manager in ["docker"] + listen: Reset_kube_router - name: Kube-router | delete kube-router crio/containerd containers shell: 'set -o pipefail && {{ bin_dir }}/crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"' @@ -22,3 +17,4 @@ until: crictl_kube_router_remove is succeeded retries: 5 when: container_manager in ["crio", "containerd"] + listen: Reset_kube_router diff --git a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 b/roles/network_plugin/kube-router/templates/kube-router.yml.j2 index 89ded6e4927..d868287d481 100644 --- a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 +++ b/roles/network_plugin/kube-router/templates/kube-router.yml.j2 @@ -178,7 +178,7 @@ metadata: namespace: kube-system rules: - apiGroups: - - "" + - "" resources: - namespaces - pods @@ -190,7 +190,7 @@ rules: - get - watch - apiGroups: - - "networking.k8s.io" + - "networking.k8s.io" resources: - networkpolicies verbs: @@ -198,13 +198,21 @@ rules: - get - watch - apiGroups: - - extensions + - extensions resources: - networkpolicies verbs: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/roles/network_plugin/macvlan/handlers/main.yml b/roles/network_plugin/macvlan/handlers/main.yml index aba4cbc0031..e4844c22174 100644 --- a/roles/network_plugin/macvlan/handlers/main.yml +++ b/roles/network_plugin/macvlan/handlers/main.yml @@ -1,10 +1,4 @@ --- -- name: Macvlan | restart network - command: /bin/true - notify: - - Macvlan | reload network - when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] - - name: Macvlan | reload network service: # noqa: jinja[spacing] @@ -18,3 +12,4 @@ {%- endif %} state: restarted when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and kube_network_plugin not in ['calico'] + listen: Macvlan | restart network