From 51069223f5fcd7e1f0540c160b38626fbce36ead Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Mon, 11 Dec 2023 16:56:17 +0100 Subject: [PATCH 1/8] Decouple kubespray-defaults from download (#10626) * Decouple role kubespray-defaults from download Avoids doing re-importing the download role on every invocation of kubespray-defaults (and skipping everything). This has a measurable effect on playbook performance. * Update docs refering to moved download defaults --- .ansible-lint-ignore | 2 +- Dockerfile | 2 +- contrib/offline/README.md | 2 +- contrib/offline/generate_list.sh | 4 ++-- docs/offline-environment.md | 2 +- extra_playbooks/migrate_openstack_provider.yml | 2 +- inventory/sample/group_vars/all/all.yml | 2 +- pipeline.Dockerfile | 4 ++-- .../defaults/main/checksums.yml | 0 .../defaults/main/download.yml} | 0 .../defaults/{main.yaml => main/main.yml} | 0 roles/kubespray-defaults/meta/main.yml | 6 ------ roles/kubespray-defaults/tasks/main.yaml | 2 +- scripts/download_hash.py | 4 ++-- scripts/download_hash.sh | 2 +- tests/scripts/check_readme_versions.sh | 4 ++-- 16 files changed, 16 insertions(+), 22 deletions(-) rename roles/{download => kubespray-defaults}/defaults/main/checksums.yml (100%) rename roles/{download/defaults/main/main.yml => kubespray-defaults/defaults/main/download.yml} (100%) rename roles/kubespray-defaults/defaults/{main.yaml => main/main.yml} (100%) delete mode 100644 roles/kubespray-defaults/meta/main.yml diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore index 03a371318b8..730b579e77d 100644 --- a/.ansible-lint-ignore +++ b/.ansible-lint-ignore @@ -5,4 +5,4 @@ roles/kubernetes/control-plane/defaults/main/main.yml jinja[spacing] roles/kubernetes/kubeadm/defaults/main.yml jinja[spacing] roles/kubernetes/node/defaults/main.yml jinja[spacing] roles/kubernetes/preinstall/defaults/main.yml jinja[spacing] -roles/kubespray-defaults/defaults/main.yaml jinja[spacing] +roles/kubespray-defaults/defaults/main/main.yml jinja[spacing] diff --git a/Dockerfile b/Dockerfile index 347c1af492f..3e81ed1d76b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN apt update -q \ MarkupSafe==2.1.3 \ ruamel.yaml==0.17.21 \ passlib==1.7.4 \ - && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \ + && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ diff --git a/contrib/offline/README.md b/contrib/offline/README.md index c059f5f8a20..ecf5ffc369c 100644 --- a/contrib/offline/README.md +++ b/contrib/offline/README.md @@ -27,7 +27,7 @@ manage-offline-container-images.sh register ## generate_list.sh -This script generates the list of downloaded files and the list of container images by `roles/download/defaults/main/main.yml` file. +This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/main/download.yml` file. Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files, all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template. diff --git a/contrib/offline/generate_list.sh b/contrib/offline/generate_list.sh index 646360f1da7..e298c8e852a 100755 --- a/contrib/offline/generate_list.sh +++ b/contrib/offline/generate_list.sh @@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd) TEMP_DIR="${CURRENT_DIR}/temp" REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}" -: ${DOWNLOAD_YML:="roles/download/defaults/main/main.yml"} +: ${DOWNLOAD_YML:="roles/kubespray-defaults/main/download.yml"} mkdir -p ${TEMP_DIR} @@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \ | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template # add kube-* images to images list template -# Those container images are downloaded by kubeadm, then roles/download/defaults/main/main.yml +# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/main/download.yml # doesn't contain those images. That is reason why here needs to put those images into the # list separately. KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy" diff --git a/docs/offline-environment.md b/docs/offline-environment.md index 743c2f7415f..33748a58cd3 100644 --- a/docs/offline-environment.md +++ b/docs/offline-environment.md @@ -95,7 +95,7 @@ If you use the settings like the one above, you'll need to define in your invent * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that the ones defined - in [Download's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/download/defaults/main/main.yml) + in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/main/download.yml) , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the same repository path, you won't have to override anything else. * `registry_addr`: Container image registry, but only have [domain or ip]:[port]. diff --git a/extra_playbooks/migrate_openstack_provider.yml b/extra_playbooks/migrate_openstack_provider.yml index a82a58710d1..bba3d918c1f 100644 --- a/extra_playbooks/migrate_openstack_provider.yml +++ b/extra_playbooks/migrate_openstack_provider.yml @@ -12,7 +12,7 @@ hosts: kube_control_plane[0] tasks: - name: Include kubespray-default variables - include_vars: ../roles/kubespray-defaults/defaults/main.yaml + include_vars: ../roles/kubespray-defaults/defaults/main/main.yml - name: Copy get_cinder_pvs.sh to master copy: src: get_cinder_pvs.sh diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index b93f1a3ac57..c7f76287002 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -57,7 +57,7 @@ loadbalancer_apiserver_healthcheck_port: 8081 # https_proxy: "" # https_proxy_cert_file: "" -## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy +## Refer to roles/kubespray-defaults/defaults/main/main.yml before modifying no_proxy # no_proxy: "" ## Some problems may occur when downloading files over https proxy due to ansible bug diff --git a/pipeline.Dockerfile b/pipeline.Dockerfile index eb4dcf6d630..4eba6a1a550 100644 --- a/pipeline.Dockerfile +++ b/pipeline.Dockerfile @@ -40,11 +40,11 @@ WORKDIR /kubespray RUN --mount=type=bind,target=./requirements.txt,src=./requirements.txt \ --mount=type=bind,target=./tests/requirements.txt,src=./tests/requirements.txt \ - --mount=type=bind,target=./roles/kubespray-defaults/defaults/main.yaml,src=./roles/kubespray-defaults/defaults/main.yaml \ + --mount=type=bind,target=./roles/kubespray-defaults/defaults/main/main.yml,src=./roles/kubespray-defaults/defaults/main/main.yml \ update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \ && pip install --no-compile --no-cache-dir pip -U \ && pip install --no-compile --no-cache-dir -r tests/requirements.txt \ - && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \ + && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \ && chmod a+x /usr/local/bin/kubectl \ diff --git a/roles/download/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml similarity index 100% rename from roles/download/defaults/main/checksums.yml rename to roles/kubespray-defaults/defaults/main/checksums.yml diff --git a/roles/download/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/download.yml similarity index 100% rename from roles/download/defaults/main/main.yml rename to roles/kubespray-defaults/defaults/main/download.yml diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main/main.yml similarity index 100% rename from roles/kubespray-defaults/defaults/main.yaml rename to roles/kubespray-defaults/defaults/main/main.yml diff --git a/roles/kubespray-defaults/meta/main.yml b/roles/kubespray-defaults/meta/main.yml deleted file mode 100644 index 88d70247e5f..00000000000 --- a/roles/kubespray-defaults/meta/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -dependencies: - - role: download - skip_downloads: true - tags: - - facts diff --git a/roles/kubespray-defaults/tasks/main.yaml b/roles/kubespray-defaults/tasks/main.yaml index 65040872612..e3defd27a8d 100644 --- a/roles/kubespray-defaults/tasks/main.yaml +++ b/roles/kubespray-defaults/tasks/main.yaml @@ -1,7 +1,7 @@ --- - name: Configure defaults debug: - msg: "Check roles/kubespray-defaults/defaults/main.yml" + msg: "Check roles/kubespray-defaults/defaults/main/main.yml" tags: - always diff --git a/scripts/download_hash.py b/scripts/download_hash.py index dfd84462b7b..6b76f21ccb0 100644 --- a/scripts/download_hash.py +++ b/scripts/download_hash.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # After a new version of Kubernetes has been released, -# run this script to update roles/download/defaults/main/main.yml +# run this script to update roles/kubespray-defaults/main/download.yml # with new hashes. import hashlib @@ -10,7 +10,7 @@ import requests from ruamel.yaml import YAML -MAIN_YML = "../roles/download/defaults/main/main.yml" +MAIN_YML = "../roles/kubespray-defaults/main/download.yml" def open_main_yaml(): yaml = YAML() diff --git a/scripts/download_hash.sh b/scripts/download_hash.sh index b37e466260d..94c18166146 100755 --- a/scripts/download_hash.sh +++ b/scripts/download_hash.sh @@ -6,7 +6,7 @@ if [[ ${DEBUG:-false} == "true" ]]; then set -o xtrace fi -checksums_file="$(git rev-parse --show-toplevel)/roles/download/defaults/main/checksums.yml" +checksums_file="$(git rev-parse --show-toplevel)/roles/kubespray-defaults/defaults/main/checksums.yml" downloads_folder=/tmp/kubespray_binaries function get_versions { diff --git a/tests/scripts/check_readme_versions.sh b/tests/scripts/check_readme_versions.sh index d796d9c437f..0f1836f9d3e 100755 --- a/tests/scripts/check_readme_versions.sh +++ b/tests/scripts/check_readme_versions.sh @@ -7,7 +7,7 @@ TARGET_COMPONENTS="containerd calico cilium flannel kube-ovn kube-router weave c cd $(dirname $0)/../../ echo checking kubernetes.. -version_from_default=$(grep "^kube_version:" ./roles/kubespray-defaults/defaults/main.yaml | awk '{print $2}' | sed s/\"//g) +version_from_default=$(grep "^kube_version:" ./roles/kubespray-defaults/defaults/main/main.yml | awk '{print $2}' | sed s/\"//g) version_from_readme=$(grep " \[kubernetes\]" ./README.md | awk '{print $3}') if [ "${version_from_default}" != "${version_from_readme}" ]; then echo "The version of kubernetes is different between main.yml(${version_from_default}) and README.md(${version_from_readme})." @@ -17,7 +17,7 @@ fi for component in $(echo ${TARGET_COMPONENTS}); do echo checking ${component}.. - version_from_default=$(grep "^$(echo ${component} | sed s/"-"/"_"/g)_version:" ./roles/download/defaults/main/main.yml | awk '{print $2}' | sed s/\"//g | sed s/^v//) + version_from_default=$(grep "^$(echo ${component} | sed s/"-"/"_"/g)_version:" ./roles/kubespray-defaults/defaults/main/download.yml | awk '{print $2}' | sed s/\"//g | sed s/^v//) if [ "${version_from_default}" = "" ]; then version_from_default=$(grep "^$(echo ${component} | sed s/"-"/"_"/g)_version:" ./roles/kubernetes/node/defaults/main.yml | awk '{print $2}' | sed s/\"//g | sed s/^v//) fi From 0fb404c775aff76916945ffb3d83dc77059ed7da Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 12 Dec 2023 11:22:29 +0100 Subject: [PATCH 2/8] etcd: use dynamic group for certs generation check (#10610) We take advantage of group_by to create the list of nodes needing new certs, instead of manually looping inside a Jinja template. This should make the role more readable and less susceptible to white space problems. --- roles/etcd/tasks/check_certs.yml | 53 ++++----------------------- roles/etcd/tasks/gen_certs_script.yml | 21 ++--------- roles/etcd/vars/main.yml | 10 +++++ 3 files changed, 21 insertions(+), 63 deletions(-) create mode 100644 roles/etcd/vars/main.yml diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index 2cb802d4e96..1611f9ec1d9 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -88,49 +88,12 @@ - kube_network_plugin != "calico" or calico_datastore == "etcd" - force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list -- name: "Check_certs | Set 'gen_master_certs' object to track whether member and admin certs exist on first etcd node" - set_fact: - # noqa: jinja[spacing] - gen_master_certs: |- - { - {% set etcd_members = groups['etcd'] -%} - {% set existing_certs = etcdcert_master.files | map(attribute='path') | list | sort %} - {% for host in etcd_members -%} - {% set member_cert = "%s/member-%s.pem" | format(etcd_cert_dir, host) %} - {% set member_key = "%s/member-%s-key.pem" | format(etcd_cert_dir, host) %} - {% set admin_cert = "%s/admin-%s.pem" | format(etcd_cert_dir, host) %} - {% set admin_key = "%s/admin-%s-key.pem" | format(etcd_cert_dir, host) %} - {% if force_etcd_cert_refresh -%} - "{{ host }}": True, - {% elif member_cert in existing_certs and member_key in existing_certs and admin_cert in existing_certs and admin_key in existing_certs -%} - "{{ host }}": False, - {% else -%} - "{{ host }}": True, - {% endif -%} - {% endfor %} - } - run_once: true - -- name: "Check_certs | Set 'gen_node_certs' object to track whether node certs exist on first etcd node" - set_fact: - # noqa: jinja[spacing] - gen_node_certs: |- - { - {% set k8s_nodes = groups['k8s_cluster'] -%} - {% set existing_certs = etcdcert_master.files | map(attribute='path') | list | sort %} - {% for host in k8s_nodes -%} - {% set host_cert = "%s/node-%s.pem" | format(etcd_cert_dir, host) %} - {% set host_key = "%s/node-%s-key.pem" | format(etcd_cert_dir, host) %} - {% if force_etcd_cert_refresh -%} - "{{ host }}": True, - {% elif host_cert in existing_certs and host_key in existing_certs -%} - "{{ host }}": False, - {% else -%} - "{{ host }}": True, - {% endif -%} - {% endfor %} - } - run_once: true +- name: "Check_certs | Set 'gen_*_certs' groups to track which nodes needs to have certs generated on first etcd node" + vars: + existing_certs: etcdcert_master.files | map(attribute='path') + ansible.builtin.group_by: + key: "gen_{{ item.node_type }}_certs_{{ force_etcd_cert_refresh or item.certs is not subset(existing_certs) }}" + loop: "{{ cert_files | dict2items(key_name='node_type', value_name='certs') }}" - name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match" set_fact: @@ -167,5 +130,5 @@ when: - etcd_member_requires_sync | default(false) or kubernetes_host_requires_sync | default(false) or - (inventory_hostname in gen_master_certs and gen_master_certs[inventory_hostname]) or - (inventory_hostname in gen_node_certs and gen_node_certs[inventory_hostname]) + 'gen_master_certs_True' in group_names or + 'gen_node_certs_True' in group_names diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml index 2ce3e14cf89..33e9d94c680 100644 --- a/roles/etcd/tasks/gen_certs_script.yml +++ b/roles/etcd/tasks/gen_certs_script.yml @@ -41,18 +41,8 @@ - name: Gen_certs | run cert generation script for etcd and kube control plane nodes command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" environment: - MASTERS: |- - {% for m in groups['etcd'] %} - {% if gen_master_certs[m] %} - {{ m }} - {% endif %} - {% endfor %} - HOSTS: |- - {% for h in groups['kube_control_plane'] %} - {% if gen_node_certs[h] %} - {{ h }} - {% endif %} - {% endfor %} + MASTERS: "{{ groups['gen_master_certs_True'] | ansible.builtin.intersect(groups['etcd']) | join(' ') }}" + HOSTS: "{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['kube_control_plane']) | join(' ') }}" run_once: yes delegate_to: "{{ groups['etcd'][0] }}" when: gen_certs | default(false) @@ -61,12 +51,7 @@ - name: Gen_certs | run cert generation script for all clients command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" environment: - HOSTS: |- - {% for h in groups['k8s_cluster'] %} - {% if gen_node_certs[h] %} - {{ h }} - {% endif %} - {% endfor %} + HOSTS: "{{ groups['gen_node_certs_True'] | ansible.builtin.intersect(groups['k8s_cluster']) | join(' ') }}" run_once: yes delegate_to: "{{ groups['etcd'][0] }}" when: diff --git a/roles/etcd/vars/main.yml b/roles/etcd/vars/main.yml new file mode 100644 index 00000000000..1e569042068 --- /dev/null +++ b/roles/etcd/vars/main.yml @@ -0,0 +1,10 @@ +--- +cert_files: + master: + - "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem" + - "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem" + - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + node: + - "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem" + - "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem" From 81a3f81aa1dbaf7e067544a81c1e85f9c16304f5 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 12 Dec 2023 11:22:38 +0100 Subject: [PATCH 3/8] Revert "Update etcd-servers for apiserver (#8253)" (#10652) This reverts commit ee0f1e9d58ed8bf1fd13ff1eb1527678fe4fa6da. Avoid restarting all api servers at once by changing their config. --- .../control-plane/tasks/kubeadm-fix-apiserver.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml b/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml index 8f2f38e26b5..5376aba81e6 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml @@ -15,10 +15,3 @@ - "Master | Restart kube-controller-manager" - "Master | Restart kube-scheduler" - "Master | reload kubelet" - -- name: Update etcd-servers for apiserver - lineinfile: - dest: "{{ kube_config_dir }}/manifests/kube-apiserver.yaml" - regexp: '^ - --etcd-servers=' - line: ' - --etcd-servers={{ etcd_access_addresses }}' - when: etcd_deployment_type != "kubeadm" From 8f2390a120bc067aaf7fca073806fbb795ec33be Mon Sep 17 00:00:00 2001 From: Louis Tu <92532497+tu1h@users.noreply.github.com> Date: Tue, 12 Dec 2023 20:47:27 +0800 Subject: [PATCH 4/8] Fix the path of download.yml (#10711) Signed-off-by: tu1h --- contrib/offline/README.md | 2 +- contrib/offline/generate_list.sh | 4 ++-- docs/offline-environment.md | 2 +- scripts/download_hash.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/offline/README.md b/contrib/offline/README.md index ecf5ffc369c..a7a87d8f021 100644 --- a/contrib/offline/README.md +++ b/contrib/offline/README.md @@ -27,7 +27,7 @@ manage-offline-container-images.sh register ## generate_list.sh -This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/main/download.yml` file. +This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file. Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files, all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template. diff --git a/contrib/offline/generate_list.sh b/contrib/offline/generate_list.sh index e298c8e852a..158ceb98a41 100755 --- a/contrib/offline/generate_list.sh +++ b/contrib/offline/generate_list.sh @@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd) TEMP_DIR="${CURRENT_DIR}/temp" REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}" -: ${DOWNLOAD_YML:="roles/kubespray-defaults/main/download.yml"} +: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"} mkdir -p ${TEMP_DIR} @@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \ | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template # add kube-* images to images list template -# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/main/download.yml +# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml # doesn't contain those images. That is reason why here needs to put those images into the # list separately. KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy" diff --git a/docs/offline-environment.md b/docs/offline-environment.md index 33748a58cd3..bbd412c5359 100644 --- a/docs/offline-environment.md +++ b/docs/offline-environment.md @@ -95,7 +95,7 @@ If you use the settings like the one above, you'll need to define in your invent * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that the ones defined - in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/main/download.yml) + in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/defaults/main/download.yml) , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the same repository path, you won't have to override anything else. * `registry_addr`: Container image registry, but only have [domain or ip]:[port]. diff --git a/scripts/download_hash.py b/scripts/download_hash.py index 6b76f21ccb0..86365e33323 100644 --- a/scripts/download_hash.py +++ b/scripts/download_hash.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # After a new version of Kubernetes has been released, -# run this script to update roles/kubespray-defaults/main/download.yml +# run this script to update roles/kubespray-defaults/defaults/main/download.yml # with new hashes. import hashlib @@ -10,7 +10,7 @@ import requests from ruamel.yaml import YAML -MAIN_YML = "../roles/kubespray-defaults/main/download.yml" +MAIN_YML = "../roles/kubespray-defaults/defaults/main/download.yml" def open_main_yaml(): yaml = YAML() From 8abf49ae13242b281da06da2728401f626ab5c23 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Tue, 12 Dec 2023 14:38:36 +0100 Subject: [PATCH 5/8] Disable podCIDR allocation from control-plane when using calico (#10639) * Disable control plane allocating podCIDR for nodes when using calico Calico does not use the .spec.podCIDR field for its IP address management. Furthermore, it can false positives from the kube controller manager if kube_network_node_prefix and calico_pool_blocksize are unaligned, which is the case with the default shipped by kubespray. If the subnets obtained from using kube_network_node_prefix are bigger, this would result at some point in the control plane thinking it does not have subnets left for a new node, while calico will work without problems. Explicitely set a default value of false for calico_ipam_host_local to facilitate its use in templates. * Don't default to kube_network_node_prefix for calico_pool_blocksize They have different semantics: kube_network_node_prefix is intended to be the size of the subnet for all pods on a node, while there can be more than on calico block of the specified size (they are allocated on demand). Besides, this commit does not actually change anything, because the current code is buggy: we don't ever default to kube_network_node_prefix, since the variable is defined in the role defaults. --- roles/kubernetes/control-plane/meta/main.yml | 1 + .../templates/kubeadm-config.v1beta3.yaml.j2 | 4 ++++ roles/network_plugin/calico/meta/main.yml | 3 +++ roles/network_plugin/calico/tasks/check.yml | 2 +- roles/network_plugin/calico/tasks/install.yml | 4 ++-- .../network_plugin/calico/templates/calico-config.yml.j2 | 2 +- .../{calico => calico_defaults}/defaults/main.yml | 8 ++++++-- 7 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 roles/network_plugin/calico/meta/main.yml rename roles/network_plugin/{calico => calico_defaults}/defaults/main.yml (97%) diff --git a/roles/kubernetes/control-plane/meta/main.yml b/roles/kubernetes/control-plane/meta/main.yml index 2657006e020..7d793f92f39 100644 --- a/roles/kubernetes/control-plane/meta/main.yml +++ b/roles/kubernetes/control-plane/meta/main.yml @@ -9,3 +9,4 @@ dependencies: when: - etcd_deployment_type == "kubeadm" - not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos) + - role: network_plugin/calico_defaults diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index dfccb20b273..cbb22182315 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -295,11 +295,15 @@ controllerManager: cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}" {% endif %} service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}" +{% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %} + allocate-node-cidrs: "false" +{% else %} {% if enable_dual_stack_networks %} node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}" node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}" {% else %} node-cidr-mask-size: "{{ kube_network_node_prefix }}" +{% endif %} {% endif %} profiling: "{{ kube_profiling }}" terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" diff --git a/roles/network_plugin/calico/meta/main.yml b/roles/network_plugin/calico/meta/main.yml new file mode 100644 index 00000000000..15e9b8c408d --- /dev/null +++ b/roles/network_plugin/calico/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: network_plugin/calico_defaults diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml index 2138be87e29..de67b154e87 100644 --- a/roles/network_plugin/calico/tasks/check.yml +++ b/roles/network_plugin/calico/tasks/check.yml @@ -168,7 +168,7 @@ - name: "Check if inventory match current cluster configuration" assert: that: - - calico_pool_conf.spec.blockSize | int == (calico_pool_blocksize | default(kube_network_node_prefix) | int) + - calico_pool_conf.spec.blockSize | int == calico_pool_blocksize | int - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet)) - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 4aac9df5b5c..6b293dcb083 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -223,7 +223,7 @@ "name": "{{ calico_pool_name }}", }, "spec": { - "blockSize": {{ calico_pool_blocksize | default(kube_network_node_prefix) }}, + "blockSize": {{ calico_pool_blocksize }}, "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ calico_ipip_mode }}", "vxlanMode": "{{ calico_vxlan_mode }}", @@ -274,7 +274,7 @@ "name": "{{ calico_pool_name }}-ipv6", }, "spec": { - "blockSize": {{ calico_pool_blocksize_ipv6 | default(kube_network_node_prefix_ipv6) }}, + "blockSize": {{ calico_pool_blocksize_ipv6 }}, "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}", "ipipMode": "{{ calico_ipip_mode_ipv6 }}", "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}", diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index 4012ef784c3..f1a3f2e76f1 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -54,7 +54,7 @@ data: "etcd_key_file": "{{ calico_cert_dir }}/key.pem", "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", {% endif %} - {% if calico_ipam_host_local is defined %} + {% if calico_ipam_host_local %} "ipam": { "type": "host-local", "subnet": "usePodCidr" diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico_defaults/defaults/main.yml similarity index 97% rename from roles/network_plugin/calico/defaults/main.yml rename to roles/network_plugin/calico_defaults/defaults/main.yml index b3c5f809cf3..8e716498bd1 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico_defaults/defaults/main.yml @@ -16,14 +16,14 @@ calico_vxlan_mode: Always # valid values are 'Always', 'Never' and 'CrossSubnet calico_cni_pool: true calico_cni_pool_ipv6: true -# add default ippool blockSize (defaults kube_network_node_prefix) +# add default ippool blockSize calico_pool_blocksize: 26 # Calico doesn't support ipip tunneling for the IPv6. calico_ipip_mode_ipv6: Never calico_vxlan_mode_ipv6: Never -# add default ipv6 ippool blockSize (defaults kube_network_node_prefix_ipv6) +# add default ipv6 ippool blockSize calico_pool_blocksize_ipv6: 122 # Calico network backend can be 'bird', 'vxlan' and 'none' @@ -161,6 +161,10 @@ calico_ipam_autoallocateblocks: true # Calico IPAM maxBlocksPerHost, default 0 calico_ipam_maxblocksperhost: 0 +# Calico host local IPAM (use node .spec.podCIDR) + +calico_ipam_host_local: false + # Calico apiserver (only with kdd) calico_apiserver_enabled: false From cb848fa7cb4c4e4599e344d746e263a1016a1b42 Mon Sep 17 00:00:00 2001 From: jandres - moscardo Date: Tue, 12 Dec 2023 14:51:26 +0100 Subject: [PATCH 6/8] New PR default node selector (#10607) --- docs/hardening.md | 5 +++++ roles/kubernetes/control-plane/defaults/main/main.yml | 2 ++ roles/kubernetes/control-plane/tasks/kubeadm-setup.yml | 9 +++++++++ .../control-plane/templates/podnodeselector.yaml.j2 | 2 ++ 4 files changed, 18 insertions(+) create mode 100644 roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 diff --git a/docs/hardening.md b/docs/hardening.md index b485c036cc6..77a010047ae 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -54,6 +54,11 @@ kube_apiserver_enable_admission_plugins: - PodNodeSelector - PodSecurity kube_apiserver_admission_control_config_file: true +# Creates config file for PodNodeSelector +# kube_apiserver_admission_plugins_needs_configuration: [PodNodeSelector] +# Define the default node selector, by default all the workloads will be scheduled on nodes +# with label network=srv1 +# kube_apiserver_admission_plugins_podnodeselector_default_node_selector: "network=srv1" # EventRateLimit plugin configuration kube_apiserver_admission_event_rate_limits: limit_1: diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 22f4886cb2a..19503817069 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -141,6 +141,8 @@ kube_webhook_token_auth_url_skip_tls_verify: false kube_webhook_authorization: false kube_webhook_authorization_url_skip_tls_verify: false +# Default podnodeselector +kube_apiserver_admission_plugins_podnodeselector_default_node_selector: "" ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 57baf411a30..dbc38ad818d 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -108,6 +108,15 @@ - item in kube_apiserver_admission_plugins_needs_configuration loop: "{{ kube_apiserver_enable_admission_plugins }}" +- name: Kubeadm | Configure default cluster podnodeslector + template: + src: "podnodeselector.yaml.j2" + dest: "{{ kube_config_dir }}/admission-controls/podnodeselector.yaml" + mode: 0640 + when: + - kube_apiserver_admission_plugins_podnodeselector_default_node_selector is defined + - kube_apiserver_admission_plugins_podnodeselector_default_node_selector | length > 0 + - name: Kubeadm | Check apiserver.crt SANs vars: apiserver_ips: "{{ apiserver_sans | map('ansible.utils.ipaddr') | reject('equalto', False) | list }}" diff --git a/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 b/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 new file mode 100644 index 00000000000..a44457f37ca --- /dev/null +++ b/roles/kubernetes/control-plane/templates/podnodeselector.yaml.j2 @@ -0,0 +1,2 @@ +podNodeSelectorPluginConfig: + clusterDefaultNodeSelector: {{ kube_apiserver_admission_plugins_podnodeselector_default_node_selector }} From ccb742c7ab17c82c7ef0edd8b6b06231dac12bcd Mon Sep 17 00:00:00 2001 From: Mohamed Omar Zaian Date: Tue, 12 Dec 2023 17:53:32 +0100 Subject: [PATCH 7/8] [containerd] add hashes for versions 1.6.25-26 and 1.7.9-11 make v1.7.11 default (#10671) --- README.md | 2 +- .../defaults/main/checksums.yml | 19 +++++++++++++++++++ .../defaults/main/download.yml | 4 ++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 70381b6fa65..d2cf298f027 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ Note: Upstart/SysV init based OS types are not supported. - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.4 - [etcd](https://github.com/etcd-io/etcd) v3.5.9 - [docker](https://www.docker.com/) v20.10 (see note) - - [containerd](https://containerd.io/) v1.7.8 + - [containerd](https://containerd.io/) v1.7.11 - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0 diff --git a/roles/kubespray-defaults/defaults/main/checksums.yml b/roles/kubespray-defaults/defaults/main/checksums.yml index f194618804e..e42029d8bc7 100644 --- a/roles/kubespray-defaults/defaults/main/checksums.yml +++ b/roles/kubespray-defaults/defaults/main/checksums.yml @@ -1230,6 +1230,9 @@ nerdctl_archive_checksums: 1.0.0: 2fb02e629a4be16b194bbfc64819132a72ede1f52596bd8e1ec2beaf7c28c117 containerd_archive_checksums: arm: + 1.7.11: 0 + 1.7.10: 0 + 1.7.9: 0 1.7.8: 0 1.7.7: 0 1.7.6: 0 @@ -1239,6 +1242,7 @@ containerd_archive_checksums: 1.7.2: 0 1.7.1: 0 1.7.0: 0 + 1.6.25: 0 1.6.24: 0 1.6.23: 0 1.6.22: 0 @@ -1270,6 +1274,9 @@ containerd_archive_checksums: 1.5.15: 0 1.5.14: 0 arm64: + 1.7.11: 5eae27cce38a14be5390d4035127aa11416bc5ae592a9ff25b11870872ce1159 + 1.7.10: 0667b12a04a896a61cf508a4a77190c280f4a1fa35f38c8a4ba63f605b5ec375 + 1.7.9: 09ca326dee14e00c439137071747c15cc280480e2c26c1e82698c992dd1889c6 1.7.8: 3fc551e8f51150804d80cc1958a271bd2252b6334f0355244d0faa5da7fa55d1 1.7.7: 0a104f487193665d2681fcb5ed83f2baa5f97849fe2661188da835c9d4eaf9e3 1.7.6: d844a1c8b993e7e9647f73b9814567004dce1287c0529ce55c50519490eafcce @@ -1279,6 +1286,8 @@ containerd_archive_checksums: 1.7.2: d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6 1.7.1: 1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b 1.7.0: e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2 + 1.6.26: 177bed65b6425255bacbe48d99ea7aa5209d381576962c0962dc8615ef16c5c5 + 1.6.25: 4948677cfc5f98a1d5d46cec90d6d6f84f6b27cd6d28fd87f7f5936d61580ceb 1.6.24: 1d741e9e2d907f02a8b2a46034a28ff9aacdba88c485cef2f4bad18be9ea23ba 1.6.23: ea7afb82dc5789307e684ef9b4a55ce1ee9a05dc02c2118df640b01207208c45 1.6.22: 7882d6e7f4e97dcba041c37592c4cb9e7a5b4d972380c74d959e388b12d57d01 @@ -1310,6 +1319,9 @@ containerd_archive_checksums: 1.5.15: 0 1.5.14: 0 amd64: + 1.7.11: d66161d54546fad502fd50a13fcb79efff033fcd895adc9c44762680dcde4e69 + 1.7.10: eacb0296bff2ae5225a18492dcb32fb28ad4a1fe0a39ea9073367c7e43dc5838 + 1.7.9: ccd5b434393666f6ebbc90eea959ffd3e61958a1e3e1cc830a678f040142d4b0 1.7.8: 5f1d017a5a7359514d6187d6656e88fb2a592d107e6298db7963dbddb9a111d9 1.7.7: 371de359d6102c51f6ee2361d08297948d134ce7379e01cb965ceeffa4365fba 1.7.6: 58408cfa025003e671b0af72183b963363d519543d0d0ba186037e9c57489ffe @@ -1319,6 +1331,8 @@ containerd_archive_checksums: 1.7.2: 2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7 1.7.1: 9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85 1.7.0: b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0 + 1.6.26: fa806d3e945a8ad25aa1f8123a98524768ead83af2ed1ab3d922d2dd5fe6b14c + 1.6.25: 878b331b5fa65df3d33c68ee355724de0044c25071486086409b374a9c62d145 1.6.24: a56fac5ba03c3d6f74ceae14abdc9fafabcba900105e9890c0ac895cc00164ad 1.6.23: bcf16bb63a295721a2603e9a56602c5d18e5443df04a9f2c1ca5328f41556fcc 1.6.22: 5671eb4eba97f0ec98223c84401c9aeb21d0ef16ac3ece3eb8fadd46174d7eab @@ -1350,6 +1364,9 @@ containerd_archive_checksums: 1.5.15: 0d09043be08dcf6bf136aa78bfd719e836cf9f9679afa4db0b6e4d478e396528 1.5.14: 8513ead11aca164b7e70bcea0429b4e51dad836b6383b806322e128821aaebbd ppc64le: + 1.7.11: 6f91c5dabdccd1fc75aae8687381bb185b9eb4200beb29d0993dea8175f5fa61 + 1.7.10: 15a5191bf7c555956a8565d8786399d51b13f2718d59b1a5b2bd380fc420bf8a + 1.7.9: 174b8af2d878ad8410205b9ba44fa8d2a9683a521abf13f168f67b7f7375d5b3 1.7.8: 2b563df9e1bddc96a99a023963c99b5faf3066d3fcbc23ff44ba24229e939444 1.7.7: 0335e7447ed84757489337686a709e95ffa379a8780f238725abb10facaeaa7f 1.7.6: 956fadb01b35c3214f2b6f82abc0dda3e1b754cb223cd24e818334b08cb09fb2 @@ -1359,6 +1376,8 @@ containerd_archive_checksums: 1.7.2: cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e 1.7.1: 17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342 1.7.0: 051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889 + 1.6.26: 75fb01a4bd3bcd16263c2f833b8e7081356e2e390dd7eb0710232cb04dac5a01 + 1.6.25: 3ddcc1739ffeb3e1df786d45518a01d93a5cef243eb6dee61cbdd4cd110bc723 1.6.24: abff9e7ec4cc21d19150d2bc55fc89cf53dc03c002cdaf5016ee82aedead9b03 1.6.23: 1099fce4a4dfc78712cc1c19be6d9f80e9321f513834dba0b2418bd5c78ad398 1.6.22: 0f8647aedd96174704a63a17b1a7cf4c4c5c2fc066606b1a419b2a860b754bfb diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 8ff47f666f0..d30c6efe399 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -76,11 +76,11 @@ image_arch: "{{ host_architecture | default('amd64') }}" # Versions kubeadm_version: "{{ kube_version }}" crun_version: 1.8.5 -runc_version: v1.1.9 +runc_version: v1.1.10 kata_containers_version: 3.1.3 youki_version: 0.1.0 gvisor_version: 20230807 -containerd_version: 1.7.8 +containerd_version: 1.7.11 cri_dockerd_version: 0.3.4 # this is relevant when container_manager == 'docker' From 45a070f1ba7743daee07c7fa4a7b0ea56c122ab6 Mon Sep 17 00:00:00 2001 From: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:04:38 +0100 Subject: [PATCH 8/8] chore(Dockerfile): python requirements file (#10700) Signed-off-by: Maxime Leroy <19607336+maxime1907@users.noreply.github.com> --- Dockerfile | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e81ed1d76b..e343e22922e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,9 @@ COPY extra_playbooks ./extra_playbooks COPY playbooks ./playbooks COPY plugins ./plugins -RUN apt update -q \ - && apt install -yq --no-install-recommends \ +RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \ + apt update -q \ + && apt install -yq --no-install-recommends \ curl \ python3 \ python3-pip \ @@ -27,16 +28,7 @@ RUN apt update -q \ vim \ rsync \ openssh-client \ - && pip install --no-compile --no-cache-dir \ - ansible==7.6.0 \ - ansible-core==2.14.6 \ - cryptography==41.0.1 \ - jinja2==3.1.2 \ - netaddr==0.8.0 \ - jmespath==1.0.1 \ - MarkupSafe==2.1.3 \ - ruamel.yaml==0.17.21 \ - passlib==1.7.4 \ + && pip install --no-compile --no-cache-dir -r requirements.txt \ && KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \ && curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \ && echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \