diff --git a/Dockerfile b/Dockerfile index 0e6a251e07b..1d2967db239 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Use imutable image tags rather than mutable tags (like ubuntu:18.04) FROM ubuntu:bionic-20200807 -ENV KUBE_VERSION=v1.19.8 +ENV KUBE_VERSION=v1.19.10 RUN mkdir /kubespray WORKDIR /kubespray diff --git a/README.md b/README.md index e9c2ce25531..9ea37346e33 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ vagrant up - **CentOS/RHEL** 7, 8 (experimental: see [centos 8 notes](docs/centos8.md)) - **Fedora** 32, 33 - **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md)) -- **openSUSE** Leap 42.3/Tumbleweed +- **openSUSE** Leap 15.x/Tumbleweed - **Oracle Linux** 7, 8 (experimental: [centos 8 notes](docs/centos8.md) apply) Note: Upstart/SysV init based OS types are not supported. @@ -116,20 +116,20 @@ Note: Upstart/SysV init based OS types are not supported. ## Supported Components - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.19.8 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.19.10 - [etcd](https://github.com/coreos/etcd) v3.4.13 - [docker](https://www.docker.com/) v19.03 (see note) - [containerd](https://containerd.io/) v1.3.9 - [cri-o](http://cri-o.io/) v1.19 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) - Network Plugin - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.0 - - [calico](https://github.com/projectcalico/calico) v3.16.6 + - [calico](https://github.com/projectcalico/calico) v3.16.9 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - - [cilium](https://github.com/cilium/cilium) v1.8.6 + - [cilium](https://github.com/cilium/cilium) v1.8.8 - [flanneld](https://github.com/coreos/flannel) v0.13.0 - - [kube-ovn](https://github.com/alauda/kube-ovn) v1.5.2 + - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.1 - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.1.1 - - [multus](https://github.com/intel/multus-cni) v3.6.0 + - [multus](https://github.com/intel/multus-cni) v3.7.0 - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.1.0 - [weave](https://github.com/weaveworks/weave) v2.7.0 - Application diff --git a/Vagrantfile b/Vagrantfile index 805044ad44e..1a0ac93dc00 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,7 +28,7 @@ SUPPORTED_OS = { "centos8-bento" => {box: "bento/centos-8", user: "vagrant"}, "fedora32" => {box: "fedora/32-cloud-base", user: "vagrant"}, "fedora33" => {box: "fedora/33-cloud-base", user: "vagrant"}, - "opensuse" => {box: "bento/opensuse-leap-15.1", user: "vagrant"}, + "opensuse" => {box: "bento/opensuse-leap-15.2", user: "vagrant"}, "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"}, "oraclelinux" => {box: "generic/oracle7", user: "vagrant"}, "oraclelinux8" => {box: "generic/oracle8", user: "vagrant"}, diff --git a/ansible_version.yml b/ansible_version.yml index b4631223aad..268136df285 100644 --- a/ansible_version.yml +++ b/ansible_version.yml @@ -15,3 +15,18 @@ - ansible_version.string is version(maximal_ansible_version, "<") tags: - check + + - name: "Check that python netaddr is installed" + assert: + msg: "Python netaddr is not present" + that: "'127.0.0.1' | ipaddr" + tags: + - check + + # CentOS 7 provides too old jinja version + - name: "Check that jinja is not too old (install via pip)" + assert: + msg: "Your Jinja version is too old, install via pip" + that: "{% set test %}It works{% endset %}{{ test == 'It works' }}" + tags: + - check diff --git a/cluster.yml b/cluster.yml index b7c89dc79f9..f278db68757 100644 --- a/cluster.yml +++ b/cluster.yml @@ -100,7 +100,6 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray-defaults } - - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] } - hosts: kube-master diff --git a/docs/offline-environment.md b/docs/offline-environment.md index 23e8a33711b..3655e15879a 100644 --- a/docs/offline-environment.md +++ b/docs/offline-environment.md @@ -28,6 +28,8 @@ cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_ar crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" # If using Calico calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" +# If using Calico with kdd +calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz" # CentOS/Redhat ## Docker / Containerd diff --git a/docs/opensuse.md b/docs/opensuse.md index 54aa57c1fa2..94bb6a35959 100644 --- a/docs/opensuse.md +++ b/docs/opensuse.md @@ -1,4 +1,4 @@ -# openSUSE Leap 15.0 and Tumbleweed +# openSUSE Leap 15.2 and Tumbleweed openSUSE Leap installation Notes: diff --git a/docs/upgrades.md b/docs/upgrades.md index b42dd69d0b1..4cac24c32be 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -284,20 +284,6 @@ follows: * kube-apiserver, kube-scheduler, and kube-controller-manager * Add-ons (such as KubeDNS) -## Upgrade considerations - -Kubespray supports rotating certificates used for etcd and Kubernetes -components, but some manual steps may be required. If you have a pod that -requires use of a service token and is deployed in a namespace other than -`kube-system`, you will need to manually delete the affected pods after -rotating certificates. This is because all service account tokens are dependent -on the apiserver token that is used to generate them. When the certificate -rotates, all service account tokens must be rotated as well. During the -kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and -recreated. All other invalidated service account tokens are cleaned up -automatically, but other pods are not deleted out of an abundance of caution -for impact to user deployed pods. - ### Component-based upgrades A deployer may want to upgrade specific components in order to minimize risk diff --git a/inventory/local/hosts.ini b/inventory/local/hosts.ini index 7834d27c0ea..425ad23ef38 100644 --- a/inventory/local/hosts.ini +++ b/inventory/local/hosts.ini @@ -12,4 +12,3 @@ node1 [k8s-cluster:children] kube-node kube-master -calico-rr diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index 291ce0b5185..57de2b7a228 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.19.8 +kube_version: v1.19.10 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) @@ -310,5 +310,8 @@ persistent_volumes_enabled: false ## Amount of time to retain events. (default 1h0m0s) event_ttl_duration: "1h0m0s" -## Force regeneration of kubernetes control plane certificates without the need of bumping the cluster version -force_certificate_regeneration: false + +## Automatically renew K8S control plane certificates on first Monday of each month +auto_renew_certificates: false +# First Monday of each month +# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube-master'].index(inventory_hostname) }}0:00" diff --git a/inventory/sample/group_vars/k8s-cluster/offline.yml b/inventory/sample/group_vars/k8s-cluster/offline.yml index 20b236b634e..e307211f617 100644 --- a/inventory/sample/group_vars/k8s-cluster/offline.yml +++ b/inventory/sample/group_vars/k8s-cluster/offline.yml @@ -32,6 +32,8 @@ # [Optional] Calico: If using Calico network plugin # calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" +# [Optional] Calico with kdd: If using Calico network plugin with kdd datastore +# calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz" ## CentOS/Redhat ### For EL7, base and extras repo must be available, for EL8, baseos and appstream diff --git a/requirements.txt b/requirements.txt index 66284a6558d..06eacf1da7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -ansible==2.9.17 -jinja2==2.11.1 +ansible==2.9.18 +cryptography==2.8 +jinja2==2.11.3 netaddr==0.7.19 pbr==5.4.4 jmespath==0.9.5 diff --git a/roles/bastion-ssh-config/tasks/main.yml b/roles/bastion-ssh-config/tasks/main.yml index cf558087cd6..c6158dbce7c 100644 --- a/roles/bastion-ssh-config/tasks/main.yml +++ b/roles/bastion-ssh-config/tasks/main.yml @@ -1,7 +1,8 @@ --- -- name: set bastion host IP +- name: set bastion host IP and port set_fact: bastion_ip: "{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}" + bastion_port: "{{ hostvars[groups['bastion'][0]]['ansible_port'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_port']) | d(22) }}" delegate_to: localhost connection: local diff --git a/roles/bastion-ssh-config/templates/ssh-bastion.conf b/roles/bastion-ssh-config/templates/ssh-bastion.conf index b460d6ba372..bd5f49c45ef 100644 --- a/roles/bastion-ssh-config/templates/ssh-bastion.conf +++ b/roles/bastion-ssh-config/templates/ssh-bastion.conf @@ -15,4 +15,4 @@ Host {{ bastion_ip }} ControlPersist 5m Host {{ vars['hosts'] }} - ProxyCommand ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} + ProxyCommand ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -p {{ bastion_port }} {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} diff --git a/roles/bootstrap-os/tasks/bootstrap-centos.yml b/roles/bootstrap-os/tasks/bootstrap-centos.yml index eb433720fab..8ef433397f1 100644 --- a/roles/bootstrap-os/tasks/bootstrap-centos.yml +++ b/roles/bootstrap-os/tasks/bootstrap-centos.yml @@ -69,6 +69,9 @@ - name: Check presence of fastestmirror.conf stat: path: /etc/yum/pluginconf.d/fastestmirror.conf + get_attributes: no + get_checksum: no + get_mime: no register: fastestmirror # the fastestmirror plugin can actually slow down Ansible deployments diff --git a/roles/bootstrap-os/tasks/bootstrap-opensuse.yml b/roles/bootstrap-os/tasks/bootstrap-opensuse.yml index 54ce51242ca..5b2b6ab94e5 100644 --- a/roles/bootstrap-os/tasks/bootstrap-opensuse.yml +++ b/roles/bootstrap-os/tasks/bootstrap-opensuse.yml @@ -4,6 +4,9 @@ - name: Check that /etc/sysconfig/proxy file exists stat: path: /etc/sysconfig/proxy + get_attributes: no + get_checksum: no + get_mime: no register: stat_result - name: Create the /etc/sysconfig/proxy empty file diff --git a/roles/bootstrap-os/tasks/bootstrap-redhat.yml b/roles/bootstrap-os/tasks/bootstrap-redhat.yml index 5f70010e5af..5a2bbf55386 100644 --- a/roles/bootstrap-os/tasks/bootstrap-redhat.yml +++ b/roles/bootstrap-os/tasks/bootstrap-redhat.yml @@ -85,6 +85,9 @@ - name: Check presence of fastestmirror.conf stat: path: /etc/yum/pluginconf.d/fastestmirror.conf + get_attributes: no + get_checksum: no + get_mime: no register: fastestmirror # the fastestmirror plugin can actually slow down Ansible deployments diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index 1138f835a43..470e96d2e37 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -65,6 +65,7 @@ containerd_default_runtime: # type: io.containerd.kata.v2 # engine: "" # root: "" +# privileged_without_host_devices: true containerd_runtimes: [] containerd_untrusted_runtime_type: '' diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml index d8ac963213e..cd97f88d9ff 100644 --- a/roles/container-engine/containerd/tasks/main.yml +++ b/roles/container-engine/containerd/tasks/main.yml @@ -2,6 +2,9 @@ - name: check if fedora coreos stat: path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no register: ostree - name: set is_ostree diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index 9ed28ef5392..0f0f80ba0d6 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -42,6 +42,7 @@ disabled_plugins = ["restart"] runtime_type = "{{ containerd_default_runtime.type }}" runtime_engine = "{{ containerd_default_runtime.engine }}" runtime_root = "{{ containerd_default_runtime.root }}" + privileged_without_host_devices = {{ containerd_default_runtime.privileged_without_host_devices|default(false)|lower }} {% if kata_containers_enabled %} [plugins.cri.containerd.runtimes.kata-qemu] @@ -55,6 +56,7 @@ disabled_plugins = ["restart"] runtime_type = "{{ runtime.type }}" runtime_engine = "{{ runtime.engine }}" runtime_root = "{{ runtime.root }}" + privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }} {% endfor %} [plugins.cri.containerd.untrusted_workload_runtime] diff --git a/roles/container-engine/cri-o/tasks/crio_repo.yml b/roles/container-engine/cri-o/tasks/crio_repo.yml index 4993bedaa5e..52cd548886c 100644 --- a/roles/container-engine/cri-o/tasks/crio_repo.yml +++ b/roles/container-engine/cri-o/tasks/crio_repo.yml @@ -14,6 +14,7 @@ until: apt_key_download is succeeded retries: 4 delay: "{{ retry_stagger | d(3) }}" + environment: "{{ proxy_env }}" - name: Add CRI-O kubic apt repo apt_repository: diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml index 63a5e4018de..c152a5232ec 100644 --- a/roles/container-engine/cri-o/tasks/main.yaml +++ b/roles/container-engine/cri-o/tasks/main.yaml @@ -2,6 +2,9 @@ - name: check if fedora coreos stat: path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no register: ostree - name: set is_ostree @@ -94,6 +97,9 @@ - name: Check if already installed stat: path: "/bin/crio" + get_attributes: no + get_checksum: no + get_mime: no register: need_bootstrap_crio when: is_ostree diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index 7cb9f93b63a..493744c2082 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -293,6 +293,7 @@ pinns_path = "" runtime_path = "{{ runtime.path }}" runtime_type = "{{ runtime.type }}" runtime_root = "{{ runtime.root }}" +privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }} {% endfor %} # Kata Containers with the Firecracker VMM diff --git a/roles/container-engine/crun/tasks/main.yml b/roles/container-engine/crun/tasks/main.yml index d580f32cfdc..aff1bfec1ae 100644 --- a/roles/container-engine/crun/tasks/main.yml +++ b/roles/container-engine/crun/tasks/main.yml @@ -9,6 +9,9 @@ - name: Check if binary exists stat: path: "{{ crun_bin_dir }}/crun" + get_attributes: no + get_checksum: no + get_mime: no register: crun_stat # TODO: use download_file.yml diff --git a/roles/container-engine/docker/tasks/main.yml b/roles/container-engine/docker/tasks/main.yml index b0ece93f013..13903e48887 100644 --- a/roles/container-engine/docker/tasks/main.yml +++ b/roles/container-engine/docker/tasks/main.yml @@ -2,6 +2,9 @@ - name: check if fedora coreos stat: path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no register: ostree - name: set is_ostree diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index d7961a6df0e..fe08e368a82 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -64,7 +64,7 @@ quay_image_repo: "quay.io" # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v3.16.6" +calico_version: "v3.16.9" calico_ctl_version: "{{ calico_version }}" calico_cni_version: "{{ calico_version }}" calico_policy_version: "{{ calico_version }}" @@ -77,13 +77,13 @@ cni_version: "v0.9.0" weave_version: 2.7.0 pod_infra_version: "3.3" -cilium_version: "v1.8.6" -kube_ovn_version: "v1.5.2" +cilium_version: "v1.8.8" +kube_ovn_version: "v1.6.1" kube_router_version: "v1.1.1" -multus_version: "v3.6" +multus_version: "v3.7" ovn4nfv_ovn_image_version: "v1.0.0" ovn4nfv_k8s_plugin_image_version: "v1.1.0" -helm_version: "v3.5.2" +helm_version: "v3.5.3" # Get kubernetes major version (i.e. 1.17.4 => 1.17) kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}" @@ -99,6 +99,7 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" +calico_crds_download_url: "https://github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz" crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" @@ -117,11 +118,16 @@ crictl_checksums: # Kubernetes versions above Kubespray's current target version are untested and should be used with caution. kubelet_checksums: arm: + v1.21.0: c2a3c926842c892ca3a124fb0526970e6ce1d92cb3ac35028b251a5672bb1af1 + v1.20.6: a7f89bc0810ad0cfcf8b8235b88ed84f0eb39a164ea536595d7fe461548e985b + v1.20.5: cd04519bc8271a990380b62bd5b397ed5e21f9d81d731792b53453baea3d7cd5 v1.20.4: 6c2dbd275d0413124bb56f347821aa11003749c3e5d31ebddec64b14dc74ea18 v1.20.3: 64a2f413987483e131a6cbbf641c3f8a4d506d5373c04d0765ccf96b4886597d v1.20.2: fa4e9891c25830cd2b5ed52b524878d04554094a0578e039c2b3c28ea85a5b12 v1.20.1: 576d89284c8fa758d7d34d789444e9a5a03f24b6400cc94bf103c1c796898dc7 v1.20.0: 132ea2f821ec1537af34b3a1414071416149972ce93619f71730c8dd741e357c + v1.19.10: 8512a832424dead065b28092d9e66eca17d1c8eba7af0994e7f7b287f5a76805 + v1.19.9: 0c2c22f226189764b7f4a73614e12ff0be150640f64b691327e925db125d8950 v1.19.8: 0a68c72574a96e7ca899610084659f156c1eeb312af7fc59aa549d47411d1dab v1.19.7: 59284dcf4ee3f62475d0d6d1070c64e0e5362280e8d70884513e1e1cee918cb0 v1.19.6: 59179a52615cb0811909558425e046cc9c5c051e3b55733b96e40a9aed0967de @@ -131,6 +137,7 @@ kubelet_checksums: v1.19.2: 631e686c34911a40a798817dcff89532c88bb649885f93ec66b339e227ebd974 v1.19.1: 3985c8d02c1c2f2016fceccd9cc14865e2d047f32c8f0b42aeedcc8450de572e v1.19.0: bb433ef7981297bdee6ffc4e23376e8db24a0e47321ebe94bf9d4b9f7a2f0e3b + v1.18.17: 6b413c325293a8319ec50e76b6043358304a28be17a36c0d16dbb666b93b28cf v1.18.16: e678fbe06c197c1a233ef3010e3e2ee4e6759d14e7247178dbe4c06843aa5c10 v1.18.15: 27c8d5ea1f837cb0148d829944d46b028ff9d9b9edf12cd39e1cb5f65fd9f41c v1.18.14: e380c14330c4f34b7682a19a493769a4ba7cf9132a5ae9d2ab4109a0ba60973b @@ -148,11 +155,16 @@ kubelet_checksums: v1.18.1: 04d8e0a080dcb23d579c69e769e75bd5abaa1977d43550ec891560d76f1f7f37 v1.18.0: 985c1a1b492ccc6e46e1cd454790dae539d5b93208efb05e35114f66a183de99 arm64: + v1.21.0: 17832b192be5ea314714f7e16efd5e5f65347974bbbf41def6b02f68931380c4 + v1.20.6: 6e7b44d1ca65f970b0646f7d093dcf0cfefc44d4a67f29d542fe1b7ca6dcf715 + v1.20.5: 2d1b6f73fd0373687c7dc6582fe5ebddfa26c3b9ef128ec3955517b9db4746f2 v1.20.4: 66bcdc7521e226e4acaa93c08e5ea7b2f57829e1a5b9decfd2b91d237e216e1d v1.20.3: 29fa80790edcdfc7389b4b88cf963ce29d82e5de39bdb70f7c6dff551299b6f3 v1.20.2: 2b2d8e567439500daa9be88c4b10ee6db7126be942ac94259b506152ee38da1e v1.20.1: 359025e549ec6e16ffa1aead7312934233a983a918f54686a2c34ff9a6ba6016 v1.20.0: 47ab6c4273fc3bb0cb8ec9517271d915890c5a6b0e54b2991e7a8fbbe77b06e4 + v1.19.10: c57a464df1e0a56bcade6863f1f80def416eba5d06892644c633eb8c95444840 + v1.19.9: 796f080c53ec50b11152558b4a744432349b800e37b80516bcdc459152766a4f v1.19.8: a00146c16266d54f961c40fc67f92c21967596c2d730fa3dc95868d4efb44559 v1.19.7: 473016cf1165ab5c705bd29c65451984e48deade6ff96bf861c46e15fc52ba3f v1.19.6: e168d57a0a04abe58db13aaf4f54ba691115d0b1a743d549136794bbecde7dc8 @@ -162,6 +174,7 @@ kubelet_checksums: v1.19.2: 86b9336aa7f6215e6f9b387bb82105657668aa8a38b0e0f7c2e647ef45c1b723 v1.19.1: 143bed1f04cba4e6749a72abf9258d19f31e1a310f94bd041cd30ce62a1f73ff v1.19.0: d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf + v1.18.17: 16cb849c8b80912e246e0cb3c7014aeb12cd6dfa9280bac717d8d941e43177f3 v1.18.16: 712fd38b397a02058df7836f34bae3a52b5cca78d7d118be2d9a4d6e1d726f0a v1.18.15: c74f46e87aae7b9bb590319632fcb106b5efd998046dd47634c89bbb875d9fb6 v1.18.14: b4ca12f6b1a5f7c838831eb3c9152ccf0f53b83a3492c812da6ac680959a4801 @@ -179,11 +192,16 @@ kubelet_checksums: v1.18.1: 2181cde9e6b24055d262b78758b365363273896968df673eb13d4f17a4f69c4a v1.18.0: db91a26f8baa2bce017172305e717e77be5cfc4272592be8cb0155e1cfa7719e amd64: + v1.21.0: 681c81b7934ae2bf38b9f12d891683972d1fbbf6d7d97e50940a47b139d41b35 + v1.20.6: 7688a663dd06222d337c8fdb5b05e1d9377e6d64aa048c6acf484bc3f2a596a8 + v1.20.5: 8a07891153398585a749285bb3caa2e16f0802d03ffdd5054a7ef1ef32989bea v1.20.4: a9f28ac492b3cbf75dee284576b2e1681e67170cd36f3f5cdc31495f1bdbf809 v1.20.3: 8efddc4a7cc63c18f1d14bcb0879b460bbbc7358304ca10dff50907c03f04c81 v1.20.2: 2447da8e0729ac6e4bdcb1ea7d41c8ae6e5c8565edea189434286697ccdc8d85 v1.20.1: 2970974fa56ee90b76c7f3f8b0075f0719bb9d645aacfcef85238b68972aa9c3 v1.20.0: ff2422571c4c1e9696e367f5f25466b96fb6e501f28aed29f414b1524a52dea0 + v1.19.10: cf2f038f1012d5883407bd82e8f0d5cd0e27c8074ab821b296c6a3214aec917e + v1.19.9: 296e72c395f030209e712167fc5f6d2fdfe3530ca4c01bcd9bfb8c5e727c3d8d v1.19.8: f5cad5260c29584dd370ec13e525c945866957b1aaa719f1b871c31dc30bcb3f v1.19.7: d8b296825f6dd7a17287b73cd6604d32210abbba86c88fb68c1b1c5016906c54 v1.19.6: 8162fa58f7fbb9b1f07b2b666c9759ad5c30950bc9f6f2084de1c0d9bc64a4c0 @@ -193,6 +211,7 @@ kubelet_checksums: v1.19.2: 7ff6d3663e8de0c654909e7a279e386286aa7ed3fc262d021bed77c92d62780f v1.19.1: 2ca2a3104d4cce26db128e3a0b7a042385df4f2c51bdbe740e067fdfaa2fcdd1 v1.19.0: 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab + v1.18.17: 56348ecef546e0a301dc36193b0c8b13d3b4bea3115e167f16a5c75ffc5e11bd v1.18.16: b6e97539ef91523f1238d6bdcddc97dc0353396519a60c1a912bfabd9356a67b v1.18.15: 2d079bc9f7ac2c2d1f86df842df55f2ec8e74e01edc347994ccf1a054b9e6077 v1.18.14: 1876c3aad83beeea1bc949fe6121d0d0d9002a0e026c15ccb568d8c5e748fba6 @@ -211,11 +230,16 @@ kubelet_checksums: v1.18.0: 3a90e7abf9910aebf9ef5845918c665afd4136a8832604ccfabca2defb35ce0f kubectl_checksums: arm: + v1.21.0: 6d79f6ebec2eda45b0808a895fa5d06bd8611c02d34b584eaa94857f365b25aa + v1.20.6: f5451e6d0e586d3643c83cfb1366b6a1fed0e8e575fa95153710af53a58cc1e1 + v1.20.5: b0b0e008dde7af257f97668620dc7b265f113833930ee03af190e7f4c4e8e545 v1.20.4: 459bfa4f35beb3496a2685f883c105a43321288619533ec7684dce922b926f60 v1.20.3: 72ad82d16b64dbb8c4f8eb61d710d062fe0f1ce7da94e457780948ad2e3ea9dc v1.20.2: a8d5b7e974200ae94a0eb3873773ec4ceffa99283f1843960d0a1b4448c2aa42 v1.20.1: f164e6efd49a6c4a278d1317f00ee1691e7cf8531a49d10ea069f5180d3540ed v1.20.0: bec2b30e37056ae354ec5a86d77634a9b832ade8d9aeea7a4514cee21cb7821e + v1.19.10: a35602edcd94d243d5c4c5ea7f75da0b0658cb8e429cb073e901d022538e14ab + v1.19.9: 697e0c4ce9a5511705abe252c717f7d84be9eadb3dbb905bd624d31870f62f45 v1.19.8: 4d3cba1a8005eabbf939577253cdf593be575d0e73ac47acc5090049d8a96781 v1.19.7: 11d49f87bf8b557066c339eea4775b32dd463fc61ce24b24744f82cb412e9277 v1.19.6: fde28a2bff5892e461d5c72c5391da5eef20450a5a074bbbfce4f476687dac95 @@ -225,6 +249,7 @@ kubectl_checksums: v1.19.2: c8cad74a586dfee41436ce866c91d79c9d2053468eccde9fed976cdf32b7849f v1.19.1: e63bbf3161c49d60e46ffaf7d3cfd689834516205be1be881d2e652115535e93 v1.19.0: 5885bb723a55ab95e8687e0ad52620ce4c08f76fd20c0973f5cd19c753b513c8 + v1.18.17: e153e7d980bc3af71ccede680d254457380fc2ae2044f0d67eb8685bcf519909 v1.18.16: a6eb328325fd2b2f3c254e6a97407df265caca39292a05993b5ef997bc2a55f3 v1.18.15: 952530dd6b272eed932e49a29625e20303b9621ba36c1cc0394476d483f6860a v1.18.14: b2b88ee6bea8ee17dd1c7687add53c9db5139abb7013ded77050d57b62070aa7 @@ -242,11 +267,16 @@ kubectl_checksums: v1.18.1: 896c90b1b9d88e121876d93718591f3ecbab880b304767806c6c9fcb3b145805 v1.18.0: 34fc6d16c2f535ed381b5fd3a4d40b642fff4f9ff95f8250b8043a29b8c062b9 arm64: + v1.21.0: a4dd7100f547a40d3e2f83850d0bab75c6ea5eb553f0a80adcf73155bef1fd0d + v1.20.6: 1d0a29420c4488b15adb44044b193588989b95515cd6c8c03907dafe9b3d53f3 + v1.20.5: bafb8af51eaacea782bf6c3574c2a531aaf14637980bfa09c43ab5023773f70c v1.20.4: 0fd64b3e5d3fda4637c174a5aea0119b46d6cbede591a4dc9130a81481fc952f v1.20.3: 5bd714a08410dbe7c69e61f0209973ccdffe6fe93de5a12d1707f9a40abec60d v1.20.2: 37fdba9fcd43cafba11ac4f82692e41aca41b59f44fd968fd84c263d71af580f v1.20.1: 1ed8762306d4d09574150ffc666a04e5a79ca08e53570cd34977fab4fc7c4611 v1.20.0: 25e4465870c99167e6c466623ed8f05a1d20fbcb48cab6688109389b52d87623 + v1.19.10: 343ba2807f167ded0672fed23e5d7b34d8c5bbddbe1998d09a8198a65d8ec8b5 + v1.19.9: 628627d01c9eaf624ffe3cf1195947a256ea5f842851e42682057e4233a9e283 v1.19.8: 8f037ab2aa798bbc66ebd1d52653f607f223b07813bcf98d9c1d0c0e136910ec v1.19.7: a0f58222e8d62b86b1148746531fa0c3b91fa208586cb068883550fc3dcd498b v1.19.6: 828ddf7183c647306a2432ff098a5b22fd6c68c069cb7047ffb1e2b2a967ef83 @@ -256,6 +286,7 @@ kubectl_checksums: v1.19.2: a460f918c03e5cd916f4ac28da137596232e344cc0755d4ceb222fc4cd203e09 v1.19.1: 332bbdb4560f9b7fcbb96c8f2cebbc4996e409384ca07510e5c5990998158c20 v1.19.0: d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f + v1.18.17: 40fb9151c0a91138973d7c21e1d615bc32217d11913d9c3a9cc83b9e7d199a2a v1.18.16: c6dbc057558cbb202eaa37cf5a414e07dd01f95ddeec8d3789ad53c7cfcfece9 v1.18.15: 6b4a63df325cdb523f16ffd8799745a8fdf979ef89e228c309a715671b6aa984 v1.18.14: ac4014f7d9001375cb600a15d77e90eb6a20053afac82f167f4b7732aa073388 @@ -273,11 +304,16 @@ kubectl_checksums: v1.18.1: 39e9645c6bed1e7340b3d764db983b9fc35326b11325fd509c3660c9f55469ed v1.18.0: 0de307f90502cd58e5785cdcbebeb552df81fa2399190f8a662afea9e30bc74d amd64: + v1.21.0: 9f74f2fa7ee32ad07e17211725992248470310ca1988214518806b39b1dad9f0 + v1.20.6: 89ae000df6bbdf38ae4307cc4ecc0347d5c871476862912c0a765db9bf05284e + v1.20.5: 7f9dbb80190945a5077dc5f4230202c22f68f9bd7f20c213c3cf5a74abf55e56 v1.20.4: 98e8aea149b00f653beeb53d4bd27edda9e73b48fed156c4a0aa1dabe4b1794c v1.20.3: 9124ace617387fdb78b95a9b2551d1b447bc8007caea68359743895784262fc8 v1.20.2: 2583b1c9fbfc5443a722fb04cf0cc83df18e45880a2cf1f6b52d9f595c5beb88 v1.20.1: 3f4b52a8072013e4cd34c9ea07e3c0c4e0350b227e00507fb1ae44a9adbf6785 v1.20.0: a5895007f331f08d2e082eb12458764949559f30bcc5beae26c38f3e2724262c + v1.19.10: dbacc1a372f00e2d15ad9a39925191c4e314e39b37dcac3e5b2f5e98f4be0d47 + v1.19.9: 7128c9e38ab9c445a3b02d3d0b3f0f15fe7fbca56fd87b84e575d7b29e999ad9 v1.19.8: a0737d3a15ca177816b6fb1fd59bdd5a3751bfdc66de4e08dffddba84e38bf3f v1.19.7: d46eb3bbe2575e5b6bedbc6d3519424b4f2f57929d7da1ef7e11c09068f37297 v1.19.6: d8a46600bcdcd01f39c11e609e8277975f04c0593f79b2a7b5c67646e1c792d8 @@ -287,6 +323,7 @@ kubectl_checksums: v1.19.2: f51adfe7968ee173dbfb3dabfc10dc774983cbf8a3a7c1c75a1423b91fda6821 v1.19.1: da4de99d4e713ba0c0a5ef6efe1806fb09c41937968ad9da5c5f74b79b3b38f5 v1.19.0: 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f + v1.18.17: db58bb46e29caecd98090e827a3f5075f01f166afb6da9463cc7d8a47787cb78 v1.18.16: 3e8cd55186ffe461f383af06cbbab2bfe5dd75def1d699523c53ee8b11b8c91a v1.18.15: eb5a5dd0a72795942ab81d1e4331625e80a90002c8bb39b2cb15aa707a3812c6 v1.18.14: 8c924c1fdf743c2a3bf0edbd4333f54c1bce64871abc1a729243321d99b567d4 @@ -305,11 +342,16 @@ kubectl_checksums: v1.18.0: bb16739fcad964c197752200ff89d89aad7b118cb1de5725dc53fe924c40e3f7 kubeadm_checksums: arm: + v1.21.0: 878e4c848412c3d6ac35323e6be1bd09f9648069ea1caa8e618fd2a6a6925b2f + v1.20.6: 8a299ce1cc63e51eb126217916fd2f57fb61bd9638467ee1219f4a81999800ce + v1.20.5: e8f9be7ffa4eac3bd8dc32229025a17da5b4fbd527b37272b890920377406db4 v1.20.4: 53759f7b62e465adc8bddcc11b23b87ceb263d83a965669dc4fc62bd6fdd228c v1.20.3: 35c78f4a86f9c3a791b3ecaf669734f790df1391e6bf310c3b2c51c3b97be05c v1.20.2: 83bcc176672ad55525ae87111f502d3d0cfec5bebb805634191063387298811c v1.20.1: 0e63af77d148e64cd1710f98d56967d5490df6a978658f6d5e2cd6b9cbf3c61a v1.20.0: 38829599cfcab3274e9cf1aff942db06d7ec091510399722159fa3fa49a6fef0 + v1.19.10: 9c2491590f34d8af5d1bc29569394d8ed96d88947f54c9a00d16bda486bc95f0 + v1.19.9: b60a386bbfbf2feb74cb94a62b32d47b2a85086deebc3127d62126c170077f5b v1.19.8: 6761f93e6983c4d7b924209a50a8664a2a78ac5dbb8c33e8e3ba898f11d49773 v1.19.7: 48722b7c93d18a13e734200de202912a324769139d0d434ff1dfff82af164814 v1.19.6: e3ecde2f2eccb177c0e4b8c6bd19ae471bc19977a8f812cb17094743d42b5b6e @@ -319,6 +361,7 @@ kubeadm_checksums: v1.19.2: effc35d1e3ab01ac80185ff9f7ca5afabbb94f5f91d7326b04b09e903315647d v1.19.1: 0e910cf9c771976f6eb079098ad428f3e99080624f478e3d71b670005a7c3651 v1.19.0: 62fca8b373f8c63409bcfcb7c585f8de882a8a119d88d39666e1ab3a11be188b + v1.18.17: 975392437bf9e427190007af37dd362f8d4413d296fd9b469bda2a6eace75fd1 v1.18.16: 277ce77d09d5f25c3fb9faa67fda6274999e4bd1deeae38186aae4d5a856f38a v1.18.15: b242890123a5ecc574157a959475c903eeb14ed17b4578902a17b8d994d4c401 v1.18.14: 7e5d4beedcaf13c0076f03c9232464946faa061effd5db8c7705d317a4ee6e95 @@ -336,11 +379,16 @@ kubeadm_checksums: v1.18.1: 4f919ad7215209dee97ea4c61668e44a2cce8f575b9cf4032e47f0c377924854 v1.18.0: 0f05bd526bb38be11459675c69bc882a2d3e583e48339fab49b620d292c2433e arm64: + v1.21.0: 50bb95d1827455346b5643dcf83a52520733c3a582b8b1ffb50f04a8e66f00e7 + v1.20.6: 33837e290bd76fcb16af27db0e814ec023c25e6c41f25a0907b48756d4a2ffc2 + v1.20.5: d3d587bb1db1411b662d4ede0305d39725a68b8453423e76b2195fa01c4f2a37 v1.20.4: c3ff7f944826889a23a002c85e8f9f9d9a8bc95e9083fbdda59831e3e34245a7 v1.20.3: d4699a79e99c4603adb13f87825ce1109fffefd803966f8187baeb8e25cd0060 v1.20.2: 50f16984e4fc5681ba1fa7fb8a19fc1ca72e79ff5c16e97f5b500228efcf9a75 v1.20.1: 91eb63944e9a033bb40dfc638913ca76e07d6877a24d25636bf833459ae90010 v1.20.0: 16faf8bf3c94cb21dcae131cbfbd0961fc5fef37cb7e37f872ff249e235e0f46 + v1.19.10: 69bd90bb93e5560996549cf93c89b8c0a2a75d8e24504d896b303cc64018de91 + v1.19.9: 403c767bef0d681aebc45d5643787fc8c0b9344866cbd339368637a05ea1d11c v1.19.8: dfb838ffb88d79e4d881326f611ae5e5999accb54cdd666c75664da264b5d58e v1.19.7: 43615e65092de34dcaf34725d8adc0af91b660fa54a885421fdb29f743478911 v1.19.6: 082ceac5f542cb49a62cf47212bf1ea9dbb15d1831126f526d15e429b4f0687d @@ -350,6 +398,7 @@ kubeadm_checksums: v1.19.2: b6900a44558ba1a0a364406e2072163f5fc561030da97045e1403cdc69ff2682 v1.19.1: dcdabd2fdec9b4dd8febd1625e1fbbe15362919041e5f4ce3aab629e4aea7540 v1.19.0: db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5 + v1.18.17: 39e6a732aa060fecd3e6b23c30ec325b5afd963466e857d8b2e65b7c729f450b v1.18.16: fbb945fff686eb2683156dee16fe2bf029e6afe575f68a54da2f81f5192dbff7 v1.18.15: d6d4d9b8e4992c218ff6185da7928289c9938796d5c08a7427625a563c74a075 v1.18.14: e4c1aaed946dd38ca6d9fdef0ef333b4d488a25810aa6f8a98de1b9dd6d47db3 @@ -367,11 +416,16 @@ kubeadm_checksums: v1.18.1: 0cb6589d9b4c09b007eae977ab8a185fc4140eda886408dced4f500a508e4e83 v1.18.0: 2ef1785159c80a9acd454a1c8be3c6b8db2260200b22e4359426e709ff786d01 amd64: + v1.21.0: 7bdaf0d58f0d286538376bc40b50d7e3ab60a3fe7a0709194f53f1605129550f + v1.20.6: ff6fca46edeccd8a4dbf162079d0b3d27841b04885b3f47f80377b3a93ab1533 + v1.20.5: eecb68547f0f99951b0ed910dcb0cfa32766587cc79a53960766164a56d6aed0 v1.20.4: dcc5629da2c31a000b9b50db077b1cd51a6840e08233fd64b67e37f3f098c392 v1.20.3: 768896db3163d9926f1a0dcf6a7dd9eeeca64a3e162758425ec43e524ada4a5a v1.20.2: e0fce64f3afd3a84bce0996ccdb483812607a91f03e726231f2aaeb622bb9519 v1.20.1: c5abaf7db925d1303866c8da6cb20ac2d4404588bc503805ef43c3cebf5ce7b9 v1.20.0: b0ef92da9901670437af869e88c2799512bcb858b886207a081882795567e807 + v1.19.10: aff5ed83e2e3c9466f4b97b53426d9267b51b8cf5466cd375414eaa300c40ad7 + v1.19.9: 917712bbd38b625aca456ffa78bf134d64f0efb186cc5772c9844ba6d74fd920 v1.19.8: 9c6646cdf03efc3194afc178647205195da4a43f58d0b70954953f566fa15c76 v1.19.7: c63ef1842533cd7888c7452cab9f320dcf45fc1c173e9d40abb712d45992db24 v1.19.6: 6204d9f16554480fe64a09e9efef31559f1da623fb34492a9a18b085afac876a @@ -381,6 +435,7 @@ kubeadm_checksums: v1.19.2: 377dbf06469709aafb7a9a6c925e890bf48727b905455c2a81fee28727716f2f v1.19.1: d5afcf4ff916b4c6810f10f76176c73238f0854b7748b2cde8afbd91de65a3c9 v1.19.0: 88ce7dc5302d8847f6e679aab9e4fa642a819e8a33d70731fb7bc8e110d8659f + v1.18.17: e09614e930e47d9eee9324e826984af447ffc9a95f4f79a14633980a8cb5b691 v1.18.16: 775d6f59cb8d6c16cb66f48087a4b8a31a8d3b84de4653b7d0cc061195306345 v1.18.15: 8a5be9e04343e0ac10320455b32a78e5ffc60f450c5c0a11914edeb86ca178d7 v1.18.14: d6143cd822218daa5faf583c9b8b862c609e66052232e3d3d23c72957fdae341 @@ -410,28 +465,25 @@ cni_binary_checksums: amd64: 58a58d389895ba9f9bbd3ef330f186c0bb7484136d0bfb9b50152eed55d9ec24 calicoctl_binary_checksums: arm: - v3.16.6: 0 + v3.16.9: 0 v3.15.2: 0 amd64: - v3.16.6: 9b82230446d4749a1043dddd8d466d275a460e570a412e6ced003368ab9c72d8 + v3.16.9: 3858748e3446f24f6176ef4125fa58bb6f934f6e838cf92f60eafee6aa8117e5 v3.15.2: 219ae954501cbe15daeda0ad52e13ec65f99c77548c7d3cbfc4ced5c7149fdf1 arm64: - v3.16.6: 4dd0548390a4ff7c6264c967da80498d10612dbd86f6d1ed4e5503352cdf947b + v3.16.9: beac9a1ac66fbed05000901aa883443397f05e0f6ffe10ceeccf89f427edc8ad v3.15.2: 49165f9e4ad55402248b578310fcf68a57363f54e66be04ac24be9714899b4d5 +calico_crds_archive_checksums: + v3.16.9: cc33648e3fffd973c8780f49589b1ecf57487f8693cfd12a335dcd2a606784c0 + v3.15.2: 82e7122ec04a89c89861b8377c39ae357c7cdbbf60b5f0f1b8fc18ba6bda7dc2 helm_archive_checksums: arm: - v3.5.0: ca8792da269b72235987ea7245d1450a859b2c0658f591737d74b6c56cd9b1fa - v3.5.1: 0b86a5a68df7376484babb6d7ffe1bae36012b4d65f1bcddb4255fb3bbe811db - v3.5.2: 98d090fc1769f5bf7451c15f6ed5a173a1ce5175eca32070ac19064d36470f1b + v3.5.3: fd9c1e1eaa6d8d2c9df6027524e80b8bfde0ea49de5f324845256b3e9cc2edb0 amd64: - v3.5.0: 3fff0354d5fba4c73ebd5db59a59db72f8a5bbe1117a0b355b0c2983e98db95b - v3.5.1: cad8f2f55a87cfd4d79312625c6af62c1e22eb1dab750f00aa1d394c601a2e6b - v3.5.2: 01b317c506f8b6ad60b11b1dc3f093276bb703281cb1ae01132752253ec706a2 + v3.5.3: 2170a1a644a9e0b863f00c17b761ce33d4323da64fc74562a3a6df2abbf6cd70 arm64: - v3.5.0: 87811b648ed9f4c84d3cb67bbea9b666bb7f6dd0ff6aca148b65f91058f73953 - v3.5.1: d0ada80576f8016d1cc38a06d225a4379a53e88e3e26b417e6de5db05a090ce4 - v3.5.2: 126a72e2b209194fd2735861f0cf8bd5b5d1386eedd6aed6e0e050dca80d493e + v3.5.3: e1348d94ce4caace43689ee2dfa5f8bcd8687c12053d9c13d79875b65d6b72aa etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}" cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}" @@ -439,6 +491,7 @@ kubelet_binary_checksum: "{{ kubelet_checksums[image_arch][kube_version] }}" kubectl_binary_checksum: "{{ kubectl_checksums[image_arch][kube_version] }}" kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}" calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" +calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}" crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" @@ -504,7 +557,7 @@ coredns_version: "1.7.0" coredns_image_repo: "{{ kube_image_repo }}/coredns" coredns_image_tag: "{{ coredns_version }}" -nodelocaldns_version: "1.16.0" +nodelocaldns_version: "1.17.1" nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache" nodelocaldns_image_tag: "{{ nodelocaldns_version }}" @@ -798,6 +851,23 @@ downloads: groups: - k8s-cluster + calico_crds: + file: true + enabled: "{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}" + version: "{{ calico_version }}" + dest: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ calico_version }}.tar.gz" + sha256: "{{ calico_crds_archive_checksum }}" + url: "{{ calico_crds_download_url }}" + unarchive: true + unarchive_extra_opts: + - "--strip=6" + - "--wildcards" + - "*/_includes/charts/calico/crds/kdd/" + owner: "root" + mode: "0755" + groups: + - kube-master + weave_kube: enabled: "{{ kube_network_plugin == 'weave' }}" container: true diff --git a/roles/download/tasks/download_container.yml b/roles/download/tasks/download_container.yml index 7f015999f0d..d2bea776d24 100644 --- a/roles/download/tasks/download_container.yml +++ b/roles/download/tasks/download_container.yml @@ -24,6 +24,9 @@ - name: download_container | Determine if image is in cache stat: path: "{{ image_path_cached }}" + get_attributes: no + get_checksum: no + get_mime: no delegate_to: localhost connection: local delegate_facts: no diff --git a/roles/download/tasks/extract_file.yml b/roles/download/tasks/extract_file.yml index 0a314307ee2..81858dd3a03 100644 --- a/roles/download/tasks/extract_file.yml +++ b/roles/download/tasks/extract_file.yml @@ -6,5 +6,6 @@ owner: "{{ download.owner | default(omit) }}" mode: "{{ download.mode | default(omit) }}" copy: no + extra_opts: "{{ download.unarchive_extra_opts|default(omit) }}" when: - download.unarchive | default(false) diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 2fa45929f30..52501a8b340 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -25,7 +25,7 @@ - name: download | Download files / images include_tasks: "{{ include_file }}" - with_dict: "{{ downloads | combine(kubeadm_images) }}" + loop: "{{ downloads | combine(kubeadm_images) | dict2items }}" vars: download: "{{ download_defaults | combine(item.value) }}" include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml" diff --git a/roles/etcd/handlers/backup.yml b/roles/etcd/handlers/backup.yml index 6d2cc8bcb69..ba6669b3f05 100644 --- a/roles/etcd/handlers/backup.yml +++ b/roles/etcd/handlers/backup.yml @@ -29,6 +29,9 @@ - name: Stat etcd v2 data directory stat: path: "{{ etcd_data_dir }}/member" + get_attributes: no + get_checksum: no + get_mime: no register: etcd_data_dir_member - name: Backup etcd v2 data diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index d3aaa9c23bd..ca070563079 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -17,6 +17,9 @@ - name: "Check certs | check if a cert already exists on node" stat: path: "{{ etcd_cert_dir }}/{{ item }}" + get_attributes: no + get_checksum: yes + get_mime: no register: etcdcert_node with_items: - ca.pem diff --git a/roles/etcdctl/tasks/main.yml b/roles/etcdctl/tasks/main.yml index 47aa0400435..1039242bd63 100644 --- a/roles/etcdctl/tasks/main.yml +++ b/roles/etcdctl/tasks/main.yml @@ -9,6 +9,9 @@ - name: Check if etcdctl exist stat: path: "{{ bin_dir }}/etcdctl" + get_attributes: no + get_checksum: no + get_mime: no register: stat_etcdctl - block: @@ -28,6 +31,9 @@ - name: Check if etcdctl still exist after version check stat: path: "{{ bin_dir }}/etcdctl" + get_attributes: no + get_checksum: no + get_mime: no register: stat_etcdctl - block: diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml index fc4a958c133..e5df6dac830 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml @@ -7,6 +7,7 @@ local_volume_provisioner_nodelabels: [] # - topology.kubernetes.io/zone # Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted # see https://github.com/ansible/ansible/issues/17324 +local_volume_provisioner_use_node_name_only: false local_volume_provisioner_storage_classes: | { "{{ local_volume_provisioner_storage_class | default('local-storage') }}": { diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 index 21807fe4386..76625b6df55 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 @@ -21,6 +21,9 @@ data: {% for nodelabel in local_volume_provisioner_nodelabels %} - {{ nodelabel }} {% endfor %} +{% endif %} +{% if local_volume_provisioner_use_node_name_only %} + useNodeNameOnly: "true" {% endif %} storageClassMap: | {% for class_name, storage_class in local_volume_provisioner_storage_classes.items() %} diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index f4d2e19046a..3a4e8884671 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -4,7 +4,7 @@ vars: download: "{{ download_defaults | combine(downloads.helm) }}" -- name: Copy helm binary from download dir +- name: Helm | Copy helm binary from download dir synchronize: src: "{{ local_release_dir }}/helm-{{ helm_version }}/linux-{{ image_arch }}/helm" dest: "{{ bin_dir }}/helm" @@ -14,19 +14,22 @@ group: no delegate_to: "{{ inventory_hostname }}" -- name: Check if bash_completion.d folder exists # noqa 503 +- name: Helm | Check if bash_completion.d folder exists # noqa 503 stat: path: "/etc/bash_completion.d/" + get_attributes: no + get_checksum: no + get_mime: no register: stat_result -- name: Get helm completion +- name: Helm | Get helm completion command: "{{ bin_dir }}/helm completion bash" changed_when: False register: helm_completion check_mode: False when: stat_result.stat.exists -- name: Install helm completion +- name: Helm | Install helm completion copy: dest: /etc/bash_completion.d/helm.sh content: "{{ helm_completion.stdout }}" diff --git a/roles/kubernetes-apps/metallb/tasks/main.yml b/roles/kubernetes-apps/metallb/tasks/main.yml index 5d3c58d6e4b..a69db4b5f1a 100644 --- a/roles/kubernetes-apps/metallb/tasks/main.yml +++ b/roles/kubernetes-apps/metallb/tasks/main.yml @@ -1,5 +1,5 @@ --- -- name: "Kubernetes Apps | Check cluster settings for MetalLB" +- name: Kubernetes Apps | Check cluster settings for MetalLB fail: msg: "MetalLB require kube_proxy_strict_arp = true, see https://github.com/danderson/metallb/issues/153#issuecomment-518651132" when: @@ -32,7 +32,7 @@ - podsecuritypolicy_enabled - inventory_hostname == groups['kube-master'][0] -- name: "Kubernetes Apps | Lay Down MetalLB" +- name: Kubernetes Apps | Lay Down MetalLB become: true template: { src: "{{ item }}.j2", dest: "{{ kube_config_dir }}/{{ item }}" } with_items: ["metallb.yml", "metallb-config.yml"] @@ -40,7 +40,7 @@ when: - "inventory_hostname == groups['kube-master'][0]" -- name: "Kubernetes Apps | Install and configure MetalLB" +- name: Kubernetes Apps | Install and configure MetalLB kube: name: "MetalLB" kubectl: "{{ bin_dir }}/kubectl" diff --git a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml deleted file mode 100644 index e9de24b5277..00000000000 --- a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Rotate Tokens | Get default token name # noqa 306 - shell: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets -o custom-columns=name:{.metadata.name} --no-headers | grep -m1 default-token" - register: default_token - changed_when: false - until: default_token.rc == 0 - delay: 4 - retries: 10 - -- name: Rotate Tokens | Get default token data - command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets {{ default_token.stdout }} -ojson" - register: default_token_data - changed_when: false - -- name: Rotate Tokens | Test if default certificate is expired - uri: - url: https://{{ kube_apiserver_ip }}/api/v1/nodes - method: GET - return_content: no - validate_certs: no - headers: - Authorization: "Bearer {{ (default_token_data.stdout|from_json)['data']['token']|b64decode }}" - register: check_secret - failed_when: false - -- name: Rotate Tokens | Determine if certificate is expired - set_fact: - needs_rotation: '{{ check_secret.status not in [200, 403] }}' - -# FIXME(mattymo): Exclude built in secrets that were automatically rotated, -# instead of filtering manually -- name: Rotate Tokens | Get all serviceaccount tokens to expire # noqa 306 - shell: >- - {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets --all-namespaces - -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}' - | grep kubernetes.io/service-account-token - | egrep 'default-token|kube-proxy|coredns|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|tiller|local-volume-provisioner' - register: tokens_to_delete - when: needs_rotation - -- name: Rotate Tokens | Delete expired tokens - command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete secrets -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}" - with_items: "{{ tokens_to_delete.stdout_lines }}" - when: needs_rotation - -- name: Rotate Tokens | Delete pods in system namespace - command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete pods -n kube-system --all --grace-period=0 --force" - when: needs_rotation diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index b0f0e09b098..148226e6df5 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -13,11 +13,17 @@ - name: Check if kubelet.conf exists stat: path: "{{ kube_config_dir }}/kubelet.conf" + get_attributes: no + get_checksum: no + get_mime: no register: kubelet_conf - name: Check if kubeadm CA cert is accessible stat: path: "{{ kube_cert_dir }}/ca.crt" + get_attributes: no + get_checksum: no + get_mime: no register: kubeadm_ca_stat delegate_to: "{{ groups['kube-master'][0] }}" run_once: true @@ -46,11 +52,6 @@ kubeadm_token: "{{ temp_token.stdout }}" when: kubeadm_token is not defined -- name: Get the kubeadm version - command: "{{ bin_dir }}/kubeadm version -o short" - register: kubeadm_output - changed_when: false - - name: Set kubeadm api version to v1beta2 set_fact: kubeadmConfig_api_version: v1beta2 diff --git a/roles/kubernetes/master/defaults/main/main.yml b/roles/kubernetes/master/defaults/main/main.yml index 24c1ddff0c8..0233e5142d0 100644 --- a/roles/kubernetes/master/defaults/main/main.yml +++ b/roles/kubernetes/master/defaults/main/main.yml @@ -194,5 +194,8 @@ secrets_encryption_query: "resources[*].providers[0].{{kube_encryption_algorithm ## Amount of time to retain events. (default 1h0m0s) event_ttl_duration: "1h0m0s" -## Force regeneration of kubernetes control plane certificates without the need of bumping the cluster version -force_certificate_regeneration: false + +## Automatically renew K8S control plane certificates on first Monday of each month +auto_renew_certificates: false +# First Monday of each month +auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube-master'].index(inventory_hostname) }}0:00" diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml index 577b433b536..e6bc321e20a 100644 --- a/roles/kubernetes/master/handlers/main.yml +++ b/roles/kubernetes/master/handlers/main.yml @@ -121,21 +121,3 @@ until: result.status == 200 retries: 60 delay: 1 - -- name: Master | set secret_changed - command: /bin/true - notify: - - Master | set secret_changed to true - - Master | Copy new kubeconfig for root user - -- name: Master | set secret_changed to true - set_fact: - secret_changed: true - -- name: Master | Copy new kubeconfig for root user - copy: - src: "{{ kube_config_dir }}/admin.conf" - dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config" - remote_src: yes - mode: "0600" - backup: yes diff --git a/roles/kubernetes/master/tasks/encrypt-at-rest.yml b/roles/kubernetes/master/tasks/encrypt-at-rest.yml index 09584dce8fc..234fa9bffee 100644 --- a/roles/kubernetes/master/tasks/encrypt-at-rest.yml +++ b/roles/kubernetes/master/tasks/encrypt-at-rest.yml @@ -2,6 +2,9 @@ - name: Check if secret for encrypting data at rest already exist stat: path: "{{ kube_cert_dir }}/secrets_encryption.yaml" + get_attributes: no + get_checksum: no + get_mime: no register: secrets_encryption_file - name: Slurp secrets_encryption file if it exists diff --git a/roles/kubernetes/master/tasks/kubeadm-backup.yml b/roles/kubernetes/master/tasks/kubeadm-backup.yml new file mode 100644 index 00000000000..1e1dda97fb3 --- /dev/null +++ b/roles/kubernetes/master/tasks/kubeadm-backup.yml @@ -0,0 +1,28 @@ +--- +- name: Backup old certs and keys + copy: + src: "{{ kube_cert_dir }}/{{ item }}" + dest: "{{ kube_cert_dir }}/{{ item }}.old" + mode: preserve + remote_src: yes + with_items: + - apiserver.crt + - apiserver.key + - apiserver-kubelet-client.crt + - apiserver-kubelet-client.key + - front-proxy-client.crt + - front-proxy-client.key + ignore_errors: yes + +- name: Backup old confs + copy: + src: "{{ kube_config_dir }}/{{ item }}" + dest: "{{ kube_config_dir }}/{{ item }}.old" + mode: preserve + remote_src: yes + with_items: + - admin.conf + - controller-manager.conf + - kubelet.conf + - scheduler.conf + ignore_errors: yes diff --git a/roles/kubernetes/master/tasks/kubeadm-certificate.yml b/roles/kubernetes/master/tasks/kubeadm-certificate.yml deleted file mode 100644 index 03ebe25365c..00000000000 --- a/roles/kubernetes/master/tasks/kubeadm-certificate.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Backup old certs and keys - copy: - src: "{{ kube_cert_dir }}/{{ item.src }}" - dest: "{{ kube_cert_dir }}/{{ item.dest }}" - mode: 0640 - remote_src: yes - with_items: - - {src: apiserver.crt, dest: apiserver.crt.old} - - {src: apiserver.key, dest: apiserver.key.old} - - {src: apiserver-kubelet-client.crt, dest: apiserver-kubelet-client.crt.old} - - {src: apiserver-kubelet-client.key, dest: apiserver-kubelet-client.key.old} - - {src: front-proxy-client.crt, dest: front-proxy-client.crt.old} - - {src: front-proxy-client.key, dest: front-proxy-client.key.old} - ignore_errors: yes diff --git a/roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml b/roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml deleted file mode 100644 index adca631c2f3..00000000000 --- a/roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: kubeadm | Retrieve files to purge - find: - paths: "{{ kube_cert_dir }}" - patterns: '*.pem' - register: files_to_purge_for_kubeadm - -- name: kubeadm | Purge old certs - file: - path: "{{ item.path }}" - state: absent - with_items: "{{ files_to_purge_for_kubeadm.files }}" - -- name: kubeadm | Purge old kubeconfig - file: - path: "{{ ansible_env.HOME | default('/root') }}/.kube/config" - state: absent diff --git a/roles/kubernetes/master/tasks/kubeadm-etcd.yml b/roles/kubernetes/master/tasks/kubeadm-etcd.yml index eb68c26fef0..ebba677cc75 100644 --- a/roles/kubernetes/master/tasks/kubeadm-etcd.yml +++ b/roles/kubernetes/master/tasks/kubeadm-etcd.yml @@ -12,20 +12,6 @@ tags: - network -- name: Ensure etcdctl binary is installed - include_tasks: "{{ role_path }}/../../etcd/tasks/install_host.yml" - vars: - etcd_cluster_setup: true - when: etcd_deployment_type == "host" and not etcd_kubeadm_enabled - -- name: Ensure etcdctl binary is installed - include_tasks: "{{ role_path }}/../../etcd/tasks/install_etcdctl_docker.yml" - vars: - etcd_cluster_setup: true - etcd_retries: 4 - when: - - etcd_deployment_type == "docker" and not etcd_kubeadm_enabled - - name: Ensure etcdctl script is installed import_role: name: etcdctl diff --git a/roles/kubernetes/master/tasks/kubeadm-fix-apiserver.yml b/roles/kubernetes/master/tasks/kubeadm-fix-apiserver.yml index 6ebfb179a91..5376aba81e6 100644 --- a/roles/kubernetes/master/tasks/kubeadm-fix-apiserver.yml +++ b/roles/kubernetes/master/tasks/kubeadm-fix-apiserver.yml @@ -1,34 +1,11 @@ --- -- name: Test if correct apiserver is set in all kubeconfigs - shell: >- - grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/admin.conf && - grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/controller-manager.conf && - grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/kubelet.conf && - grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/scheduler.conf - register: kubeconfig_correct_apiserver - changed_when: False - failed_when: False -- name: Create temporary directory - tempfile: - state: directory - register: kubeconfig_temp_dir - when: kubeconfig_correct_apiserver.rc != 0 - -- name: Generate new kubeconfigs with correct apiserver - command: >- - {{ bin_dir }}/kubeadm init phase kubeconfig all - --config {{ kube_config_dir }}/kubeadm-config.yaml - --kubeconfig-dir {{ kubeconfig_temp_dir.path }} - when: kubeconfig_correct_apiserver.rc != 0 - -- name: Copy new kubeconfigs to kube config dir - copy: - src: "{{ kubeconfig_temp_dir.path }}/{{ item }}" +- name: Update server field in component kubeconfigs + lineinfile: dest: "{{ kube_config_dir }}/{{ item }}" - mode: 0640 - remote_src: yes - when: kubeconfig_correct_apiserver.rc != 0 + regexp: '^ server: https' + line: ' server: {{ kube_apiserver_endpoint }}' + backup: yes with_items: - admin.conf - controller-manager.conf @@ -38,9 +15,3 @@ - "Master | Restart kube-controller-manager" - "Master | Restart kube-scheduler" - "Master | reload kubelet" - -- name: Cleanup temporary directory - file: - path: "{{ kubeconfig_temp_dir.path }}" - state: absent - when: kubeconfig_correct_apiserver.rc != 0 diff --git a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml deleted file mode 100644 index cae5749cf83..00000000000 --- a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: Copy old certs to the kubeadm expected path - copy: - src: "{{ kube_cert_dir }}/{{ item.src }}" - dest: "{{ kube_cert_dir }}/{{ item.dest }}" - mode: 0640 - remote_src: yes - with_items: - - {src: apiserver.pem, dest: apiserver.crt} - - {src: apiserver-key.pem, dest: apiserver.key} - - {src: ca.pem, dest: ca.crt} - - {src: ca-key.pem, dest: ca.key} - - {src: front-proxy-ca.pem, dest: front-proxy-ca.crt} - - {src: front-proxy-ca-key.pem, dest: front-proxy-ca.key} - - {src: front-proxy-client.pem, dest: front-proxy-client.crt} - - {src: front-proxy-client-key.pem, dest: front-proxy-client.key} - - {src: service-account-key.pem, dest: sa.pub} - - {src: service-account-key.pem, dest: sa.key} - - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt} - - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key} - register: kubeadm_copy_old_certs diff --git a/roles/kubernetes/master/tasks/kubeadm-secondary.yml b/roles/kubernetes/master/tasks/kubeadm-secondary.yml index b81c4221268..6f961f2bcbe 100644 --- a/roles/kubernetes/master/tasks/kubeadm-secondary.yml +++ b/roles/kubernetes/master/tasks/kubeadm-secondary.yml @@ -66,7 +66,3 @@ when: - inventory_hostname != groups['kube-master']|first - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - -- name: Set secret_changed to false to avoid extra token rotation - set_fact: - secret_changed: false diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 43655a30de6..5ed944e4598 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -1,15 +1,4 @@ --- -- name: kubeadm | Check if old apiserver cert exists on host - stat: - path: "{{ kube_cert_dir }}/apiserver.pem" - register: old_apiserver_cert - delegate_to: "{{ groups['kube-master'] | first }}" - run_once: true - -- name: kubeadm | Migrate old certs if necessary - import_tasks: kubeadm-migrate-certs.yml - when: old_apiserver_cert.stat.exists - - name: Install OIDC certificate copy: content: "{{ kube_oidc_ca_cert | b64decode }}" @@ -21,39 +10,18 @@ - kube_oidc_auth - kube_oidc_ca_cert is defined -- name: kubeadm | Check serviceaccount key - stat: - path: "{{ kube_cert_dir }}/sa.key" - register: sa_key_before - run_once: true - - name: kubeadm | Check if kubeadm has already run stat: path: "/var/lib/kubelet/config.yaml" + get_attributes: no + get_checksum: no + get_mime: no register: kubeadm_already_run -- name: kubeadm | Delete old admin.conf - file: - path: "{{ kube_config_dir }}/admin.conf" - state: absent - when: - - not kubeadm_already_run.stat.exists - -- name: kubeadm | Delete old static pods - file: - path: "{{ kube_config_dir }}/manifests/{{ item }}.manifest" - state: absent - with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"] - when: - - old_apiserver_cert.stat.exists - -- name: kubeadm | Forcefully delete old static pods - shell: "set -o pipefail && docker ps -f name=k8s_{{ item }} -q | xargs --no-run-if-empty docker rm -f" - args: - executable: /bin/bash - with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] +- name: kubeadm | Backup kubeadm certs / kubeconfig + import_tasks: kubeadm-backup.yml when: - - old_apiserver_cert.stat.exists + - kubeadm_already_run.stat.exists - name: kubeadm | aggregate all SANs set_fact: @@ -102,22 +70,33 @@ kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}" when: loadbalancer_apiserver is defined -- name: kubeadm | set kubeadm version - import_tasks: kubeadm-version.yml +- name: Set kubeadm api version to v1beta2 + set_fact: + kubeadmConfig_api_version: v1beta2 -- name: kubeadm | Certificate management with kubeadm - import_tasks: kubeadm-certificate.yml - when: - - not upgrade_cluster_setup - - kubeadm_already_run.stat.exists +- name: kubeadm | Create kubeadm config + template: + src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/kubeadm-config.yaml" + mode: 0640 - name: kubeadm | Check if apiserver.crt contains all needed SANs - command: openssl x509 -noout -in "{{ kube_cert_dir }}/apiserver.crt" -check{{ item|ipaddr|ternary('ip','host') }} "{{ item }}" - with_items: "{{ apiserver_sans }}" + shell: | + set -o pipefail + for IP in {{ apiserver_ips | join(' ') }}; do + openssl x509 -noout -in "{{ kube_cert_dir }}/apiserver.crt" -checkip $IP | grep -q 'does match certificate' || echo 'NEED-RENEW' + done + for HOST in {{ apiserver_hosts | join(' ') }}; do + openssl x509 -noout -in "{{ kube_cert_dir }}/apiserver.crt" -checkhost $HOST | grep -q 'does match certificate' || echo 'NEED-RENEW' + done + vars: + apiserver_ips: "{{ apiserver_sans|map('ipaddr')|reject('equalto', False)|list }}" + apiserver_hosts: "{{ apiserver_sans|difference(apiserver_ips) }}" + args: + executable: /bin/bash register: apiserver_sans_check - changed_when: "'does match certificate' not in apiserver_sans_check.stdout" + changed_when: "'NEED-RENEW' in apiserver_sans_check.stdout" when: - - inventory_hostname == groups['kube-master']|first - kubeadm_already_run.stat.exists - name: kubeadm | regenerate apiserver cert 1/2 @@ -128,9 +107,8 @@ - apiserver.crt - apiserver.key when: - - inventory_hostname == groups['kube-master']|first - kubeadm_already_run.stat.exists - - apiserver_sans_check.changed or force_certificate_regeneration + - apiserver_sans_check.changed - name: kubeadm | regenerate apiserver cert 2/2 command: >- @@ -138,9 +116,8 @@ init phase certs apiserver --config={{ kube_config_dir }}/kubeadm-config.yaml when: - - inventory_hostname == groups['kube-master']|first - kubeadm_already_run.stat.exists - - apiserver_sans_check.changed or force_certificate_regeneration + - apiserver_sans_check.changed - name: kubeadm | Initialize first master command: >- @@ -208,22 +185,6 @@ - upgrade_cluster_setup - kubeadm_already_run.stat.exists -- name: kubeadm | Check serviceaccount key again - stat: - path: "{{ kube_cert_dir }}/sa.key" - register: sa_key_after - run_once: true - -- name: kubeadm | Set secret_changed if service account key was updated - command: /bin/true - notify: Master | set secret_changed - when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum - -- name: kubeadm | cleanup old certs if necessary - import_tasks: kubeadm-cleanup-old-certs.yml - when: - - old_apiserver_cert.stat.exists - # FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file. - name: kubeadm | Remove taint for master with node role command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-" diff --git a/roles/kubernetes/master/tasks/kubeadm-version.yml b/roles/kubernetes/master/tasks/kubeadm-version.yml deleted file mode 100644 index 8c7feea3510..00000000000 --- a/roles/kubernetes/master/tasks/kubeadm-version.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: Get the kubeadm version - command: "{{ bin_dir }}/kubeadm version -o short" - register: kubeadm_output - changed_when: false - -- name: Set kubeadm api version to v1beta2 - set_fact: - kubeadmConfig_api_version: v1beta2 - -- name: kubeadm | Create kubeadm config - template: - src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2" - dest: "{{ kube_config_dir }}/kubeadm-config.yaml" - mode: 0640 diff --git a/roles/kubernetes/master/tasks/kubelet-fix-client-cert-rotation.yml b/roles/kubernetes/master/tasks/kubelet-fix-client-cert-rotation.yml new file mode 100644 index 00000000000..7d0c1a0d59e --- /dev/null +++ b/roles/kubernetes/master/tasks/kubelet-fix-client-cert-rotation.yml @@ -0,0 +1,18 @@ +--- +- name: Fixup kubelet client cert rotation 1/2 + lineinfile: + path: "{{ kube_config_dir }}/kubelet.conf" + regexp: '^ client-certificate-data: ' + line: ' client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem' + backup: yes + notify: + - "Master | reload kubelet" + +- name: Fixup kubelet client cert rotation 2/2 + lineinfile: + path: "{{ kube_config_dir }}/kubelet.conf" + regexp: '^ client-key-data: ' + line: ' client-key: /var/lib/kubelet/pki/kubelet-client-current.pem' + backup: yes + notify: + - "Master | reload kubelet" diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index a85dddfb9c5..6fba951c28d 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -62,3 +62,31 @@ - name: Include kubeadm secondary server apiserver fixes include_tasks: kubeadm-fix-apiserver.yml + +- name: Include kubelet client cert rotation fixes + include_tasks: kubelet-fix-client-cert-rotation.yml + when: kubelet_rotate_certificates + +- name: Install script to renew K8S control plane certificates + template: + src: k8s-certs-renew.sh.j2 + dest: "{{ bin_dir }}/k8s-certs-renew.sh" + mode: '755' + +- name: Renew K8S control plane certificates monthly 1/2 + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + with_items: + - k8s-certs-renew.service + - k8s-certs-renew.timer + register: k8s_certs_units + when: auto_renew_certificates + +- name: Renew K8S control plane certificates monthly 2/2 + systemd: + name: k8s-certs-renew.timer + enabled: yes + state: started + daemon-reload: "{{ k8s_certs_units is changed }}" + when: auto_renew_certificates diff --git a/roles/kubernetes/master/templates/k8s-certs-renew.service.j2 b/roles/kubernetes/master/templates/k8s-certs-renew.service.j2 new file mode 100644 index 00000000000..64610c2bc02 --- /dev/null +++ b/roles/kubernetes/master/templates/k8s-certs-renew.service.j2 @@ -0,0 +1,6 @@ +[Unit] +Description=Renew K8S control plane certificates + +[Service] +Type=oneshot +ExecStart={{ bin_dir }}/k8s-certs-renew.sh diff --git a/roles/kubernetes/master/templates/k8s-certs-renew.sh.j2 b/roles/kubernetes/master/templates/k8s-certs-renew.sh.j2 new file mode 100644 index 00000000000..b3dae43cc26 --- /dev/null +++ b/roles/kubernetes/master/templates/k8s-certs-renew.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash + +echo "## Expiration before renewal ##" +{{ bin_dir }}/kubeadm {{ 'alpha ' if kube_version is version('v1.20.0', '<') else '' }}certs check-expiration + +echo "## Renewing certificates managed by kubeadm ##" +{{ bin_dir }}/kubeadm {{ 'alpha ' if kube_version is version('v1.20.0', '<') else '' }}certs renew all + +echo "## Restarting control plane pods managed by kubeadm ##" +{% if container_manager == "docker" %} +{{ docker_bin_dir }}/docker ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs {{ docker_bin_dir }}/docker rm -f +{% else %} +{{ bin_dir }}/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs {{ bin_dir }}/crictl rmp -f +{% endif %} + +echo "## Updating /root/.kube/config ##" +/usr/bin/cp {{ kube_config_dir }}/admin.conf /root/.kube/config + +echo "## Waiting for apiserver to be up again ##" +until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/6443; do sleep 1; done + +echo "## Expiration after renewal ##" +{{ bin_dir }}/kubeadm {{ 'alpha ' if kube_version is version('v1.20.0', '<') else '' }}certs check-expiration diff --git a/roles/kubernetes/master/templates/k8s-certs-renew.timer.j2 b/roles/kubernetes/master/templates/k8s-certs-renew.timer.j2 new file mode 100644 index 00000000000..904f0073cf4 --- /dev/null +++ b/roles/kubernetes/master/templates/k8s-certs-renew.timer.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Timer to renew K8S control plane certificates + +[Timer] +OnCalendar={{ auto_renew_certificates_systemd_calendar }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 index 47570710c5c..b33963a0cd4 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 @@ -59,12 +59,12 @@ etcd: quota-backend-bytes: "{{ etcd_quota_backend_bytes }}" {% endif %} {% if etcd_log_package_levels is defined %} - log-package_levels: "{{ etcd_log_package_levels }}" + log-package-levels: "{{ etcd_log_package_levels }}" {% endif %} {% for key, value in etcd_extra_vars.items() %} {{ key }}: "{{ value }}" {% endfor %} -{% if host_architecture != "amd64" -%} +{% if host_architecture != "amd64" %} etcd-unsupported-arch: {{host_architecture}} {% endif %} serverCertSANs: diff --git a/roles/kubernetes/node/tasks/kubelet.yml b/roles/kubernetes/node/tasks/kubelet.yml index cb95cc174a8..8bff4077c1e 100644 --- a/roles/kubernetes/node/tasks/kubelet.yml +++ b/roles/kubernetes/node/tasks/kubelet.yml @@ -6,11 +6,6 @@ state: directory when: dynamic_kubelet_configuration -- name: Get the kubeadm version - command: "{{ bin_dir }}/kubeadm version -o short" - register: kubeadm_output - changed_when: false - - name: Set kubelet api version to v1beta1 set_fact: kubeletConfig_api_version: v1beta1 diff --git a/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml b/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml index fcbf6b5f274..972878bf79f 100644 --- a/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml +++ b/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml @@ -22,6 +22,9 @@ - name: haproxy | Get checksum from config stat: path: "{{ haproxy_config_dir }}/haproxy.cfg" + get_attributes: no + get_checksum: yes + get_mime: no register: haproxy_stat - name: haproxy | Write static pod diff --git a/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml b/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml index 91908791b6c..f90084cbc59 100644 --- a/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml +++ b/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml @@ -22,6 +22,9 @@ - name: nginx-proxy | Get checksum from config stat: path: "{{ nginx_config_dir }}/nginx.conf" + get_attributes: no + get_checksum: yes + get_mime: no register: nginx_stat - name: nginx-proxy | Write static pod diff --git a/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 index a557d6e6058..1efcbaede39 100644 --- a/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2 @@ -22,8 +22,6 @@ spec: requests: cpu: {{ loadbalancer_apiserver_cpu_requests }} memory: {{ loadbalancer_apiserver_memory_requests }} - securityContext: - privileged: true {% if loadbalancer_apiserver_healthcheck_port is defined -%} livenessProbe: httpGet: diff --git a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 index b05977d93ad..04b9b734e1b 100644 --- a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 @@ -22,8 +22,6 @@ spec: requests: cpu: {{ loadbalancer_apiserver_cpu_requests }} memory: {{ loadbalancer_apiserver_memory_requests }} - securityContext: - privileged: true {% if loadbalancer_apiserver_healthcheck_port is defined -%} livenessProbe: httpGet: diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index 6818de27a90..ec78c50b6d0 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -33,7 +33,6 @@ service: name: NetworkManager.service state: restarted - when: is_fedora_coreos - name: Preinstall | reload kubelet service: @@ -50,13 +49,21 @@ # FIXME(mattymo): Also restart for kubeadm mode - name: Preinstall | kube-apiserver configured - stat: path="{{ kube_manifest_dir }}/kube-apiserver.manifest" + stat: + path: "{{ kube_manifest_dir }}/kube-apiserver.manifest" + get_attributes: no + get_checksum: no + get_mime: no register: kube_apiserver_set when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' # FIXME(mattymo): Also restart for kubeadm mode - name: Preinstall | kube-controller configured - stat: path="{{ kube_manifest_dir }}/kube-controller-manager.manifest" + stat: + path: "{{ kube_manifest_dir }}/kube-controller-manager.manifest" + get_attributes: no + get_checksum: no + get_mime: no register: kube_controller_set when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' diff --git a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml index 1bb9e286bed..0e44298d934 100644 --- a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml @@ -9,6 +9,9 @@ - name: check if booted with ostree stat: path: /run/ostree-booted + get_attributes: no + get_checksum: no + get_mime: no register: ostree - name: set is_fedora_coreos @@ -59,6 +62,9 @@ - name: check if kubelet is configured stat: path: "{{ kube_config_dir }}/kubelet.env" + get_attributes: no + get_checksum: no + get_mime: no register: kubelet_configured changed_when: false @@ -84,6 +90,9 @@ - name: check if /etc/dhclient.conf exists stat: path: /etc/dhclient.conf + get_attributes: no + get_checksum: no + get_mime: no register: dhclient_stat - name: target dhclient conf file for /etc/dhclient.conf @@ -94,6 +103,9 @@ - name: check if /etc/dhcp/dhclient.conf exists stat: path: /etc/dhcp/dhclient.conf + get_attributes: no + get_checksum: no + get_mime: no register: dhcp_dhclient_stat - name: target dhclient conf file for /etc/dhcp/dhclient.conf @@ -170,6 +182,9 @@ - name: check /usr readonly stat: path: "/usr" + get_attributes: no + get_checksum: no + get_mime: no register: usr - name: set alternate flexvolume path diff --git a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml index 8be2b6c3a1f..4020406743b 100644 --- a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml +++ b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml @@ -46,6 +46,9 @@ - name: Check if kubernetes kubeadm compat cert dir exists stat: path: "{{ kube_cert_compat_dir }}" + get_attributes: no + get_checksum: no + get_mime: no register: kube_cert_compat_dir_check when: - inventory_hostname in groups['k8s-cluster'] diff --git a/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml b/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml new file mode 100644 index 00000000000..b8b673bd232 --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml @@ -0,0 +1,36 @@ +--- +- name: NetworkManager | Check if host has NetworkManager + # noqa 303 Should we use service_facts for this? + command: systemctl is-active --quiet NetworkManager.service + register: nm_check + failed_when: false + changed_when: false + +- name: NetworkManager | Ensure NetworkManager conf.d dir + file: + path: "/etc/NetworkManager/conf.d" + state: directory + recurse: yes + when: nm_check.rc == 0 + +- name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico) + copy: + content: | + [keyfile] + unmanaged-devices+=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico + dest: /etc/NetworkManager/conf.d/calico.conf + when: + - nm_check.rc == 0 + - kube_network_plugin == "calico" + notify: Preinstall | reload NetworkManager + +# TODO: add other network_plugin interfaces + +- name: NetworkManager | Prevent NetworkManager from managing K8S interfaces (kube-ipvs0/nodelocaldns) + copy: + content: | + [keyfile] + unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns + dest: /etc/NetworkManager/conf.d/k8s.conf + when: nm_check.rc == 0 + notify: Preinstall | reload NetworkManager diff --git a/roles/kubernetes/preinstall/tasks/0062-networkmanager.yml b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml similarity index 100% rename from roles/kubernetes/preinstall/tasks/0062-networkmanager.yml rename to roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml diff --git a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml index be0200921d6..96c9d20059a 100644 --- a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml +++ b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml @@ -3,6 +3,9 @@ - name: Confirm selinux deployed stat: path: /etc/selinux/config + get_attributes: no + get_checksum: no + get_mime: no when: - ansible_os_family == "RedHat" - "'Amazon' not in ansible_distribution" @@ -36,6 +39,9 @@ - name: Stat sysctl file configuration stat: path: "{{ sysctl_file_path }}" + get_attributes: no + get_checksum: no + get_mime: no register: sysctl_file_stat tags: - bootstrap-os @@ -74,3 +80,10 @@ - { name: kernel.panic, value: 10 } - { name: kernel.panic_on_oops, value: 1 } when: kubelet_protect_kernel_defaults|bool + +- name: Check dummy module + modprobe: + name: dummy + state: present + params: 'numdummies=0' + when: enable_nodelocaldns diff --git a/roles/kubernetes/preinstall/tasks/0090-etchosts.yml b/roles/kubernetes/preinstall/tasks/0090-etchosts.yml index 81913cb104f..491d24a6aba 100644 --- a/roles/kubernetes/preinstall/tasks/0090-etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/0090-etchosts.yml @@ -56,7 +56,7 @@ etc_hosts_localhosts_dict_target: >- {%- set target_entries = (etc_hosts_localhosts_dict|default({})).get(item.key, []) | difference(item.value.get('unexpected' ,[])) -%} {{ etc_hosts_localhosts_dict_target|default({}) | combine({item.key: (target_entries + item.value.expected)|unique}) }} - with_dict: "{{ etc_hosts_localhost_entries }}" + loop: "{{ etc_hosts_localhost_entries|dict2items }}" - name: Hosts | Update (if necessary) hosts file lineinfile: @@ -66,7 +66,7 @@ state: present backup: yes unsafe_writes: yes - with_dict: "{{ etc_hosts_localhosts_dict_target }}" + loop: "{{ etc_hosts_localhosts_dict_target|dict2items }}" # gather facts to update ansible_fqdn - name: Update facts diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 69a21ed13fe..2a3418b0ea1 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -39,7 +39,11 @@ - bootstrap-os - resolvconf -- import_tasks: 0062-networkmanager.yml +- import_tasks: 0062-networkmanager-unmanaged-devices.yml + tags: + - bootstrap-os + +- import_tasks: 0063-networkmanager-dns.yml when: - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' @@ -92,6 +96,9 @@ - name: Check if we are running inside a Azure VM stat: path: /var/lib/waagent/ + get_attributes: no + get_checksum: no + get_mime: no register: azure_check when: - not dns_late diff --git a/roles/kubernetes/tokens/tasks/check-tokens.yml b/roles/kubernetes/tokens/tasks/check-tokens.yml index 160f46bb8e0..c8fe3812fec 100644 --- a/roles/kubernetes/tokens/tasks/check-tokens.yml +++ b/roles/kubernetes/tokens/tasks/check-tokens.yml @@ -2,6 +2,9 @@ - name: "Check_tokens | check if the tokens have already been generated on first master" stat: path: "{{ kube_token_dir }}/known_tokens.csv" + get_attributes: no + get_checksum: yes + get_mime: no delegate_to: "{{ groups['kube-master'][0] }}" register: known_tokens_master run_once: true @@ -20,6 +23,9 @@ - name: "Check tokens | check if a cert already exists" stat: path: "{{ kube_token_dir }}/known_tokens.csv" + get_attributes: no + get_checksum: yes + get_mime: no register: known_tokens - name: "Check_tokens | Set 'sync_tokens' to true" diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 617591d6f74..1a4c326fe50 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -1,7 +1,7 @@ --- # Use proxycommand if bastion host is in group all # This change obseletes editing ansible.cfg file depending on bastion existence -ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p {{ hostvars['bastion']['ansible_user'] }}@{{ hostvars['bastion']['ansible_host'] }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} ' {% endif %}" +ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -W %h:%p -p {{ hostvars['bastion']['ansible_port'] | default(22) }} {{ hostvars['bastion']['ansible_user'] }}@{{ hostvars['bastion']['ansible_host'] }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %} ' {% endif %}" # selinux state preinstall_selinux_state: permissive @@ -15,7 +15,7 @@ is_fedora_coreos: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.19.8 +kube_version: v1.19.10 ## The minimum version working kube_version_min_required: v1.18.0 diff --git a/roles/network_plugin/calico/handlers/main.yml b/roles/network_plugin/calico/handlers/main.yml index 696729a24f3..bae5750473c 100644 --- a/roles/network_plugin/calico/handlers/main.yml +++ b/roles/network_plugin/calico/handlers/main.yml @@ -25,9 +25,3 @@ until: crictl_calico_node_remove is succeeded retries: 5 when: container_manager in ["crio", "containerd"] - -- name: Calico | Reload NetworkManager - service: - name: NetworkManager - state: reloaded - when: '"running" in nm_check.stdout' diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index f5a9afc3305..4911cbeea21 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -6,29 +6,6 @@ mode: 0755 remote_src: yes -- name: Calico | Check if host has NetworkManager - # noqa 303 Should we use service_facts for this? - command: systemctl is-active --quiet NetworkManager.service - register: nm_check - failed_when: false - changed_when: false - -- name: Calico | Ensure NetworkManager conf.d dir - file: - path: "/etc/NetworkManager/conf.d" - state: directory - recurse: yes - when: nm_check.rc == 0 - -- name: Calico | Prevent NetworkManager from managing Calico interfaces - copy: - content: | - [keyfile] - unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico - dest: /etc/NetworkManager/conf.d/calico.conf - when: nm_check.rc == 0 - notify: Calico | Reload NetworkManager - - name: Calico | Write Calico cni config template: src: "cni-calico.conflist.j2" @@ -108,33 +85,35 @@ - 'calico_conf.stdout == "0"' - calico_pool_cidr is defined -- name: Calico | Create calico manifests for kdd - template: - src: "{{ item.file }}.j2" - dest: "{{ kube_config_dir }}/{{ item.file }}" - with_items: - - {name: calico, file: kdd-crds.yml, type: kdd} - register: calico_node_kdd_manifest +- block: + - name: Calico | Remove unwanted annotations and creationTimestamp keys from metadata in Calico manifests + # noqa 303 - sed avoids using nested loop + shell: >- + sed -E -i + -e '/^\s{2,4}creationTimestamp: null$/d' + -e '/^\s{2,4}annotations:/{:1;/\(devel\)$/!{N;b 1}; /.*/d}' + {{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/*.yaml + when: + - calico_version is version('v3.17.0', '<') + + - name: Calico | Create calico manifests for kdd + assemble: + src: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds" + dest: "{{ kube_config_dir }}/kdd-crds.yml" + regexp: ".*\\.yaml" + remote_src: true + + - name: Calico | Create Calico Kubernetes datastore resources + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/kdd-crds.yml" + state: "latest" + when: + - inventory_hostname == groups['kube-master'][0] when: - inventory_hostname in groups['kube-master'] - calico_datastore == "kdd" -- name: Calico | Create Calico Kubernetes datastore resources - kube: - name: "{{ item.item.name }}" - namespace: "kube-system" - kubectl: "{{ bin_dir }}/kubectl" - resource: "{{ item.item.type }}" - filename: "{{ kube_config_dir }}/{{ item.item.file }}" - state: "latest" - with_items: - - "{{ calico_node_kdd_manifest.results }}" - when: - - inventory_hostname == groups['kube-master'][0] - - not item is skipped - loop_control: - label: "{{ item.item.file }}" - - name: Calico | Configure calico network pool command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" @@ -304,7 +283,6 @@ state: "latest" with_items: - "{{ calico_node_manifests.results }}" - - "{{ calico_node_kdd_manifest.results }}" - "{{ calico_node_typha_manifest.results }}" when: - inventory_hostname == groups['kube-master'][0] diff --git a/roles/network_plugin/calico/tasks/reset.yml b/roles/network_plugin/calico/tasks/reset.yml index 0d7a4bf72eb..5c32d0e25c4 100644 --- a/roles/network_plugin/calico/tasks/reset.yml +++ b/roles/network_plugin/calico/tasks/reset.yml @@ -2,6 +2,9 @@ - name: reset | check dummy0 network device stat: path: /sys/class/net/dummy0 + get_attributes: no + get_checksum: no + get_mime: no register: dummy0 - name: reset | remove the network device created by calico diff --git a/roles/network_plugin/calico/templates/kdd-crds.yml.j2 b/roles/network_plugin/calico/templates/kdd-crds.yml.j2 deleted file mode 100644 index ca3869ef4e6..00000000000 --- a/roles/network_plugin/calico/templates/kdd-crds.yml.j2 +++ /dev/null @@ -1,2987 +0,0 @@ -# Create all the CustomResourceDefinitions needed for - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: felixconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: FelixConfiguration - listKind: FelixConfigurationList - plural: felixconfigurations - singular: felixconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: Felix Configuration contains the configuration for Felix. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FelixConfigurationSpec contains the values of the Felix configuration. - properties: - awsSrcDstCheck: - description: 'Set source-destination-check on AWS EC2 instances. Accepted - value must be one of "DoNothing", "Enabled" or "Disabled". [Default: - DoNothing]' - enum: - - DoNothing - - Enable - - Disable - type: string - bpfConnectTimeLoadBalancingEnabled: - description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, - controls whether Felix installs the connection-time load balancer. The - connect-time load balancer is required for the host to be able to - reach Kubernetes services and it improves the performance of pod-to-service - connections. The only reason to disable it is for debugging purposes. [Default: - true]' - type: boolean - bpfDataIfacePattern: - description: 'BPFDataIfacePattern is a regular expression that controls - which interfaces Felix should attach BPF programs to in order to - catch traffic to/from the network. This needs to match the interfaces - that Calico workload traffic flows over as well as any interfaces - that handle incoming traffic to nodeports and services from outside - the cluster. It should not match the workload interfaces (usually - named cali...). [Default: ^(en.*|eth.*|tunl0$)]' - type: string - bpfDisableUnprivileged: - description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled - sysctl to disable unprivileged use of BPF. This ensures that unprivileged - users cannot access Calico''s BPF maps and cannot insert their own - BPF programs to interfere with Calico''s. [Default: true]' - type: boolean - bpfEnabled: - description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. - [Default: false]' - type: boolean - bpfExternalServiceMode: - description: 'BPFExternalServiceMode in BPF mode, controls how connections - from outside the cluster to services (node ports and cluster IPs) - are forwarded to remote workloads. If set to "Tunnel" then both - request and response traffic is tunneled to the remote node. If - set to "DSR", the request traffic is tunneled but the response traffic - is sent directly from the remote node. In "DSR" mode, the remote - node appears to use the IP of the ingress node; this requires a - permissive L2 network. [Default: Tunnel]' - type: string - bpfKubeProxyEndpointSlicesEnabled: - description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls - whether Felix's embedded kube-proxy accepts EndpointSlices or not. - type: boolean - bpfKubeProxyIptablesCleanupEnabled: - description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF - mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s - iptables chains. Should only be enabled if kube-proxy is not running. [Default: - true]' - type: boolean - bpfKubeProxyMinSyncPeriod: - description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the - minimum time between updates to the dataplane for Felix''s embedded - kube-proxy. Lower values give reduced set-up latency. Higher values - reduce Felix CPU usage by batching up more work. [Default: 1s]' - type: string - bpfLogLevel: - description: 'BPFLogLevel controls the log level of the BPF programs - when in BPF dataplane mode. One of "Off", "Info", or "Debug". The - logs are emitted to the BPF trace pipe, accessible with the command - `tc exec bpf debug`. [Default: Off].' - type: string - chainInsertMode: - description: 'ChainInsertMode controls whether Felix hooks the kernel’s - top-level iptables chains by inserting a rule at the top of the - chain or by appending a rule at the bottom. insert is the safe default - since it prevents Calico’s rules from being bypassed. If you switch - to append mode, be sure that the other rules in the chains signal - acceptance by falling through to the Calico rules, otherwise the - Calico policy will be bypassed. [Default: insert]' - type: string - dataplaneDriver: - type: string - debugDisableLogDropping: - type: boolean - debugMemoryProfilePath: - type: string - debugSimulateCalcGraphHangAfter: - type: string - debugSimulateDataplaneHangAfter: - type: string - defaultEndpointToHostAction: - description: 'DefaultEndpointToHostAction controls what happens to - traffic that goes from a workload endpoint to the host itself (after - the traffic hits the endpoint egress policy). By default Calico - blocks traffic from workload endpoints to the host itself with an - iptables “DROP” action. If you want to allow some or all traffic - from endpoint to host, set this parameter to RETURN or ACCEPT. Use - RETURN if you have your own rules in the iptables “INPUT” chain; - Calico will insert its rules at the top of that chain, then “RETURN” - packets to the “INPUT” chain once it has completed processing workload - endpoint egress policy. Use ACCEPT to unconditionally accept packets - from workloads after processing workload endpoint egress policy. - [Default: Drop]' - type: string - deviceRouteProtocol: - description: This defines the route protocol added to programmed device - routes, by default this will be RTPROT_BOOT when left blank. - type: integer - deviceRouteSourceAddress: - description: This is the source address to use on programmed device - routes. By default the source address is left blank, leaving the - kernel to choose the source address used. - type: string - disableConntrackInvalidCheck: - type: boolean - endpointReportingDelay: - type: string - endpointReportingEnabled: - type: boolean - externalNodesList: - description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes - which may source tunnel traffic and have the tunneled traffic be - accepted at calico nodes. - items: - type: string - type: array - failsafeInboundHostPorts: - description: 'FailsafeInboundHostPorts is a comma-delimited list of - UDP/TCP ports that Felix will allow incoming traffic to host endpoints - on irrespective of the security policy. This is useful to avoid - accidentally cutting off a host with incorrect configuration. Each - port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all inbound host ports, use the value none. - The default value allows ssh access and DHCP. [Default: tcp:22, - udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' - items: - description: ProtoPort is combination of protocol and port, both - must be specified. - properties: - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - failsafeOutboundHostPorts: - description: 'FailsafeOutboundHostPorts is a comma-delimited list - of UDP/TCP ports that Felix will allow outgoing traffic from host - endpoints to irrespective of the security policy. This is useful - to avoid accidentally cutting off a host with incorrect configuration. - Each port should be specified as tcp: or udp:. - For back-compatibility, if the protocol is not specified, it defaults - to “tcp”. To disable all outbound host ports, use the value none. - The default value opens etcd’s standard ports to ensure that Felix - does not get cut off from etcd as well as allowing DHCP and DNS. - [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667, - udp:53, udp:67]' - items: - description: ProtoPort is combination of protocol and port, both - must be specified. - properties: - port: - type: integer - protocol: - type: string - required: - - port - - protocol - type: object - type: array - genericXDPEnabled: - description: 'GenericXDPEnabled enables Generic XDP so network cards - that don''t support XDP offload or driver modes can use XDP. This - is not recommended since it doesn''t provide better performance - than iptables. [Default: false]' - type: boolean - healthEnabled: - type: boolean - healthHost: - type: string - healthPort: - type: integer - interfaceExclude: - description: 'InterfaceExclude is a comma-separated list of interfaces - that Felix should exclude when monitoring for host endpoints. The - default value ensures that Felix ignores Kubernetes'' IPVS dummy - interface, which is used internally by kube-proxy. If you want to - exclude multiple interface names using a single value, the list - supports regular expressions. For regular expressions you must wrap - the value with ''/''. For example having values ''/^kube/,veth1'' - will exclude all interfaces that begin with ''kube'' and also the - interface ''veth1''. [Default: kube-ipvs0]' - type: string - interfacePrefix: - description: 'InterfacePrefix is the interface name prefix that identifies - workload endpoints and so distinguishes them from host endpoint - interfaces. Note: in environments other than bare metal, the orchestrators - configure this appropriately. For example our Kubernetes and Docker - integrations set the ‘cali’ value, and our OpenStack integration - sets the ‘tap’ value. [Default: cali]' - type: string - ipipEnabled: - type: boolean - ipipMTU: - description: 'IPIPMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - ipsetsRefreshInterval: - description: 'IpsetsRefreshInterval is the period at which Felix re-checks - all iptables state to ensure that no other process has accidentally - broken Calico’s rules. Set to 0 to disable iptables refresh. [Default: - 90s]' - type: string - iptablesBackend: - description: IptablesBackend specifies which backend of iptables will - be used. The default is legacy. - type: string - iptablesFilterAllowAction: - type: string - iptablesLockFilePath: - description: 'IptablesLockFilePath is the location of the iptables - lock file. You may need to change this if the lock file is not in - its standard location (for example if you have mapped it into Felix’s - container at a different path). [Default: /run/xtables.lock]' - type: string - iptablesLockProbeInterval: - description: 'IptablesLockProbeInterval is the time that Felix will - wait between attempts to acquire the iptables lock if it is not - available. Lower values make Felix more responsive when the lock - is contended, but use more CPU. [Default: 50ms]' - type: string - iptablesLockTimeout: - description: 'IptablesLockTimeout is the time that Felix will wait - for the iptables lock, or 0, to disable. To use this feature, Felix - must share the iptables lock file with all other processes that - also take the lock. When running Felix inside a container, this - requires the /run directory of the host to be mounted into the calico/node - or calico/felix container. [Default: 0s disabled]' - type: string - iptablesMangleAllowAction: - type: string - iptablesMarkMask: - description: 'IptablesMarkMask is the mask that Felix selects its - IPTables Mark bits from. Should be a 32 bit hexadecimal number with - at least 8 bits set, none of which clash with any other mark bits - in use on the system. [Default: 0xff000000]' - format: int32 - type: integer - iptablesNATOutgoingInterfaceFilter: - type: string - iptablesPostWriteCheckInterval: - description: 'IptablesPostWriteCheckInterval is the period after Felix - has done a write to the dataplane that it schedules an extra read - back in order to check the write was not clobbered by another process. - This should only occur if another application on the system doesn’t - respect the iptables lock. [Default: 1s]' - type: string - iptablesRefreshInterval: - description: 'IptablesRefreshInterval is the period at which Felix - re-checks the IP sets in the dataplane to ensure that no other process - has accidentally broken Calico’s rules. Set to 0 to disable IP sets - refresh. Note: the default for this value is lower than the other - refresh intervals as a workaround for a Linux kernel bug that was - fixed in kernel version 4.11. If you are using v4.11 or greater - you may want to set this to, a higher value to reduce Felix CPU - usage. [Default: 10s]' - type: string - ipv6Support: - type: boolean - kubeNodePortRanges: - description: 'KubeNodePortRanges holds list of port ranges used for - service node ports. Only used if felix detects kube-proxy running - in ipvs mode. Felix uses these ranges to separate host and workload - traffic. [Default: 30000:32767].' - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - logFilePath: - description: 'LogFilePath is the full path to the Felix log. Set to - none to disable file logging. [Default: /var/log/calico/felix.log]' - type: string - logPrefix: - description: 'LogPrefix is the log prefix that Felix uses when rendering - LOG rules. [Default: calico-packet]' - type: string - logSeverityFile: - description: 'LogSeverityFile is the log severity above which logs - are sent to the log file. [Default: Info]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - logSeveritySys: - description: 'LogSeveritySys is the log severity above which logs - are sent to the syslog. Set to None for no logging to syslog. [Default: - Info]' - type: string - maxIpsetSize: - type: integer - metadataAddr: - description: 'MetadataAddr is the IP address or domain name of the - server that can answer VM queries for cloud-init metadata. In OpenStack, - this corresponds to the machine running nova-api (or in Ubuntu, - nova-api-metadata). A value of none (case insensitive) means that - Felix should not set up any NAT rule for the metadata path. [Default: - 127.0.0.1]' - type: string - metadataPort: - description: 'MetadataPort is the port of the metadata server. This, - combined with global.MetadataAddr (if not ‘None’), is used to set - up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. - In most cases this should not need to be changed [Default: 8775].' - type: integer - natOutgoingAddress: - description: NATOutgoingAddress specifies an address to use when performing - source NAT for traffic in a natOutgoing pool that is leaving the - network. By default the address used is an address on the interface - the traffic is leaving on (ie it uses the iptables MASQUERADE target) - type: string - natPortRange: - anyOf: - - type: integer - - type: string - description: NATPortRange specifies the range of ports that is used - for port mapping when doing outgoing NAT. When unset the default - behavior of the network stack is used. - pattern: ^.* - x-kubernetes-int-or-string: true - netlinkTimeout: - type: string - openstackRegion: - description: 'OpenstackRegion is the name of the region that a particular - Felix belongs to. In a multi-region Calico/OpenStack deployment, - this must be configured somehow for each Felix (here in the datamodel, - or in felix.cfg or the environment on each compute node), and must - match the [calico] openstack_region value configured in neutron.conf - on each node. [Default: Empty]' - type: string - policySyncPathPrefix: - description: 'PolicySyncPathPrefix is used to by Felix to communicate - policy changes to external services, like Application layer policy. - [Default: Empty]' - type: string - prometheusGoMetricsEnabled: - description: 'PrometheusGoMetricsEnabled disables Go runtime metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - prometheusMetricsEnabled: - description: 'PrometheusMetricsEnabled enables the Prometheus metrics - server in Felix if set to true. [Default: false]' - type: boolean - prometheusMetricsHost: - description: 'PrometheusMetricsHost is the host that the Prometheus - metrics server should bind to. [Default: empty]' - type: string - prometheusMetricsPort: - description: 'PrometheusMetricsPort is the TCP port that the Prometheus - metrics server should bind to. [Default: 9091]' - type: integer - prometheusProcessMetricsEnabled: - description: 'PrometheusProcessMetricsEnabled disables process metrics - collection, which the Prometheus client does by default, when set - to false. This reduces the number of metrics reported, reducing - Prometheus load. [Default: true]' - type: boolean - removeExternalRoutes: - description: Whether or not to remove device routes that have not - been programmed by Felix. Disabling this will allow external applications - to also add device routes. This is enabled by default which means - we will remove externally added routes. - type: boolean - reportingInterval: - description: 'ReportingInterval is the interval at which Felix reports - its status into the datastore or 0 to disable. Must be non-zero - in OpenStack deployments. [Default: 30s]' - type: string - reportingTTL: - description: 'ReportingTTL is the time-to-live setting for process-wide - status reports. [Default: 90s]' - type: string - routeRefreshInterval: - description: 'RouterefreshInterval is the period at which Felix re-checks - the routes in the dataplane to ensure that no other process has - accidentally broken Calico’s rules. Set to 0 to disable route refresh. - [Default: 90s]' - type: string - routeSource: - description: 'RouteSource configures where Felix gets its routing - information. - WorkloadIPs: use workload endpoints to construct - routes. - CalicoIPAM: the default - use IPAM data to construct routes.' - type: string - routeTableRange: - description: Calico programs additional Linux route tables for various - purposes. RouteTableRange specifies the indices of the route tables - that Calico should use. - properties: - max: - type: integer - min: - type: integer - required: - - max - - min - type: object - sidecarAccelerationEnabled: - description: 'SidecarAccelerationEnabled enables experimental sidecar - acceleration [Default: false]' - type: boolean - usageReportingEnabled: - description: 'UsageReportingEnabled reports anonymous Calico version - number and cluster size to projectcalico.org. Logs warnings returned - by the usage server. For example, if a significant security vulnerability - has been discovered in the version of Calico being used. [Default: - true]' - type: boolean - usageReportingInitialDelay: - description: 'UsageReportingInitialDelay controls the minimum delay - before Felix makes a report. [Default: 300s]' - type: string - usageReportingInterval: - description: 'UsageReportingInterval controls the interval at which - Felix makes reports. [Default: 86400s]' - type: string - useInternalDataplaneDriver: - type: boolean - vxlanEnabled: - type: boolean - vxlanMTU: - description: 'VXLANMTU is the MTU to set on the tunnel device. See - Configuring MTU [Default: 1440]' - type: integer - vxlanPort: - type: integer - vxlanVNI: - type: integer - wireguardEnabled: - description: 'WireguardEnabled controls whether Wireguard is enabled. - [Default: false]' - type: boolean - wireguardInterfaceName: - description: 'WireguardInterfaceName specifies the name to use for - the Wireguard interface. [Default: wg.calico]' - type: string - wireguardListeningPort: - description: 'WireguardListeningPort controls the listening port used - by Wireguard. [Default: 51820]' - type: integer - wireguardMTU: - description: 'WireguardMTU controls the MTU on the Wireguard interface. - See Configuring MTU [Default: 1420]' - type: integer - wireguardRoutingRulePriority: - description: 'WireguardRoutingRulePriority controls the priority value - to use for the Wireguard routing rule. [Default: 99]' - type: integer - xdpEnabled: - description: 'XDPEnabled enables XDP acceleration for suitable untracked - incoming deny rules. [Default: true]' - type: boolean - xdpRefreshInterval: - description: 'XDPRefreshInterval is the period at which Felix re-checks - all XDP state to ensure that no other process has accidentally broken - Calico''s BPF maps or attached programs. Set to 0 to disable XDP - refresh. [Default: 90s]' - type: string - required: - - bpfLogLevel - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamblocks.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMBlock - listKind: IPAMBlockList - plural: ipamblocks - singular: ipamblock - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMBlockSpec contains the specification for an IPAMBlock - resource. - properties: - affinity: - type: string - allocations: - items: - type: integer - # TODO: This nullable is manually added in. We should update controller-gen - # to handle []*int properly itself. - nullable: true - type: array - attributes: - items: - properties: - handle_id: - type: string - secondary: - additionalProperties: - type: string - type: object - type: object - type: array - cidr: - type: string - deleted: - type: boolean - strictAffinity: - type: boolean - unallocated: - items: - type: integer - type: array - required: - - allocations - - attributes - - cidr - - deleted - - strictAffinity - - unallocated - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: blockaffinities.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BlockAffinity - listKind: BlockAffinityList - plural: blockaffinities - singular: blockaffinity - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BlockAffinitySpec contains the specification for a BlockAffinity - resource. - properties: - cidr: - type: string - deleted: - description: Deleted indicates that this block affinity is being deleted. - This field is a string for compatibility with older releases that - mistakenly treat this field as a string. - type: string - node: - type: string - state: - type: string - required: - - cidr - - deleted - - node - - state - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamhandles.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMHandle - listKind: IPAMHandleList - plural: ipamhandles - singular: ipamhandle - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMHandleSpec contains the specification for an IPAMHandle - resource. - properties: - block: - additionalProperties: - type: integer - type: object - handleID: - type: string - required: - - block - - handleID - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ipamconfigs.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPAMConfig - listKind: IPAMConfigList - plural: ipamconfigs - singular: ipamconfig - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPAMConfigSpec contains the specification for an IPAMConfig - resource. - properties: - autoAllocateBlocks: - type: boolean - strictAffinity: - type: boolean - required: - - autoAllocateBlocks - - strictAffinity - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgppeers.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPPeer - listKind: BGPPeerList - plural: bgppeers - singular: bgppeer - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPPeerSpec contains the specification for a BGPPeer resource. - properties: - asNumber: - description: The AS Number of the peer. - format: int32 - type: integer - keepOriginalNextHop: - description: Option to keep the original nexthop field when routes - are sent to a BGP Peer. Setting "true" configures the selected BGP - Peers node to use the "next hop keep;" instead of "next hop self;"(default) - in the specific branch of the Node on "bird.cfg". - type: boolean - node: - description: The node name identifying the Calico node instance that - is peering with this peer. If this is not set, this represents a - global peer, i.e. a peer that peers with every node in the deployment. - type: string - nodeSelector: - description: Selector for the nodes that should have this peering. When - this is set, the Node field must be empty. - type: string - password: - description: Optional BGP password for the peerings generated by this - BGPPeer resource. - properties: - secretKeyRef: - description: Selects a key of a secret in the node pod's namespace. - properties: - key: - description: The key of the secret to select from. Must be - a valid secret key. - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: Specify whether the Secret or its key must be - defined - type: boolean - required: - - key - type: object - type: object - peerIP: - description: The IP address of the peer followed by an optional port - number to peer with. If port number is given, format should be `[]:port` - or `:` for IPv4. If optional port number is not set, - and this peer IP and ASNumber belongs to a calico/node with ListenPort - set in BGPConfiguration, then we use that port to peer. - type: string - peerSelector: - description: Selector for the remote nodes to peer with. When this - is set, the PeerIP and ASNumber fields must be empty. For each - peering between the local node and selected remote nodes, we configure - an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, - and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The - remote AS number comes from the remote node’s NodeBGPSpec.ASNumber, - or the global default if that is not set. - type: string - required: - - asNumber - - peerIP - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: bgpconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: BGPConfiguration - listKind: BGPConfigurationList - plural: bgpconfigurations - singular: bgpconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: BGPConfiguration contains the configuration for any BGP routing. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BGPConfigurationSpec contains the values of the BGP configuration. - properties: - asNumber: - description: 'ASNumber is the default AS number used by a node. [Default: - 64512]' - format: int32 - type: integer - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: INFO]' - type: string - nodeToNodeMeshEnabled: - description: 'NodeToNodeMeshEnabled sets whether full node to node - BGP mesh is enabled. [Default: true]' - type: boolean - serviceClusterIPs: - description: ServiceClusterIPs are the CIDR blocks from which service - cluster IPs are allocated. If specified, Calico will advertise these - blocks, as well as any cluster IPs within them. - items: - description: ServiceClusterIPBlock represents a single whitelisted - CIDR block for ClusterIPs. - properties: - cidr: - type: string - type: object - type: array - serviceExternalIPs: - description: ServiceExternalIPs are the CIDR blocks for Kubernetes - Service External IPs. Kubernetes Service ExternalIPs will only be - advertised if they are within one of these blocks. - items: - description: ServiceExternalIPBlock represents a single whitelisted - CIDR External IP block. - properties: - cidr: - type: string - type: object - type: array - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ippools.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: IPPool - listKind: IPPoolList - plural: ippools - singular: ippool - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: IPPoolSpec contains the specification for an IPPool resource. - properties: - blockSize: - description: The block size to use for IP address assignments from - this pool. Defaults to 26 for IPv4 and 112 for IPv6. - type: integer - cidr: - description: The pool CIDR. - type: string - disabled: - description: When disabled is true, Calico IPAM will not assign addresses - from this pool. - type: boolean - ipip: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - properties: - enabled: - description: When enabled is true, ipip tunneling will be used - to deliver packets to destinations within this pool. - type: boolean - mode: - description: The IPIP mode. This can be one of "always" or "cross-subnet". A - mode of "always" will also use IPIP tunneling for routing to - destination IP addresses within this pool. A mode of "cross-subnet" - will only use IPIP tunneling when the destination node is on - a different subnet to the originating node. The default value - (if not specified) is "always". - type: string - type: object - ipipMode: - description: Contains configuration for IPIP tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. IPIP tunelling - is disabled). - type: string - nat-outgoing: - description: 'Deprecated: this field is only used for APIv1 backwards - compatibility. Setting this field is not allowed, this field is - for internal use only.' - type: boolean - natOutgoing: - description: When nat-outgoing is true, packets sent from Calico networked - containers in this pool to destinations outside of this pool will - be masqueraded. - type: boolean - nodeSelector: - description: Allows IPPool to allocate for a specific node by label - selector. - type: string - vxlanMode: - description: Contains configuration for VXLAN tunneling for this pool. - If not specified, then this is defaulted to "Never" (i.e. VXLAN - tunelling is disabled). - type: string - required: - - cidr - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: hostendpoints.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: HostEndpoint - listKind: HostEndpointList - plural: hostendpoints - singular: hostendpoint - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HostEndpointSpec contains the specification for a HostEndpoint - resource. - properties: - expectedIPs: - description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. - If \"InterfaceName\" is not present, Calico will look for an interface - matching any of the IPs in the list and apply policy to that. Note: - \tWhen using the selector match criteria in an ingress or egress - security Policy \tor Profile, Calico converts the selector into - a set of IP addresses. For host \tendpoints, the ExpectedIPs field - is used for that purpose. (If only the interface \tname is specified, - Calico does not learn the IPs of the interface for use in match - \tcriteria.)" - items: - type: string - type: array - interfaceName: - description: "Either \"*\", or the name of a specific Linux interface - to apply policy to; or empty. \"*\" indicates that this HostEndpoint - governs all traffic to, from or through the default network namespace - of the host named by the \"Node\" field; entering and leaving that - namespace via any interface, including those from/to non-host-networked - local workloads. \n If InterfaceName is not \"*\", this HostEndpoint - only governs traffic that enters or leaves the host through the - specific interface named by InterfaceName, or - when InterfaceName - is empty - through the specific interface that has one of the IPs - in ExpectedIPs. Therefore, when InterfaceName is empty, at least - one expected IP must be specified. Only external interfaces (such - as “eth0”) are supported here; it isn't possible for a HostEndpoint - to protect traffic through a specific local workload interface. - \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; - initially just pre-DNAT policy. Please check Calico documentation - for the latest position." - type: string - node: - description: The node name identifying the Calico node instance. - type: string - ports: - description: Ports contains the endpoint's named ports, which may - be referenced in security policy rules. - items: - properties: - name: - type: string - port: - type: integer - protocol: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - required: - - name - - port - - protocol - type: object - type: array - profiles: - description: A list of identifiers of security Profile objects that - apply to this endpoint. Each profile is applied in the order that - they appear in this list. Profile rules are applied after the selector-based - security policy. - items: - type: string - type: array - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusterinformations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: ClusterInformation - listKind: ClusterInformationList - plural: clusterinformations - singular: clusterinformation - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: ClusterInformation contains the cluster specific information. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ClusterInformationSpec contains the values of describing - the cluster. - properties: - calicoVersion: - description: CalicoVersion is the version of Calico that the cluster - is running - type: string - clusterGUID: - description: ClusterGUID is the GUID of the cluster - type: string - clusterType: - description: ClusterType describes the type of the cluster - type: string - datastoreReady: - description: DatastoreReady is used during significant datastore migrations - to signal to components such as Felix that it should wait before - accessing the datastore. - type: boolean - variant: - description: Variant declares which variant of Calico should be active. - type: string - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkPolicy - listKind: GlobalNetworkPolicyList - plural: globalnetworkpolicies - singular: globalnetworkpolicy - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - applyOnForward: - description: ApplyOnForward indicates to apply the rules in this policy - on forward traffic. - type: boolean - doNotTrack: - description: DoNotTrack indicates whether packets matched by the rules - in this policy should go through the data plane's connection tracking, - such as Linux conntrack. If True, the rules in this policy are - applied before any data plane connection tracking, and packets allowed - by this policy are marked as not to be tracked. - type: boolean - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - namespaceSelector: - description: NamespaceSelector is an optional field for an expression - used to select a pod based on namespaces. - type: string - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - preDNAT: - description: PreDNAT indicates to apply the rules in this policy before - any DNAT. - type: boolean - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress rules are present in the policy. The - default is: \n - [ PolicyTypeIngress ], if there are no Egress rules - (including the case where there are also no Ingress rules) \n - - [ PolicyTypeEgress ], if there are Egress rules but no Ingress - rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are - both Ingress and Egress rules. \n When the policy is read back again, - Types will always be one of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: globalnetworksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: GlobalNetworkSet - listKind: GlobalNetworkSetList - plural: globalnetworksets - singular: globalnetworkset - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs - that share labels to allow rules to refer to them via selectors. The labels - of GlobalNetworkSet are not namespaced. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: GlobalNetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networkpolicies.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkPolicy - listKind: NetworkPolicyList - plural: networkpolicies - singular: networkpolicy - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - egress: - description: The ordered set of egress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - ingress: - description: The ordered set of ingress rules. Each rule contains - a set of packet match criteria and a corresponding action to apply. - items: - description: "A Rule encapsulates a set of match criteria and an - action. Both selector-based security Policy and security Profiles - reference rules - separated out as a list of rules for both ingress - and egress packet matching. \n Each positive match criteria has - a negated version, prefixed with ”Not”. All the match criteria - within a rule must be satisfied for a packet to match. A single - rule can contain the positive and negative version of a match - and both must be satisfied for the rule to match." - properties: - action: - type: string - destination: - description: Destination contains the match criteria that apply - to destination entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - http: - description: HTTP contains match criteria that apply to HTTP - requests. - properties: - methods: - description: Methods is an optional field that restricts - the rule to apply only to HTTP requests that use one of - the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple - methods are OR'd together. - items: - type: string - type: array - paths: - description: 'Paths is an optional field that restricts - the rule to apply to HTTP requests that use one of the - listed HTTP Paths. Multiple paths are OR''d together. - e.g: - exact: /foo - prefix: /bar NOTE: Each entry may - ONLY specify either a `exact` or a `prefix` match. The - validator will check for it.' - items: - description: 'HTTPPath specifies an HTTP path to match. - It may be either of the form: exact: : which matches - the path exactly or prefix: : which matches - the path prefix' - properties: - exact: - type: string - prefix: - type: string - type: object - type: array - type: object - icmp: - description: ICMP is an optional field that restricts the rule - to apply to a specific type and code of ICMP traffic. This - should only be specified if the Protocol field is set to "ICMP" - or "ICMPv6". - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - ipVersion: - description: IPVersion is an optional field that restricts the - rule to only match a specific IP version. - type: integer - metadata: - description: Metadata contains additional information for this - rule - properties: - annotations: - additionalProperties: - type: string - description: Annotations is a set of key value pairs that - give extra information about the rule - type: object - type: object - notICMP: - description: NotICMP is the negated version of the ICMP field. - properties: - code: - description: Match on a specific ICMP code. If specified, - the Type value must also be specified. This is a technical - limitation imposed by the kernel’s iptables firewall, - which Calico uses to enforce the rule. - type: integer - type: - description: Match on a specific ICMP type. For example - a value of 8 refers to ICMP Echo Request (i.e. pings). - type: integer - type: object - notProtocol: - anyOf: - - type: integer - - type: string - description: NotProtocol is the negated version of the Protocol - field. - pattern: ^.* - x-kubernetes-int-or-string: true - protocol: - anyOf: - - type: integer - - type: string - description: "Protocol is an optional field that restricts the - rule to only apply to traffic of a specific IP protocol. Required - if any of the EntityRules contain Ports (because ports only - apply to certain protocols). \n Must be one of these string - values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", - \"UDPLite\" or an integer in the range 1-255." - pattern: ^.* - x-kubernetes-int-or-string: true - source: - description: Source contains the match criteria that apply to - source entity. - properties: - namespaceSelector: - description: "NamespaceSelector is an optional field that - contains a selector expression. Only traffic that originates - from (or terminates at) endpoints within the selected - namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." - type: string - nets: - description: Nets is an optional field that restricts the - rule to only apply to traffic that originates from (or - terminates at) IP addresses in any of the given subnets. - items: - type: string - type: array - notNets: - description: NotNets is the negated version of the Nets - field. - items: - type: string - type: array - notPorts: - description: NotPorts is the negated version of the Ports - field. Since only some protocols have ports, if any ports - are specified it requires the Protocol match in the Rule - to be set to "TCP" or "UDP". - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - notSelector: - description: NotSelector is the negated version of the Selector - field. See Selector field for subtleties with negated - selectors. - type: string - ports: - description: "Ports is an optional field that restricts - the rule to only apply to traffic that has a source (destination) - port that matches one of these ranges/values. This value - is a list of integers or strings that represent ranges - of ports. \n Since only some protocols have ports, if - any ports are specified it requires the Protocol match - in the Rule to be set to \"TCP\" or \"UDP\"." - items: - anyOf: - - type: integer - - type: string - pattern: ^.* - x-kubernetes-int-or-string: true - type: array - selector: - description: "Selector is an optional field that contains - a selector expression (see Policy for sample syntax). - \ Only traffic that originates from (terminates at) endpoints - matching the selector will be matched. \n Note that: in - addition to the negated version of the Selector (see NotSelector - below), the selector expression syntax itself supports - negation. The two types of negation are subtly different. - One negates the set of matched endpoints, the other negates - the whole match: \n \tSelector = \"!has(my_label)\" matches - packets that are from other Calico-controlled \tendpoints - that do not have the label “my_label”. \n \tNotSelector - = \"has(my_label)\" matches packets that are not from - Calico-controlled \tendpoints that do have the label “my_label”. - \n The effect is that the latter will accept packets from - non-Calico sources whereas the former is limited to packets - from Calico-controlled endpoints." - type: string - serviceAccounts: - description: ServiceAccounts is an optional field that restricts - the rule to only apply to traffic that originates from - (or terminates at) a pod running as a matching service - account. - properties: - names: - description: Names is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account whose name is in the list. - items: - type: string - type: array - selector: - description: Selector is an optional field that restricts - the rule to only apply to traffic that originates - from (or terminates at) a pod running as a service - account that matches the given label selector. If - both Names and Selector are specified then they are - AND'ed. - type: string - type: object - type: object - required: - - action - type: object - type: array - order: - description: Order is an optional field that specifies the order in - which the policy is applied. Policies with higher "order" are applied - after those with lower order. If the order is omitted, it may be - considered to be "infinite" - i.e. the policy will be applied last. Policies - with identical order will be applied in alphanumerical order based - on the Policy "Name". - type: number - selector: - description: "The selector is an expression used to pick pick out - the endpoints that the policy should be applied to. \n Selector - expressions follow this syntax: \n \tlabel == \"string_literal\" - \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" - \ -> not equal; also matches if label is not present \tlabel in - { \"a\", \"b\", \"c\", ... } -> true if the value of label X is - one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", - ... } -> true if the value of label X is not one of \"a\", \"b\", - \"c\" \thas(label_name) -> True if that label is present \t! expr - -> negation of expr \texpr && expr -> Short-circuit and \texpr - || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() - or the empty selector -> matches all endpoints. \n Label names are - allowed to contain alphanumerics, -, _ and /. String literals are - more permissive but they do not support escape characters. \n Examples - (with made-up labels): \n \ttype == \"webserver\" && deployment - == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != - \"dev\" \t! has(label_name)" - type: string - serviceAccountSelector: - description: ServiceAccountSelector is an optional field for an expression - used to select a pod based on service accounts. - type: string - types: - description: "Types indicates whether this policy applies to ingress, - or to egress, or to both. When not explicitly specified (and so - the value on creation is empty or nil), Calico defaults Types according - to what Ingress and Egress are present in the policy. The default - is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including - the case where there are also no Ingress rules) \n - [ PolicyTypeEgress - ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, - PolicyTypeEgress ], if there are both Ingress and Egress rules. - \n When the policy is read back again, Types will always be one - of these values, never empty or nil." - items: - description: PolicyType enumerates the possible values of the PolicySpec - Types field. - type: string - type: array - type: object - type: object - served: true - storage: true - ---- - -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: networksets.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: NetworkSet - listKind: NetworkSetList - plural: networksets - singular: networkset - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NetworkSetSpec contains the specification for a NetworkSet - resource. - properties: - nets: - description: The list of IP networks that belong to this set. - items: - type: string - type: array - type: object - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: kubecontrollersconfigurations.crd.projectcalico.org -spec: - group: crd.projectcalico.org - names: - kind: KubeControllersConfiguration - listKind: KubeControllersConfigurationList - plural: kubecontrollersconfigurations - singular: kubecontrollersconfiguration - scope: Cluster - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KubeControllersConfigurationSpec contains the values of the - Kubernetes controllers configuration. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host endpoints. - Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation of - host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform reconciliation - with the Calico datastore. [Default: 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which logs - are sent to the stdout. [Default: Info]' - type: string - required: - - controllers - type: object - status: - description: KubeControllersConfigurationStatus represents the status - of the configuration. It's useful for admins to be able to see the actual - config that was applied, which can be modified by environment variables - on the kube-controllers process. - properties: - environmentVars: - additionalProperties: - type: string - description: EnvironmentVars contains the environment variables on - the kube-controllers that influenced the RunningConfig. - type: object - runningConfig: - description: RunningConfig contains the effective config that is running - in the kube-controllers pod, after merging the API resource with - any environment variables. - properties: - controllers: - description: Controllers enables and configures individual Kubernetes - controllers - properties: - namespace: - description: Namespace enables and configures the namespace - controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - node: - description: Node enables and configures the node controller. - Enabled by default, set to nil to disable. - properties: - hostEndpoint: - description: HostEndpoint controls syncing nodes to host - endpoints. Disabled by default, set to nil to disable. - properties: - autoCreate: - description: 'AutoCreate enables automatic creation - of host endpoints for every node. [Default: Disabled]' - type: string - type: object - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - syncLabels: - description: 'SyncLabels controls whether to copy Kubernetes - node labels to Calico nodes. [Default: Enabled]' - type: string - type: object - policy: - description: Policy enables and configures the policy controller. - Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - serviceAccount: - description: ServiceAccount enables and configures the service - account controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - workloadEndpoint: - description: WorkloadEndpoint enables and configures the workload - endpoint controller. Enabled by default, set to nil to disable. - properties: - reconcilerPeriod: - description: 'ReconcilerPeriod is the period to perform - reconciliation with the Calico datastore. [Default: - 5m]' - type: string - type: object - type: object - etcdV3CompactionPeriod: - description: 'EtcdV3CompactionPeriod is the period between etcdv3 - compaction requests. Set to 0 to disable. [Default: 10m]' - type: string - healthChecks: - description: 'HealthChecks enables or disables support for health - checks [Default: Enabled]' - type: string - logSeverityScreen: - description: 'LogSeverityScreen is the log severity above which - logs are sent to the stdout. [Default: Info]' - type: string - required: - - controllers - type: object - type: object - type: object - served: true - storage: true diff --git a/roles/network_plugin/cilium/tasks/reset_iface.yml b/roles/network_plugin/cilium/tasks/reset_iface.yml index d4a5aad90f5..d84a065af67 100644 --- a/roles/network_plugin/cilium/tasks/reset_iface.yml +++ b/roles/network_plugin/cilium/tasks/reset_iface.yml @@ -2,6 +2,9 @@ - name: "reset | check if network device {{ iface }} is present" stat: path: "/sys/class/net/{{ iface }}" + get_attributes: no + get_checksum: no + get_mime: no register: device_remains - name: "reset | remove network device {{ iface }}" diff --git a/roles/network_plugin/flannel/tasks/reset.yml b/roles/network_plugin/flannel/tasks/reset.yml index b2777b4efee..2fd86e2bd9b 100644 --- a/roles/network_plugin/flannel/tasks/reset.yml +++ b/roles/network_plugin/flannel/tasks/reset.yml @@ -2,6 +2,9 @@ - name: reset | check cni network device stat: path: /sys/class/net/cni0 + get_attributes: no + get_checksum: no + get_mime: no register: cni - name: reset | remove the network device created by the flannel @@ -11,6 +14,9 @@ - name: reset | check flannel network device stat: path: /sys/class/net/flannel.1 + get_attributes: no + get_checksum: no + get_mime: no register: flannel - name: reset | remove the network device created by the flannel diff --git a/roles/network_plugin/kube-ovn/defaults/main.yml b/roles/network_plugin/kube-ovn/defaults/main.yml index 8f02a8cf164..a4e43917e0b 100644 --- a/roles/network_plugin/kube-ovn/defaults/main.yml +++ b/roles/network_plugin/kube-ovn/defaults/main.yml @@ -1,14 +1,18 @@ --- -kube_ovn_db_cpu_request: 200m -kube_ovn_db_memory_request: 300Mi -kube_ovn_db_cpu_limit: 400m -kube_ovn_db_memory_limit: 500Mi -kube_ovn_node_cpu_request: 100m -kube_ovn_node_memory_request: 300Mi -kube_ovn_node_cpu_limit: 200m -kube_ovn_node_memory_limit: 500Mi +kube_ovn_db_cpu_request: 500m +kube_ovn_db_memory_request: 200Mi +kube_ovn_db_cpu_limit: 3000m +kube_ovn_db_memory_limit: 3000Mi +kube_ovn_node_cpu_request: 200m +kube_ovn_node_memory_request: 200Mi +kube_ovn_node_cpu_limit: 1000m +kube_ovn_node_memory_limit: 800Mi +kube_ovn_controller_cpu_request: 200m +kube_ovn_controller_memory_request: 200Mi +kube_ovn_controller_cpu_limit: 1000m +kube_ovn_controller_memory_limit: 1Gi kube_ovn_pinger_cpu_request: 100m -kube_ovn_pinger_memory_request: 300Mi +kube_ovn_pinger_memory_request: 200Mi kube_ovn_pinger_cpu_limit: 200m kube_ovn_pinger_memory_limit: 400Mi diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 index 28906be4f1e..44d683de504 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -78,6 +78,12 @@ spec: subresources: status: {} additionalPrinterColumns: + - name: Provider + type: string + jsonPath: .spec.provider + - name: Vpc + type: string + jsonPath: .spec.vpc - name: Protocol type: string jsonPath: .spec.protocol @@ -135,6 +141,8 @@ spec: spec: type: object properties: + vpc: + type: string default: type: boolean protocol: @@ -169,6 +177,8 @@ spec: type: string underlayGateway: type: boolean + disableInterConnection: + type: boolean scope: Cluster names: plural: subnets @@ -219,3 +229,93 @@ spec: kind: Vlan shortNames: - vlan +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpcs.kubeovn.io +spec: + group: kubeovn.io + versions: + - additionalPrinterColumns: + - jsonPath: .status.standby + name: Standby + type: boolean + - jsonPath: .status.subnets + name: Subnets + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + namespaces: + items: + type: string + type: array + staticRoutes: + items: + properties: + policy: + type: string + cidr: + type: string + nextHopIP: + type: string + type: object + type: array + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + type: string + lastUpdateTime: + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + default: + type: boolean + defaultLogicalSwitch: + type: string + router: + type: string + standby: + type: boolean + subnets: + items: + type: string + type: array + tcpLoadBalancer: + type: string + tcpSessionLoadBalancer: + type: string + udpLoadBalancer: + type: string + udpSessionLoadBalancer: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + names: + kind: Vpc + listKind: VpcList + plural: vpcs + shortNames: + - vpc + singular: vpc + scope: Cluster diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index d405336d123..c0a20449b6f 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -66,19 +66,26 @@ spec: readinessProbe: exec: command: - - sh + - bash - /kube-ovn/kube-ovn-controller-healthcheck.sh periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - - sh + - bash - /kube-ovn/kube-ovn-controller-healthcheck.sh initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 timeoutSeconds: 45 + resources: + requests: + cpu: {{ kube_ovn_controller_cpu_request }} + memory: {{ kube_ovn_controller_memory_request }} + limits: + cpu: {{ kube_ovn_controller_cpu_limit }} + memory: {{ kube_ovn_controller_memory_limit }} nodeSelector: kubernetes.io/os: "linux" volumes: @@ -128,7 +135,7 @@ spec: image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: - - sh + - bash - /kube-ovn/start-cniserver.sh args: - --enable-mirror={{ traffic_mirror }} diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 index dfaffca6c83..1d6c643195b 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -48,11 +48,21 @@ metadata: rbac.authorization.k8s.io/system-only: "true" name: system:ovn rules: + - apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - kube-ovn - apiGroups: - "kubeovn.io" resources: - subnets - subnets/status + - vpcs + - vpcs/status - ips - vlans - networks @@ -157,8 +167,8 @@ spec: replicas: 1 strategy: rollingUpdate: - maxSurge: 0% - maxUnavailable: 100% + maxSurge: 0 + maxUnavailable: 1 type: RollingUpdate selector: matchLabels: @@ -234,14 +244,14 @@ spec: readinessProbe: exec: command: - - sh + - bash - /kube-ovn/ovn-is-leader.sh periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - - sh + - bash - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 7 @@ -350,14 +360,14 @@ spec: readinessProbe: exec: command: - - sh + - bash - /kube-ovn/ovs-healthcheck.sh periodSeconds: 5 timeoutSeconds: 45 livenessProbe: exec: command: - - sh + - bash - /kube-ovn/ovs-healthcheck.sh initialDelaySeconds: 10 periodSeconds: 5 diff --git a/roles/network_plugin/kube-router/tasks/reset.yml b/roles/network_plugin/kube-router/tasks/reset.yml index 23f79da4f5c..7b8ad2cebae 100644 --- a/roles/network_plugin/kube-router/tasks/reset.yml +++ b/roles/network_plugin/kube-router/tasks/reset.yml @@ -2,6 +2,9 @@ - name: reset | check kube-dummy-if network device stat: path: /sys/class/net/kube-dummy-if + get_attributes: no + get_checksum: no + get_mime: no register: kube_dummy_if - name: reset | remove the network device created by kube-router @@ -11,6 +14,9 @@ - name: check kube-bridge exists stat: path: /sys/class/net/kube-bridge + get_attributes: no + get_checksum: no + get_mime: no register: kube_bridge_if - name: reset | donw the network bridge create by kube-router diff --git a/roles/remove-node/pre-remove/defaults/main.yml b/roles/remove-node/pre-remove/defaults/main.yml index 1e5b849cc92..deaa8afb7f5 100644 --- a/roles/remove-node/pre-remove/defaults/main.yml +++ b/roles/remove-node/pre-remove/defaults/main.yml @@ -2,3 +2,5 @@ allow_ungraceful_removal: false drain_grace_period: 300 drain_timeout: 360s +drain_retries: 3 +drain_retry_delay_seconds: 10 diff --git a/roles/remove-node/pre-remove/tasks/main.yml b/roles/remove-node/pre-remove/tasks/main.yml index 563fa036eb8..fb895540673 100644 --- a/roles/remove-node/pre-remove/tasks/main.yml +++ b/roles/remove-node/pre-remove/tasks/main.yml @@ -1,14 +1,11 @@ --- -- name: cordon-node | Mark all nodes as unschedulable before drain # noqa 301 +- name: remove-node | List nodes command: >- - {{ bin_dir }}/kubectl cordon {{ hostvars[item]['kube_override_hostname']|default(item) }} - with_items: - - "{{ node.split(',') | default(groups['kube-node']) }}" - register: result - failed_when: result.rc != 0 and not allow_ungraceful_removal + {{ bin_dir }}/kubectl get nodes -o go-template={% raw %}'{{ range .items }}{{ .metadata.name }}{{ "\n" }}{{ end }}'{% endraw %} + register: nodes delegate_to: "{{ groups['kube-master']|first }}" + changed_when: false run_once: true - ignore_errors: yes - name: remove-node | Drain node except daemonsets resource # noqa 301 command: >- @@ -18,10 +15,13 @@ --grace-period {{ drain_grace_period }} --timeout {{ drain_timeout }} --delete-local-data {{ hostvars[item]['kube_override_hostname']|default(item) }} - with_items: - - "{{ node.split(',') | default(groups['kube-node']) }}" + loop: "{{ node.split(',') | default(groups['kube-node']) }}" + # ignore servers that are not nodes + when: hostvars[item]['kube_override_hostname']|default(item) in nodes.stdout_lines register: result failed_when: result.rc != 0 and not allow_ungraceful_removal delegate_to: "{{ groups['kube-master']|first }}" run_once: true - ignore_errors: yes + until: result.rc == 0 or allow_ungraceful_removal + retries: "{{ drain_retries }}" + delay: "{{ drain_retry_delay_seconds }}" diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 3c61bdfc567..5aa42b7e324 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -21,6 +21,8 @@ - containerd.service.d/http-proxy.conf - crio.service.d/http-proxy.conf - vault.service.d/http-proxy.conf + - k8s-certs-renew.service + - k8s-certs-renew.timer register: services_removed tags: - services @@ -68,6 +70,9 @@ - name: reset | check if crictl is present stat: path: "{{ bin_dir }}/crictl" + get_attributes: no + get_checksum: no + get_mime: no register: crictl - name: reset | stop all cri containers @@ -197,6 +202,7 @@ - filter - nat - mangle + - raw when: flush_iptables|bool tags: - iptables @@ -209,6 +215,9 @@ - name: reset | check kube-ipvs0 network device stat: path: /sys/class/net/kube-ipvs0 + get_attributes: no + get_checksum: no + get_mime: no register: kube_ipvs0 - name: reset | Remove kube-ipvs0 @@ -220,6 +229,9 @@ - name: reset | check nodelocaldns network device stat: path: /sys/class/net/nodelocaldns + get_attributes: no + get_checksum: no + get_mime: no register: nodelocaldns_device - name: reset | Remove nodelocaldns @@ -248,6 +260,8 @@ - /etc/dnsmasq.d-available - /etc/etcd.env - /etc/calico + - /etc/NetworkManager/conf.d/calico.conf + - /etc/NetworkManager/conf.d/k8s.conf - /etc/weave.env - /opt/cni - /etc/dhcp/dhclient.d/zdnsupdate.sh @@ -281,6 +295,7 @@ - "{{ bin_dir }}/weave" - "{{ bin_dir }}/crictl" - "{{ bin_dir }}/netctl" + - "{{ bin_dir }}/k8s-certs-renew.sh" - /var/lib/cni - /etc/openvswitch - /run/openvswitch diff --git a/scale.yml b/scale.yml index c6c4f1ec845..40af2858ce5 100644 --- a/scale.yml +++ b/scale.yml @@ -67,9 +67,9 @@ hosts: kube-master | first environment: "{{ proxy_disable_env }}" tags: kubeadm + roles: + - { role: kubespray-defaults } tasks: - - name: include needed vars - include_vars: roles/kubespray-defaults/defaults/main.yaml - name: Upload control plane certificates command: >- {{ bin_dir }}/kubeadm init phase diff --git a/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml b/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml index b2078b3c512..56828c47f93 100644 --- a/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml +++ b/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml @@ -84,8 +84,8 @@ images: tag: "latest" opensuse-leap-15: - filename: openSUSE-Leap-15.1-OpenStack.x86_64-0.0.4-Build6.106.qcow2 - url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.1/images/openSUSE-Leap-15.1-OpenStack.x86_64-0.0.4-Build6.106.qcow2 - checksum: sha256:e3c016a889505c5ae51dafe6eedc836a9e9546ab951fdc96f07eb35e34d12b8c + filename: openSUSE-Leap-15.2-OpenStack.x86_64-0.0.4-Build8.25.qcow2 + url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.2/images/openSUSE-Leap-15.2-OpenStack.x86_64-0.0.4-Build8.25.qcow2 + checksum: sha256:4df9cee9281d1f57d20f79dc65d76e255592b904760e73c0dd44ac753a54330f converted: true tag: "latest" diff --git a/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml b/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml index 95ce250a499..e2ef4e8fc78 100644 --- a/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml +++ b/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml @@ -10,34 +10,29 @@ url: "{{ item.value.url }}" dest: "{{ images_dir }}/{{ item.value.filename }}" checksum: "{{ item.value.checksum }}" - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" - name: Unxz compressed images command: unxz --force {{ images_dir }}/{{ item.value.filename }} - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" when: - item.value.filename.endswith('.xz') - name: Convert images which is not in qcow2 format command: qemu-img convert -O qcow2 {{ images_dir }}/{{ item.value.filename.rstrip('.xz') }} {{ images_dir }}/{{ item.key }}.qcow2 - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" when: - not (item.value.converted|bool) - name: Make sure all images are ending with qcow2 command: cp {{ images_dir }}/{{ item.value.filename.rstrip('.xz') }} {{ images_dir }}/{{ item.key }}.qcow2 - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" when: - item.value.converted|bool - name: Resize images # noqa 301 command: qemu-img resize {{ images_dir }}/{{ item.key }}.qcow2 +8G - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" # STEP 2: Include the images inside a container - name: Template default Dockerfile @@ -47,16 +42,14 @@ - name: Create docker images for each OS # noqa 301 command: docker build -t {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} --build-arg cloud_image="{{ item.key }}.qcow2" {{ images_dir }} - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" - name: docker login # noqa 301 command: docker login -u="{{ docker_user }}" -p="{{ docker_password }}" "{{ docker_host }}" - name: docker push image # noqa 301 command: docker push {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} - with_dict: - - "{{ images }}" + loop: "{{ images|dict2items }}" - name: docker logout # noqa 301 command: docker logout -u="{{ docker_user }}" "{{ docker_host }}" diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml index 5cde2e7b027..f327cd32fbd 100644 --- a/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml @@ -3,6 +3,9 @@ - name: Check if temp directory for {{ test_name }} exists stat: path: "/tmp/{{ test_name }}" + get_attributes: no + get_checksum: no + get_mime: no register: temp_dir_details - name: "Cleanup temp directory for {{ test_name }}" diff --git a/tests/files/packet_centos7-calico-ha-once-localhost.yml b/tests/files/packet_centos7-calico-ha-once-localhost.yml index 6a44c6d0133..dc08a314bdd 100644 --- a/tests/files/packet_centos7-calico-ha-once-localhost.yml +++ b/tests/files/packet_centos7-calico-ha-once-localhost.yml @@ -13,3 +13,5 @@ typha_enabled: true calico_backend: kdd typha_secure: true disable_ipv6_dns: true + +auto_renew_certificates: true diff --git a/tests/files/packet_centos7-calico-ha.yml b/tests/files/packet_centos7-calico-ha.yml index 526f128994c..7e9ba134638 100644 --- a/tests/files/packet_centos7-calico-ha.yml +++ b/tests/files/packet_centos7-calico-ha.yml @@ -12,3 +12,5 @@ dns_min_replicas: 1 typha_enabled: true calico_backend: kdd typha_secure: true + +auto_renew_certificates: true diff --git a/tests/files/packet_centos8-crio.yml b/tests/files/packet_centos8-crio.yml index 5baaaf83230..dda9223435d 100644 --- a/tests/files/packet_centos8-crio.yml +++ b/tests/files/packet_centos8-crio.yml @@ -13,3 +13,5 @@ etcd_deployment_type: host # required calico_iptables_backend: "Auto" + +auto_renew_certificates: true diff --git a/tests/files/packet_debian10-containerd.yml b/tests/files/packet_debian10-containerd.yml index e4367e61963..a6725bcb640 100644 --- a/tests/files/packet_debian10-containerd.yml +++ b/tests/files/packet_debian10-containerd.yml @@ -14,3 +14,5 @@ helm_enabled: true # https://gitlab.com/miouge/kubespray-ci/-/blob/a4fd5ed6857807f1c353cb60848aedebaf7d2c94/manifests/http-proxy.yml#L42 http_proxy: http://172.30.30.30:8888 https_proxy: http://172.30.30.30:8888 + +auto_renew_certificates: true diff --git a/tests/files/packet_debian9-macvlan.yml b/tests/files/packet_debian9-macvlan.yml index 01e4fc3fcf6..7a80202f650 100644 --- a/tests/files/packet_debian9-macvlan.yml +++ b/tests/files/packet_debian9-macvlan.yml @@ -10,3 +10,5 @@ enable_nodelocaldns: false dns_min_replicas: 1 kube_proxy_masquerade_all: true macvlan_interface: "eth0" + +auto_renew_certificates: true diff --git a/tests/files/packet_fedora33-calico.yml b/tests/files/packet_fedora33-calico.yml index 51b843d0c95..0721734ded7 100644 --- a/tests/files/packet_fedora33-calico.yml +++ b/tests/files/packet_fedora33-calico.yml @@ -10,3 +10,5 @@ kube_network_plugin: calico # Only docker package 20.10 for Fedora33 docker_version: '20.10' + +auto_renew_certificates: true diff --git a/tests/files/packet_opensuse-canal.yml b/tests/files/packet_opensuse-canal.yml index a82a07cd462..aec66b88b83 100644 --- a/tests/files/packet_opensuse-canal.yml +++ b/tests/files/packet_opensuse-canal.yml @@ -11,3 +11,5 @@ dns_min_replicas: 1 # test Ambassador ingress_ambassador_enabled: true + +auto_renew_certificates: true diff --git a/tests/files/packet_oracle7-canal-ha.yml b/tests/files/packet_oracle7-canal-ha.yml index 01ca011a5bd..993bcf3dd6b 100644 --- a/tests/files/packet_oracle7-canal-ha.yml +++ b/tests/files/packet_oracle7-canal-ha.yml @@ -9,3 +9,5 @@ kube_network_plugin: canal dynamic_kubelet_configuration: true deploy_netchecker: true dns_min_replicas: 1 + +auto_renew_certificates: true diff --git a/tests/files/packet_ubuntu16-weave-sep.yml b/tests/files/packet_ubuntu16-weave-sep.yml index db23cd64711..72073d08810 100644 --- a/tests/files/packet_ubuntu16-weave-sep.yml +++ b/tests/files/packet_ubuntu16-weave-sep.yml @@ -7,3 +7,5 @@ mode: separate kube_network_plugin: weave deploy_netchecker: true dns_min_replicas: 1 + +auto_renew_certificates: true diff --git a/tests/files/packet_ubuntu18-cilium-sep.yml b/tests/files/packet_ubuntu18-cilium-sep.yml index df6caaea905..394df8c6175 100644 --- a/tests/files/packet_ubuntu18-cilium-sep.yml +++ b/tests/files/packet_ubuntu18-cilium-sep.yml @@ -8,3 +8,5 @@ kube_network_plugin: cilium deploy_netchecker: true enable_network_policy: true dns_min_replicas: 1 + +auto_renew_certificates: true diff --git a/tests/files/packet_ubuntu20-calico-aio.yml b/tests/files/packet_ubuntu20-calico-aio.yml index 7f0b73ad7f4..2e711dab348 100644 --- a/tests/files/packet_ubuntu20-calico-aio.yml +++ b/tests/files/packet_ubuntu20-calico-aio.yml @@ -12,3 +12,5 @@ dns_min_replicas: 1 # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: False + +auto_renew_certificates: true diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index 79cd3711728..c64a27622c9 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -134,7 +134,6 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray-defaults } - - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] } - hosts: calico-rr