From 190e952e2ed72fcf602232f59190cc5ad5a9ec91 Mon Sep 17 00:00:00 2001 From: Victor Morales Date: Thu, 15 Oct 2020 15:07:22 -0700 Subject: [PATCH 1/4] Enable Kata Containers for CRI-O runtime Kata Containers is an OCI runtime where containers are run inside lightweight VMs. This runtime has been enabled for containerd runtime thru the kata_containers_enabled variable. This change enables Kata Containers to CRI-O container runtime. Signed-off-by: Victor Morales --- .../container-engine/cri-o/defaults/main.yml | 23 ++++++++++++++++++- roles/container-engine/cri-o/tasks/main.yaml | 6 +++++ .../cri-o/templates/crio.conf.j2 | 22 +++++------------- roles/container-engine/cri-o/vars/debian.yml | 7 +++++- roles/container-engine/cri-o/vars/ubuntu.yml | 7 +++++- roles/kubespray-defaults/defaults/main.yaml | 2 +- 6 files changed, 47 insertions(+), 20 deletions(-) diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml index b76bf27ea0b..a2d690b3f88 100644 --- a/roles/container-engine/cri-o/defaults/main.yml +++ b/roles/container-engine/cri-o/defaults/main.yml @@ -11,7 +11,6 @@ crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}" # By default unqualified images are not allowed for security reasons crio_registries: [] -crio_runc_path: "/usr/bin/runc" crio_seccomp_profile: "" crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}" crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}" @@ -29,3 +28,25 @@ crio_kubernetes_version_matrix: "1.17": "1.17" crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.19') }}" + +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/bin/runc + type: oci + root: /run/runc + +# Kata Containers is an OCI runtime, where containers are run inside lightweight +# VMs. Kata provides additional isolation towards the host, minimizing the host attack +# surface and mitigating the consequences of containers breakout. +kata_runtimes: + # Kata Containers with the default configured VMM + - name: kata-runtime + path: /opt/kata/bin/kata-runtime + type: oci + root: /run/kata-containers + # Kata Containers with the QEMU VMM + - name: kata-qemu + path: /opt/kata/bin/kata-qemu + type: oci + root: /run/kata-containers diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml index 504330d13e9..61bea7af600 100644 --- a/roles/container-engine/cri-o/tasks/main.yaml +++ b/roles/container-engine/cri-o/tasks/main.yaml @@ -29,6 +29,12 @@ - import_tasks: "crictl.yml" +- name: Build a list of crio runtimes + set_fact: + crio_runtimes: "{{ crio_runtimes + kata_runtimes }}" + when: + - kata_containers_enabled + - name: Make sure needed folders exist in the system with_items: - /etc/crio diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index fd85e5eb46a..683cc075d8e 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -280,22 +280,12 @@ pinns_path = "" # - runtime_root (optional, string): root directory for storage of containers # state. - -[crio.runtime.runtimes.runc] -runtime_path = "{{ crio_runc_path }}" -runtime_type = "oci" -runtime_root = "/run/runc" - - -# Kata Containers is an OCI runtime, where containers are run inside lightweight -# VMs. Kata provides additional isolation towards the host, minimizing the host attack -# surface and mitigating the consequences of containers breakout. - -# Kata Containers with the default configured VMM -#[crio.runtime.runtimes.kata-runtime] - -# Kata Containers with the QEMU VMM -#[crio.runtime.runtimes.kata-qemu] +{% for runtime in crio_runtimes %} +[crio.runtime.runtimes.{{ runtime.name }}] +runtime_path = "{{ runtime.path }}" +runtime_type = "{{ runtime.type }}" +runtime_root = "{{ runtime.root }}" +{% endfor %} # Kata Containers with the Firecracker VMM #[crio.runtime.runtimes.kata-fc] diff --git a/roles/container-engine/cri-o/vars/debian.yml b/roles/container-engine/cri-o/vars/debian.yml index 0ba558e02dd..cc9ca84425e 100644 --- a/roles/container-engine/cri-o/vars/debian.yml +++ b/roles/container-engine/cri-o/vars/debian.yml @@ -4,4 +4,9 @@ crio_packages: - "cri-o" - "cri-o-runc" -crio_runc_path: /usr/sbin/runc +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/sbin/runc + type: oci + root: /run/runc diff --git a/roles/container-engine/cri-o/vars/ubuntu.yml b/roles/container-engine/cri-o/vars/ubuntu.yml index 0ba558e02dd..cc9ca84425e 100644 --- a/roles/container-engine/cri-o/vars/ubuntu.yml +++ b/roles/container-engine/cri-o/vars/ubuntu.yml @@ -4,4 +4,9 @@ crio_packages: - "cri-o" - "cri-o-runc" -crio_runc_path: /usr/sbin/runc +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/sbin/runc + type: oci + root: /run/runc diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 15171cca966..e9dbb357aae 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -221,7 +221,7 @@ kube_profiling: false container_manager: docker # Enable Kata Containers as additional container runtime -# When enabled, it requires container_manager=containerd +# When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false # Container on localhost (download images when download_localhost is true) From 57fada3ec60f8f2f1eda7969f67605641dd26487 Mon Sep 17 00:00:00 2001 From: Pasquale Toscano Date: Mon, 19 Oct 2020 18:09:51 +0200 Subject: [PATCH 2/4] Set appropiate conmon_cgroup when crio_cgroup_manager is 'cgroupfs' --- roles/container-engine/cri-o/templates/crio.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index 683cc075d8e..c637ddcc108 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -110,7 +110,11 @@ decryption_keys_path = "/etc/crio/keys/" conmon = "{{ crio_conmon }}" # Cgroup setting for conmon +{% if crio_cgroup_manager == "cgroupfs" %} +conmon_cgroup = "pod" +{% else %} conmon_cgroup = "system.slice" +{% endif %} # Environment variable list for the conmon process, used for passing necessary # environment variables to conmon or the runtime. From 65b4ff6f9c7a86363b0e99b08e21387283fc2b3e Mon Sep 17 00:00:00 2001 From: Pasquale Toscano Date: Mon, 19 Oct 2020 18:17:54 +0200 Subject: [PATCH 3/4] Set manage_ns_lifecycle=true when KataContainers is enabed --- roles/container-engine/cri-o/templates/crio.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index c637ddcc108..a456d16d48f 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -254,7 +254,11 @@ ctr_stop_timeout = 30 # manage_ns_lifecycle determines whether we pin and remove namespaces # and manage their lifecycle +{% if kata_containers_enabled %} +manage_ns_lifecycle = true +{% else %} manage_ns_lifecycle = false +{% endif %} # The directory where the state of the managed namespaces gets tracked. # Only used when manage_ns_lifecycle is true. From dba0d82659b1882dbc9d004b77bd8b24e8153070 Mon Sep 17 00:00:00 2001 From: Victor Morales Date: Thu, 22 Oct 2020 10:28:19 -0700 Subject: [PATCH 4/4] Add preinstall check for katacontainers Signed-off-by: Victor Morales --- roles/kubernetes/preinstall/tasks/0020-verify-settings.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 95086e6f928..7ee49884875 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -277,6 +277,12 @@ msg: "download_run_once support only for docker. See https://github.com/containerd/containerd/issues/4075 for details" when: download_run_once or download_force_cache +- name: Stop if kata_containers_enabled is enabled when container_manager is docker + assert: + that: container_manager != 'docker' + msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" + when: kata_containers_enabled + - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]