From 7c3e766f73323084cd234ffbd1d98bded434208e Mon Sep 17 00:00:00 2001 From: Calin Cristian Andrei Date: Mon, 31 May 2021 17:56:07 +0300 Subject: [PATCH] gVisor: initial support for gVisor container runtime --- docs/_sidebar.md | 1 + docs/gvisor.md | 16 +++++++ .../containerd/templates/config.toml.j2 | 4 ++ .../gvisor/molecule/default/converge.yml | 11 +++++ .../molecule/default/files/10-mynet.conf | 17 +++++++ .../molecule/default/files/container.json | 10 ++++ .../molecule/default/files/sandbox.json | 10 ++++ .../gvisor/molecule/default/molecule.yml | 44 +++++++++++++++++ .../gvisor/molecule/default/prepare.yml | 48 +++++++++++++++++++ .../molecule/default/tests/test_default.py | 29 +++++++++++ roles/container-engine/gvisor/tasks/main.yml | 20 ++++++++ roles/container-engine/meta/main.yml | 8 ++++ roles/download/defaults/main.yml | 47 ++++++++++++++++++ .../container_runtimes/gvisor/tasks/main.yaml | 33 +++++++++++++ .../templates/runtimeclass-gvisor.yml.j2 | 6 +++ .../container_runtimes/meta/main.yml | 7 +++ .../preinstall/tasks/0020-verify-settings.yml | 6 +++ roles/kubespray-defaults/defaults/main.yaml | 4 ++ 18 files changed, 321 insertions(+) create mode 100644 docs/gvisor.md create mode 100644 roles/container-engine/gvisor/molecule/default/converge.yml create mode 100644 roles/container-engine/gvisor/molecule/default/files/10-mynet.conf create mode 100644 roles/container-engine/gvisor/molecule/default/files/container.json create mode 100644 roles/container-engine/gvisor/molecule/default/files/sandbox.json create mode 100644 roles/container-engine/gvisor/molecule/default/molecule.yml create mode 100644 roles/container-engine/gvisor/molecule/default/prepare.yml create mode 100644 roles/container-engine/gvisor/molecule/default/tests/test_default.py create mode 100644 roles/container-engine/gvisor/tasks/main.yml create mode 100644 roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml create mode 100644 roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 diff --git a/docs/_sidebar.md b/docs/_sidebar.md index e07deab4eef..d704e8b448e 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -41,6 +41,7 @@ * [Containerd](docs/containerd.md) * [CRI-O](docs/cri-o.md) * [Kata Containers](docs/kata-containers.md) + * [gVisor](docs/gvisor.md) * Advanced * [Proxy](/docs/proxy.md) * [Downloads](docs/downloads.md) diff --git a/docs/gvisor.md b/docs/gvisor.md new file mode 100644 index 00000000000..ef0a64b1081 --- /dev/null +++ b/docs/gvisor.md @@ -0,0 +1,16 @@ +# gVisor + +[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system. + +gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers. + +## Usage + +To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`. + +Containerd support: + +```yaml +container_manager: containerd +gvisor_enabled: true +``` diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index 8fca5681435..a344e4b96b6 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }} {% if kata_containers_enabled %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu] runtime_type = "io.containerd.kata-qemu.v2" +{% endif %} +{% if gvisor_enabled %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" {% endif %} [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] diff --git a/roles/container-engine/gvisor/molecule/default/converge.yml b/roles/container-engine/gvisor/molecule/default/converge.yml new file mode 100644 index 00000000000..8bf5478e870 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + vars: + gvisor_enabled: true + container_manager: containerd + roles: + - role: kubespray-defaults + - role: containerd + - role: gvisor diff --git a/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf b/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf new file mode 100644 index 00000000000..f10935b753b --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/roles/container-engine/gvisor/molecule/default/files/container.json b/roles/container-engine/gvisor/molecule/default/files/container.json new file mode 100644 index 00000000000..3490359d0f3 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1" + }, + "image": { + "image": "docker.io/library/hello-world:latest" + }, + "log_path": "gvisor1.0.log", + "linux": {} +} diff --git a/roles/container-engine/gvisor/molecule/default/files/sandbox.json b/roles/container-engine/gvisor/molecule/default/files/sandbox.json new file mode 100644 index 00000000000..a8da54d91db --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/roles/container-engine/gvisor/molecule/default/molecule.yml b/roles/container-engine/gvisor/molecule/default/molecule.yml new file mode 100644 index 00000000000..fc4ec0276ba --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -0,0 +1,44 @@ +--- +driver: + name: vagrant + provider: + name: libvirt + options: + driver: kvm +lint: | + set -e + yamllint -c ../../../.yamllint . +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + - name: centos8 + box: generic/centos8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callback_whitelist: profile_tasks + lint: + name: ansible-lint + options: + c: ../../../.ansible-lint + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra + lint: + name: flake8 diff --git a/roles/container-engine/gvisor/molecule/default/prepare.yml b/roles/container-engine/gvisor/molecule/default/prepare.yml new file mode 100644 index 00000000000..084824830cc --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/prepare.yml @@ -0,0 +1,48 @@ +--- +- name: Prepare generic + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: ../adduser + user: "{{ addusers.kube }}" + tasks: + - include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: ../network_plugin/cni + - role: crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: kube + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/roles/container-engine/gvisor/molecule/default/tests/test_default.py b/roles/container-engine/gvisor/molecule/default/tests/test_default.py new file mode 100644 index 00000000000..eb40b00cb3a --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/tests/test_default.py @@ -0,0 +1,29 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run(host): + gvisorruntime = "/usr/local/bin/runsc" + with host.sudo(): + cmd = host.command(gvisorruntime + " --version") + assert cmd.rc == 0 + assert "runsc version" in cmd.stdout + + +def test_run_pod(host): + runtime = "runsc" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/gvisor1.0.log") + + assert log_f.exists + assert b"Hello from Docker!" in log_f.content diff --git a/roles/container-engine/gvisor/tasks/main.yml b/roles/container-engine/gvisor/tasks/main.yml new file mode 100644 index 00000000000..fa5bd725e4e --- /dev/null +++ b/roles/container-engine/gvisor/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: gVisor | Download runsc binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}" + +- name: gVisor | Download containerd-shim-runsc-v1 binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}" + +- name: gVisor | Copy binaries + copy: + src: "{{ local_release_dir }}/gvisor-{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + mode: 0755 + remote_src: yes + with_items: + - runsc + - containerd-shim-runsc-v1 diff --git a/roles/container-engine/meta/main.yml b/roles/container-engine/meta/main.yml index 6218db932f4..2f6bff147e5 100644 --- a/roles/container-engine/meta/main.yml +++ b/roles/container-engine/meta/main.yml @@ -7,6 +7,14 @@ dependencies: - container-engine - kata-containers + - role: container-engine/gvisor + when: + - gvisor_enabled + - container_manager in ['docker', 'containerd'] + tags: + - container-engine + - gvisor + - role: container-engine/crun when: - crun_enabled diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1074b09b649..41d2c58d493 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}" etcd_version: v3.4.13 crun_version: 0.19 kata_containers_version: 2.1.0 +gvisor_version: 20210518.0 # gcr and kubernetes image repo define gcr_image_repo: "gcr.io" @@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}" kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz" +# gVisor only supports amd64 and uses x86_64 to in the download link +gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}" nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz" @@ -401,6 +404,22 @@ kata_containers_binary_checksums: 2.0.4: 0 2.1.0: 0 +gvisor_runsc_binary_checksums: + arm: + 20210518.0: 0 + arm64: + 20210518.0: 0 + amd64: + 20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0 + +gvisor_containerd_shim_binary_checksums: + arm: + 20210518.0: 0 + arm64: + 20210518.0: 0 + amd64: + 20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec + nerdctl_archive_checksums: arm: 0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233 @@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}" +gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}" +gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}" nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}" krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}" @@ -714,6 +735,32 @@ downloads: groups: - k8s_cluster + gvisor_runsc: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-runsc" + sha256: "{{ gvisor_runsc_binary_checksum }}" + url: "{{ gvisor_download_url }}/runsc" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + + gvisor_containerd_shim: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1" + sha256: "{{ gvisor_containerd_shim_binary_checksum }}" + url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + nerdctl: file: true enabled: "{{ nerdctl_enabled }}" diff --git a/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml b/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml new file mode 100644 index 00000000000..f12f5feae89 --- /dev/null +++ b/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml @@ -0,0 +1,33 @@ +--- +- name: gVisor | Create addon dir + file: + path: "{{ kube_config_dir}}/addons/gvisor" + owner: root + group: root + mode: 0755 + recurse: true + +- name: gVisor | Templates List + set_fact: + gvisor_templates: + - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass } + +- name: gVisort | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}" + with_items: "{{ gvisor_templates }}" + register: gvisor_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: gVisor | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir}}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}" + state: "latest" + with_items: "{{ gvisor_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 b/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 new file mode 100644 index 00000000000..92b20b4c6e2 --- /dev/null +++ b/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 @@ -0,0 +1,6 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }} +metadata: + name: gvisor +handler: runsc diff --git a/roles/kubernetes-apps/container_runtimes/meta/main.yml b/roles/kubernetes-apps/container_runtimes/meta/main.yml index 3c56e992e7d..f63d38d618e 100644 --- a/roles/kubernetes-apps/container_runtimes/meta/main.yml +++ b/roles/kubernetes-apps/container_runtimes/meta/main.yml @@ -7,6 +7,13 @@ dependencies: - kata-containers - container-runtimes + - role: kubernetes-apps/container_runtimes/gvisor + when: gvisor_enabled + tags: + - apps + - gvisor + - container-runtimes + - role: kubernetes-apps/container_runtimes/crun when: crun_enabled tags: diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index ae618243de1..0585a11165e 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -312,6 +312,12 @@ msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" when: kata_containers_enabled +- name: Stop if gvisor_enabled is enabled when container_manager is not containerd + assert: + that: container_manager == 'containerd' + msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details" + when: gvisor_enabled + - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"] diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 2bcba470f7d..960aceff355 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -237,6 +237,10 @@ container_manager: docker # When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false +# Enable gVisor as an additional container runtime +# gVisor is only supported with container_manager Docker or containerd +gvisor_enabled: false + # Enable crun as additional container runtime # When enabled, it requires container_manager=crio crun_enabled: false