Enable gvisor addon in minikube (#3399)

This PR adds the code for enabling gvisor in minikube. It adds the pod
that will run when the addon is enabled, and the code for the image
which will run when this happens.

When gvisor is enabled, the pod will download runsc and the
gvisor-containerd-shim. It will replace the containerd config.toml and
restart containerd.

When gvisor is disabled, the pod will be deleted by the addon manager.
This will trigger a pre-stop hook which will revert the config.toml to
it's original state and restart containerd.

Makefile

@@ -303,6 +303,18 @@ storage-provisioner-image: out/storage-provisioner
 push-storage-provisioner-image: storage-provisioner-image
 	gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
 
+.PHONY: out/gvisor-addon
+out/gvisor-addon:
+	GOOS=linux CGO_ENABLED=0 go build -o $@ cmd/gvisor/gvisor.go
+
+.PHONY: gvisor-addon-image
+gvisor-addon-image: out/gvisor-addon
+	docker build -t $(REGISTRY)/gvisor-addon:latest -f deploy/gvisor/Dockerfile .
+
+.PHONY: push-gvisor-addon-image
+push-gvisor-addon-image: gvisor-addon-image
+	gcloud docker -- push $(REGISTRY)/gvisor-addon:latest
+
 .PHONY: release-iso
 release-iso: minikube_iso checksum
 	gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso

cmd/gvisor/gvisor.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"k8s.io/minikube/pkg/gvisor"
+)
+
+func main() {
+	if err := gvisor.Enable(); err != nil {
+		log.Print(err)
+		os.Exit(1)
+	}
+}

cmd/minikube/cmd/config/config.go

@@ -212,6 +212,12 @@ var settings = []Setting{
 		validations: []setFn{IsValidAddon},
 		callbacks:   []setFn{EnableOrDisableAddon},
 	},
+	{
+		name:        "gvisor",
+		set:         SetBool,
+		validations: []setFn{IsValidAddon, IsContainerdRuntime},
+		callbacks:   []setFn{EnableOrDisableAddon},
+	},
 	{
 		name: "hyperv-virtual-switch",
 		set:  SetString,

cmd/minikube/cmd/config/validations.go

@@ -18,15 +18,17 @@ package config
 
 import (
 	"fmt"
-	"github.com/docker/go-units"
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/constants"
 	"net"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
+
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 )
 
 func IsValidDriver(string, driver string) error {
@@ -125,3 +127,23 @@ func IsValidAddon(name string, val string) error {
 	}
 	return errors.Errorf("Cannot enable/disable invalid addon %s", name)
 }
+
+func IsContainerdRuntime(_, _ string) error {
+	config, err := config.Load()
+	if err != nil {
+		return fmt.Errorf("error getting cluster config: %v", err)
+	}
+	if config.KubernetesConfig.ContainerRuntime != constants.ContainerdRuntime {
+		return fmt.Errorf(`This addon can only be enabled with the containerd runtime backend.
+
+To enable this backend, please first stop minikube with:
+
+minikube stop
+
+and then start minikube again with the following flags:
+
+minikube start --container-runtime=containerd  --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni`)
+	}
+
+	return nil
+}

deploy/addons/gvisor/README.md

@@ -0,0 +1,71 @@
+## gVisor Addon
+[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
+
+### Starting Minikube
+gVisor depends on the containerd runtime to run in Minikube.
+When starting minikube, specify the following flags, along with any additional desired flags:
+
+```shell
+$ minikube start --container-runtime=containerd  \
+    --docker-opt containerd=/var/run/containerd/containerd.sock \
+    --network-plugin=cni
+```
+
+### Enabling gVisor
+To enable this addon, simply run:
+
+```
+$ minikube addons enable gvisor
+```
+
+Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod:
+
+```
+$ kubectl get pod gvisor -n kube-system
+NAME      READY     STATUS    RESTARTS   AGE
+gvisor    1/1       Running   0          3m
+```
+
+Once the pod has status `Running`, gVisor is enabled in Minikube. 
+
+### Running pods in gVisor
+To run a pod in gVisor, add this annotation to the Kubernetes yaml:
+
+```
+io.kubernetes.cri.untrusted-workload: "true"
+```
+
+An example Pod is shown below:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-untrusted
+  annotations:
+    io.kubernetes.cri.untrusted-workload: "true"
+spec:
+  containers:
+  - name: nginx
+    image: nginx
+```
+
+_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._
+
+### Disabling gVisor
+To disable gVisor, run:
+
+```
+$ minikube addons disable gvisor
+```
+
+Within one minute, the addon manager should pick up the change.
+Once the `gvisor` pod has status `Terminating`, or has been deleted, the gvisor addon should be disabled.
+
+```
+$ kubectl get pod gvisor -n kube-system
+NAME      READY     STATUS        RESTARTS   AGE
+gvisor    1/1       Terminating   0          5m
+```
+
+_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._

deploy/addons/gvisor/gvisor-config.toml

@@ -0,0 +1,69 @@
+root = "/var/lib/containerd"
+state = "/run/containerd"
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[plugins]
+  [plugins.cgroups]
+    no_prometheus = false
+  [plugins.cri]
+    stream_server_address = ""
+    stream_server_port = "10010"
+    enable_selinux = false
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    [plugins.cri.containerd]
+      snapshotter = "overlayfs"
+      no_pivot = true
+      [plugins.cri.containerd.default_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = ""
+        runtime_root = ""
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = "/usr/local/bin/runsc"
+        runtime_root = "/run/containerd/runsc"
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+    [plugins.cri.registry]
+      [plugins.cri.registry.mirrors]
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+  [plugins.diff-service]
+    default = ["walking"]
+  [plugins.linux]
+    shim = "gvisor-containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = true
+  [plugins.scheduler]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"

deploy/addons/gvisor/gvisor-containerd-shim.toml

@@ -0,0 +1,3 @@
+runc_shim = "/bin/containerd-shim"
+[runsc_config]
+ user-log="/tmp/runsc/user-log-%ID%.log"

deploy/addons/gvisor/gvisor-pod.yaml

@@ -0,0 +1,72 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata: 
+  name: gvisor
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/minikube-addons: gvisor
+spec: 
+  hostPID: true
+  containers: 
+    - name: gvisor
+      image: gcr.io/k8s-minikube/gvisor-addon:latest
+      securityContext:
+        privileged: true
+      volumeMounts:
+      - mountPath: /node/
+        name: node
+      - mountPath: /usr/libexec/sudo
+        name: sudo
+      - mountPath: /var/run
+        name: varrun
+      - mountPath: /usr/bin
+        name: usrbin
+      - mountPath: /usr/lib
+        name: usrlib
+      - mountPath: /bin
+        name: bin
+      - mountPath: /tmp/gvisor
+        name: gvisor
+      env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin
+        - name: SYSTEMD_IGNORE_CHROOT
+          value: "yes"
+  volumes:
+  - name: node
+    hostPath:
+      path: /
+  - name: sudo
+    hostPath:
+      path: /usr/libexec/sudo
+  - name: varrun
+    hostPath:
+      path: /var/run
+  - name: usrlib
+    hostPath:
+      path: /usr/lib
+  - name: usrbin
+    hostPath:
+      path: /usr/bin
+  - name: bin
+    hostPath:
+      path: /bin
+  - name: gvisor
+    hostPath:
+      path: /tmp/gvisor
+  restartPolicy: Never

deploy/gvisor/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:18.04
+RUN apt-get update && \
+    apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \
+    rm -rf /var/lib/apt/lists/*
+COPY out/gvisor-addon /gvisor-addon
+CMD ["/gvisor-addon"]

docs/addons.md

@@ -36,6 +36,7 @@ The currently supported addons include:
 * [Freshpod](https://github.com/GoogleCloudPlatform/freshpod)
 * [nvidia-driver-installer](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nvidia-driver-installer/minikube)
 * [nvidia-gpu-device-plugin](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/cmd/nvidia_gpu)
+* [gvisor](../deploy/addons/gvisor/README.md)
 
 If you would like to have minikube properly start/restart custom addons, place the addon(s) you wish to be launched with minikube in the `.minikube/addons` directory. Addons in this folder will be moved to the minikube VM and launched each time minikube is started/restarted.