kata: introduce kata container support

Signed-off-by: Eric Ernst <[email protected]>
Azure · Jul 11, 2018 · 1d1eeb8 · 1d1eeb8
1 parent 455b4e8
commit 1d1eeb8
Show file tree

Hide file tree

Showing 9 changed files with 169 additions and 6 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -36,7 +36,7 @@ Here are the valid values for the orchestrator types:
 | apiServerConfig                 | no       | Configure various runtime configuration for apiserver. See `apiServerConfig` [below](#feat-apiserver-config)                                                                                                                                                                                                                                                                                                  |
 | cloudControllerManagerConfig    | no       | Configure various runtime configuration for cloud-controller-manager. See `cloudControllerManagerConfig` [below](#feat-cloud-controller-manager-config)                                                                                                                                                                                                                                                       |
 | clusterSubnet                   | no       | The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. Default value is 10.244.0.0/16                                                                                                                                                                                                                                                       |
-| containerRuntime                | no       | The container runtime to use as a backend. The default is `docker`. The other options are `clear-containers` and `containerd`                                                                                                                                                                                                                                                                                 |
+| containerRuntime                | no       | The container runtime to use as a backend. The default is `docker`. The other options are `clear-containers`, `kata-containers`, and `containerd`                                                                                                                                                                                                                                                             |
 | controllerManagerConfig         | no       | Configure various runtime configuration for controller-manager. See `controllerManagerConfig` [below](#feat-controller-manager-config)                                                                                                                                                                                                                                                                        |
 | customWindowsPackageURL         | no       | Configure custom windows Kubernetes release package URL for deployment on Windows                                                                                                                                                                                                                                                                                                                             |
 | dnsServiceIP                    | no       | IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr`                                                                                                                                                                                                                                                                                                                      |

diff --git a/docs/kubernetes/features.md b/docs/kubernetes/features.md
@@ -7,6 +7,7 @@
 |Cilium Network Policy|Alpha|`vlabs`|[kubernetes-cilium.json](../../examples/networkpolicy/kubernetes-cilium.json)|[Description](#feat-cilium)|
 |Custom VNET|Beta|`vlabs`|[kubernetesvnet-azure-cni.json](../../examples/vnet/kubernetesvnet-azure-cni.json)|[Description](#feat-custom-vnet)|
 |Clear Containers Runtime|Alpha|`vlabs`|[kubernetes-clear-containers.json](../../examples/kubernetes-clear-containers.json)|[Description](#feat-clear-containers)|
+|Kata Containers Runtime|Alpha|`vlabs`|[kubernetes-kata-containers.json](../../examples/kubernetes-kata-containers.json)|[Description](#feat-kata-containers)|
 |Private Cluster|Alpha|`vlabs`|[kubernetes-private-cluster.json](../../examples/kubernetes-config/kubernetes-private-cluster.json)|[Description](#feat-private-cluster)|
 |Azure Key Vault Encryption|Alpha|`vlabs`|[kubernetes-keyvault-encryption.json](../../examples/kubernetes-config/kubernetes-keyvault-encryption.json)|[Description](#feat-keyvault-encryption)|
 
@@ -306,6 +307,37 @@ This should look like:
     ],
 ```
 
+<a name="feat-kata-containers"></a>
+
+## Kata Containers
+
+You can designate kubernetes agents to use Kata Containers as the
+container runtime by setting:
+
+```
+      "kubernetesConfig": {
+        "containerRuntime": "kata-containers"
+      }
+```
+
+You will need to make sure your agents are using a `vmSize` that [supports
+nested virtualization](https://azure.microsoft.com/en-us/blog/nested-virtualization-in-azure/).
+These are the `Dv3` or `Ev3` series nodes.
+
+This should look like:
+
+```
+"agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D4s_v3",
+        "availabilityProfile": "AvailabilitySet",
+        "diskSizesGB": [1023]
+      }
+    ],
+```
+
 <a name="feat-private-cluster"></a>
 
 ## Private Cluster

diff --git a/examples/e2e-tests/kubernetes/kubernetes-config/kata-containers.json b/examples/e2e-tests/kubernetes/kubernetes-config/kata-containers.json
@@ -0,0 +1,40 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.10",
+      "kubernetesConfig": {
+        "networkPlugin": "flannel",
+        "containerRuntime": "kata-containers"
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D4s_v3",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/examples/kubernetes-kata-containers.json b/examples/kubernetes-kata-containers.json
@@ -0,0 +1,50 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.10",
+      "kubernetesConfig": {
+        "networkPlugin": "flannel",
+        "containerRuntime": "kata-containers",
+        "addons": [
+          {
+            "name": "tiller",
+            "enabled" : false
+          },
+          {
+            "name": "kubernetes-dashboard",
+            "enabled" : false
+          }
+        ]
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D4s_v3",
+        "availabilityProfile": "AvailabilitySet"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh
@@ -281,6 +281,24 @@ function configNetworkPlugin() {
     fi
 }
 
+function installKataContainersRuntime() {
+    # Add Kata Containers repository key
+    echo "Adding Kata Containers repository key..."
+    KATA_RELEASE_KEY_TMP=/tmp/kata-containers-release.key
+    KATA_URL=http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/Release.key
+    retrycmd_if_failure_no_stats 20 1 5 curl -fsSL $KATA_URL > $KATA_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT
+    retrycmd_if_failure 10 5 10 apt-key add $KATA_RELEASE_KEY_TMP || exit $ERR_APT_INSTALL_TIMEOUT
+
+    # Add Kata Container repository
+    echo "Adding Kata Containers repository..."
+    echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/release/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/kata-containers.list
+
+    # Install Kata Containers runtime
+    echo "Installing Kata Containers runtime..."
+    apt_get_update
+    apt_get_install 20 30 120 kata-runtime
+}
+
 function installClearContainersRuntime() {
 	# Add Clear Containers repository key
 	echo "Adding Clear Containers repository key..."
@@ -323,6 +341,8 @@ function setupContainerd() {
 	echo "runtime_type = 'io.containerd.runtime.v1.linux'" >> "$CRI_CONTAINERD_CONFIG"
 	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
 		echo "runtime_engine = '/usr/bin/cc-runtime'" >> "$CRI_CONTAINERD_CONFIG"
+	elif [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
+		echo "runtime_engine = '/usr/bin/kata-runtime'" >> "$CRI_CONTAINERD_CONFIG"
 	else
 		echo "runtime_engine = '/usr/local/sbin/runc'" >> "$CRI_CONTAINERD_CONFIG"
 	fi
@@ -344,13 +364,13 @@ function installContainerd() {
 	sed -i '/\[Service\]/a ExecStartPost=\/sbin\/iptables -P FORWARD ACCEPT' /etc/systemd/system/containerd.service
 
 	echo "Successfully installed cri-containerd..."
-	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
+	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
 		setupContainerd
 	fi
 }
 
 function ensureContainerd() {
-	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
+	if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] [[ "$CONTAINER_RUNTIME" == "kata-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then
 		# Enable and start cri-containerd service
 		# Make sure this is done after networking plugins are installed
 		echo "Enabling and starting cri-containerd service..."
@@ -546,6 +566,14 @@ if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then
 		installClearContainersRuntime
 	fi
 fi
+
+if [[ "$CONTAINER_RUNTIME" == "kata-containers" ]]; then
+	# Ensure we can nest virtualization
+	if grep -q vmx /proc/cpuinfo; then
+		installKataContainersRuntime
+	fi
+fi
+
 echo `date`,`hostname`, ensureContainerdStart>>/opt/m
 ensureContainerd
 

diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t
@@ -659,11 +659,12 @@
     "containerRuntime": {
       "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.ContainerRuntime}}",
       "metadata": {
-        "description": "The container runtime to use (docker|clear-containers|containerd)"
+        "description": "The container runtime to use (docker|clear-containers|kata-containers|containerd)"
       },
       "allowedValues": [
         "docker",
         "clear-containers",
+        "kata-containers",
         "containerd"
       ],
       "type": "string"

diff --git a/pkg/api/vlabs/const.go b/pkg/api/vlabs/const.go
@@ -79,7 +79,7 @@ var (
 	NetworkPolicyValues = [...]string{"", "calico", "cilium", "azure", "none"}
 
 	// ContainerRuntimeValues holds the valid values for container runtimes
-	ContainerRuntimeValues = [...]string{"", "docker", "clear-containers", "containerd"}
+	ContainerRuntimeValues = [...]string{"", "docker", "clear-containers", "kata-containers", "containerd"}
 )
 
 // Kubernetes configuration

diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go
@@ -1092,7 +1092,7 @@ func (a *Properties) validateContainerRuntime() error {
 	}
 
 	// Make sure we don't use clear containers on windows.
-	if (containerRuntime == "clear-containers" || containerRuntime == "containerd") && a.HasWindows() {
+	if (containerRuntime == "clear-containers" || containerRuntime == "kata-containers" || containerRuntime == "containerd") && a.HasWindows() {
 		return fmt.Errorf("containerRuntime %q is not supporting windows agents", containerRuntime)
 	}
 

diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go
@@ -1083,6 +1083,18 @@ func Test_Properties_ValidateContainerRuntime(t *testing.T) {
 		)
 	}
 
+	p.OrchestratorProfile.KubernetesConfig.ContainerRuntime = "kata-containers"
+	p.AgentPoolProfiles = []*AgentPoolProfile{
+		{
+			OSType: Windows,
+		},
+	}
+	if err := p.validateContainerRuntime(); err == nil {
+		t.Errorf(
+			"should error on kata-containers for windows clusters",
+		)
+	}
+
 	p.OrchestratorProfile.KubernetesConfig.ContainerRuntime = "containerd"
 	p.AgentPoolProfiles = []*AgentPoolProfile{
 		{