clear containers (Azure#1945)

* clear-containers: add runtime to api and pass through parameters Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: add scripts Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: add example Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: fix variables Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: add docs Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: update install script Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: fix script Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: update example Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: update features docs Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: make test linters happy Signed-off-by: Jess Frazelle <[email protected]> * setKubeletOpts to work better with kubeconfig Signed-off-by: Jess Frazelle <[email protected]> * whitespace cruft * more whitespace fun
yangl900 · Jan 17, 2018 · 8bd7c2c · 8bd7c2c
1 parent d53a2fc
commit 8bd7c2c
Show file tree

Hide file tree

Showing 18 changed files with 404 additions and 15 deletions.
diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md
@@ -33,6 +33,7 @@ Here are the valid values for the orchestrator types:
 |kubernetesImageBase|no|This specifies the base URL (everything preceding the actual image filename) of the kubernetes hyperkube image to use for cluster deployment, e.g., `k8s-gcrio.azureedge.net/`.|
 |dockerEngineVersion|no|Which version of docker-engine to use in your cluster, e.g.. "17.03.*"|
 |networkPolicy|no|Specifies the network policy tool for the cluster. Valid values are:<br>`"azure"` (default), which provides an Azure native networking experience,<br>`none` for not enforcing any network policy,<br>`calico` for Calico network policy (clusters with Linux agents only).<br>See [network policy examples](../examples/networkpolicy) for more information.|
+|containerRuntime|no|The container runtime to use as a backend. The default is `docker`. The only other option is `clear-containers`.|
 |clusterSubnet|no|The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. Default value is 10.244.0.0/16.|
 |dnsServiceIP|no|IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr`.|
 |dockerBridgeSubnet|no|The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0).|

diff --git a/docs/kubernetes/features.md b/docs/kubernetes/features.md
@@ -5,6 +5,7 @@
 |Managed Disks|Beta|`vlabs`|[kubernetes-vmas.json](../../examples/disks-managed/kubernetes-vmss.json)|[Description](#feat-managed-disks)|
 |Calico Network Policy|Alpha|`vlabs`|[kubernetes-calico.json](../../examples/networkpolicy/kubernetes-calico.json)|[Description](#feat-calico)|
 |Custom VNET|Beta|`vlabs`|[kubernetesvnet-azure-cni.json](../../examples/vnet/kubernetesvnet-azure-cni.json)|[Description](#feat-custom-vnet)|
+|Clear Containers Runtime|Alpha|`vlabs`|[kubernetes-clear-containers.json](../../examples/kubernetes-clear-containers.json)|[Description](#feat-clear-containers)|
 
 <a name="feat-kubernetes-msi"></a>
 
@@ -236,3 +237,37 @@ E.g.:
     }
 ]
 ```
+
+<a name="feat-clear-containers"></a>
+
+## Clear Containers
+
+You can designate kubernetes agents to use Intel's Clear Containers as the
+container runtime by setting:
+
+```
+      "kubernetesConfig": {
+        "containerRuntime": "clear-containers"
+      }
+```
+
+You will need to make sure your agents are using a `vmSize` that [supports
+nested
+virtualization](https://azure.microsoft.com/en-us/blog/nested-virtualization-in-azure/).
+These are the `Dv3` or `Ev3` series nodes.
+
+You will also need to attach a disk to those nodes for the device-mapper disk that clear containers will use.
+This should look like:
+
+```
+"agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D4s_v3",
+        "availabilityProfile": "AvailabilitySet",
+        "storageProfile": "ManagedDisks",
+        "diskSizesGB": [1023]
+      }
+    ],
+```
diff --git a/examples/kubernetes-clear-containers.json b/examples/kubernetes-clear-containers.json
@@ -0,0 +1,53 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorRelease": "1.8",
+      "kubernetesConfig": {
+        "networkPolicy": "azure",
+        "containerRuntime": "clear-containers",
+        "etcdVersion": "3.1.10",
+        "addons": [
+          {
+            "name": "tiller",
+            "enabled" : false
+          },
+          {
+            "name": "kubernetes-dashboard",
+            "enabled" : false
+          }
+        ]
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v2"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D4s_v3",
+        "availabilityProfile": "AvailabilitySet",
+        "storageProfile": "ManagedDisks",
+        "diskSizesGB": [1023]
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    }
+  }
+}
diff --git a/parts/k8s/artifacts/kuberneteskubelet.service b/parts/k8s/artifacts/kuberneteskubelet.service
@@ -25,6 +25,7 @@ ExecStart=/usr/bin/docker run \
   --volume=/sys:/sys:ro \
   --volume=/var/run:/var/run:rw \
   --volume=/var/lib/docker/:/var/lib/docker:rw \
+  --volume=/var/lib/containers/:/var/lib/containers:rw \
   --volume=/var/lib/kubelet/:/var/lib/kubelet:shared \
   --volume=/var/log:/var/log:rw \
   --volume=/etc/kubernetes/:/etc/kubernetes:ro \
@@ -39,7 +40,7 @@ ExecStart=/usr/bin/docker run \
         --v=2 ${KUBELET_FEATURE_GATES} \
         --non-masquerade-cidr=${KUBELET_NON_MASQUERADE_CIDR} \
         --volume-plugin-dir=/etc/kubernetes/volumeplugins \
-        $KUBELET_CONFIG \
+        $KUBELET_CONFIG $KUBELET_OPTS \
         ${KUBELET_REGISTER_NODE} ${KUBELET_REGISTER_WITH_TAINTS}
 
 [Install]

diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml
@@ -106,6 +106,7 @@ write_files:
     KUBELET_CONFIG={{GetKubeletConfigKeyVals .KubernetesConfig }}
     KUBELET_IMAGE={{WrapAsVariable "kubernetesHyperkubeSpec"}}
     DOCKER_OPTS=
+    KUBELET_OPTS=
     KUBELET_REGISTER_SCHEDULABLE=true
     KUBELET_NODE_LABELS={{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}
 {{if IsKubernetesVersionGe "1.6.0"}}
@@ -194,4 +195,4 @@ runcmd:
 - apt-mark unhold walinuxagent
 - mkdir -p /opt/azure/containers && touch /opt/azure/containers/runcmd.complete
 - echo `date`,`hostname`, endruncmd>>/opt/m 
-{{end}}
+{{end}}
diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -149,6 +149,7 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER
     KUBELET_CONFIG={{GetKubeletConfigKeyVals .MasterProfile.KubernetesConfig}}
     KUBELET_IMAGE={{WrapAsVariable "kubernetesHyperkubeSpec"}}
     DOCKER_OPTS=
+    KUBELET_OPTS=
     KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}}
 {{if IsKubernetesVersionGe "1.6.0"}}
   {{if HasLinuxAgents}}