diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index a3cba2312f..cf2ed881e8 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -33,6 +33,7 @@ Here are the valid values for the orchestrator types: |kubernetesImageBase|no|This specifies the base URL (everything preceding the actual image filename) of the kubernetes hyperkube image to use for cluster deployment, e.g., `k8s-gcrio.azureedge.net/`.| |dockerEngineVersion|no|Which version of docker-engine to use in your cluster, e.g.. "17.03.*"| |networkPolicy|no|Specifies the network policy tool for the cluster. Valid values are:
`"azure"` (default), which provides an Azure native networking experience,
`none` for not enforcing any network policy,
`calico` for Calico network policy (clusters with Linux agents only).
See [network policy examples](../examples/networkpolicy) for more information.| +|containerRuntime|no|The container runtime to use as a backend. The default is `docker`. The only other option is `clear-containers`.| |clusterSubnet|no|The IP subnet used for allocating IP addresses for pod network interfaces. The subnet must be in the VNET address space. Default value is 10.244.0.0/16.| |dnsServiceIP|no|IP address for kube-dns to listen on. If specified must be in the range of `serviceCidr`.| |dockerBridgeSubnet|no|The specific IP and subnet used for allocating IP addresses for the docker bridge network created on the kubernetes master and agents. Default value is 172.17.0.1/16. This value is used to configure the docker daemon using the [--bip flag](https://docs.docker.com/engine/userguide/networking/default_network/custom-docker0).| diff --git a/docs/kubernetes/features.md b/docs/kubernetes/features.md index ea10b6dd9c..f25e098bf5 100644 --- a/docs/kubernetes/features.md +++ b/docs/kubernetes/features.md @@ -5,6 +5,7 @@ |Managed Disks|Beta|`vlabs`|[kubernetes-vmas.json](../../examples/disks-managed/kubernetes-vmss.json)|[Description](#feat-managed-disks)| |Calico Network Policy|Alpha|`vlabs`|[kubernetes-calico.json](../../examples/networkpolicy/kubernetes-calico.json)|[Description](#feat-calico)| |Custom VNET|Beta|`vlabs`|[kubernetesvnet-azure-cni.json](../../examples/vnet/kubernetesvnet-azure-cni.json)|[Description](#feat-custom-vnet)| +|Clear Containers Runtime|Alpha|`vlabs`|[kubernetes-clear-containers.json](../../examples/kubernetes-clear-containers.json)|[Description](#feat-clear-containers)| @@ -236,3 +237,37 @@ E.g.: } ] ``` + + + +## Clear Containers + +You can designate kubernetes agents to use Intel's Clear Containers as the +container runtime by setting: + +``` + "kubernetesConfig": { + "containerRuntime": "clear-containers" + } +``` + +You will need to make sure your agents are using a `vmSize` that [supports +nested +virtualization](https://azure.microsoft.com/en-us/blog/nested-virtualization-in-azure/). +These are the `Dv3` or `Ev3` series nodes. + +You will also need to attach a disk to those nodes for the device-mapper disk that clear containers will use. +This should look like: + +``` +"agentPoolProfiles": [ + { + "name": "agentpool1", + "count": 3, + "vmSize": "Standard_D4s_v3", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "ManagedDisks", + "diskSizesGB": [1023] + } + ], +``` diff --git a/examples/kubernetes-clear-containers.json b/examples/kubernetes-clear-containers.json new file mode 100644 index 0000000000..fb28815857 --- /dev/null +++ b/examples/kubernetes-clear-containers.json @@ -0,0 +1,53 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.8", + "kubernetesConfig": { + "networkPolicy": "azure", + "containerRuntime": "clear-containers", + "etcdVersion": "3.1.10", + "addons": [ + { + "name": "tiller", + "enabled" : false + }, + { + "name": "kubernetes-dashboard", + "enabled" : false + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool1", + "count": 3, + "vmSize": "Standard_D4s_v3", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "ManagedDisks", + "diskSizesGB": [1023] + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} diff --git a/parts/k8s/artifacts/kuberneteskubelet.service b/parts/k8s/artifacts/kuberneteskubelet.service index cf84a3136c..9c87c98d77 100644 --- a/parts/k8s/artifacts/kuberneteskubelet.service +++ b/parts/k8s/artifacts/kuberneteskubelet.service @@ -25,6 +25,7 @@ ExecStart=/usr/bin/docker run \ --volume=/sys:/sys:ro \ --volume=/var/run:/var/run:rw \ --volume=/var/lib/docker/:/var/lib/docker:rw \ + --volume=/var/lib/containers/:/var/lib/containers:rw \ --volume=/var/lib/kubelet/:/var/lib/kubelet:shared \ --volume=/var/log:/var/log:rw \ --volume=/etc/kubernetes/:/etc/kubernetes:ro \ @@ -39,7 +40,7 @@ ExecStart=/usr/bin/docker run \ --v=2 ${KUBELET_FEATURE_GATES} \ --non-masquerade-cidr=${KUBELET_NON_MASQUERADE_CIDR} \ --volume-plugin-dir=/etc/kubernetes/volumeplugins \ - $KUBELET_CONFIG \ + $KUBELET_CONFIG $KUBELET_OPTS \ ${KUBELET_REGISTER_NODE} ${KUBELET_REGISTER_WITH_TAINTS} [Install] diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 6bf37aab6d..78a3afb2bf 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -106,6 +106,7 @@ write_files: KUBELET_CONFIG={{GetKubeletConfigKeyVals .KubernetesConfig }} KUBELET_IMAGE={{WrapAsVariable "kubernetesHyperkubeSpec"}} DOCKER_OPTS= + KUBELET_OPTS= KUBELET_REGISTER_SCHEDULABLE=true KUBELET_NODE_LABELS={{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}} {{if IsKubernetesVersionGe "1.6.0"}} @@ -194,4 +195,4 @@ runcmd: - apt-mark unhold walinuxagent - mkdir -p /opt/azure/containers && touch /opt/azure/containers/runcmd.complete - echo `date`,`hostname`, endruncmd>>/opt/m -{{end}} +{{end}} \ No newline at end of file diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index fe139c44ba..a179c9ac1a 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -149,6 +149,7 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER KUBELET_CONFIG={{GetKubeletConfigKeyVals .MasterProfile.KubernetesConfig}} KUBELET_IMAGE={{WrapAsVariable "kubernetesHyperkubeSpec"}} DOCKER_OPTS= + KUBELET_OPTS= KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}} {{if IsKubernetesVersionGe "1.6.0"}} {{if HasLinuxAgents}} diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 9a3ad9e9f8..d365a2d9e2 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -220,6 +220,10 @@ function setNetworkPlugin () { sed -i "s/^KUBELET_NETWORK_PLUGIN=.*/KUBELET_NETWORK_PLUGIN=${1}/" /etc/default/kubelet } +function setKubeletOpts () { + sed -i "s#^KUBELET_OPTS=.*#KUBELET_OPTS=${1}#" /etc/default/kubelet +} + function setDockerOpts () { sed -i "s#^DOCKER_OPTS=.*#DOCKER_OPTS=${1}#" /etc/default/kubelet } @@ -272,6 +276,193 @@ function configNetworkPolicy() { fi } +# Install the Clear Containers runtime +function installClearContainersRuntime() { + # Add Clear Containers repository key + echo "Adding Clear Containers repository key..." + curl -sSL "https://download.opensuse.org/repositories/home:clearcontainers:clear-containers-3/xUbuntu_16.04/Release.key" | apt-key add - + + # Add Clear Container repository + echo "Adding Clear Containers repository..." + echo 'deb http://download.opensuse.org/repositories/home:/clearcontainers:/clear-containers-3/xUbuntu_16.04/ /' > /etc/apt/sources.list.d/cc-runtime.list + + # Install Clear Containers runtime + echo "Installing Clear Containers runtime..." + apt-get update + apt-get install --no-install-recommends -y \ + cc-runtime + + # Install thin tools for devicemapper configuration + echo "Installing thin tools to provision devicemapper..." + apt-get install --no-install-recommends -y \ + lvm2 \ + thin-provisioning-tools + + # Load systemd changes + echo "Loading changes to systemd service files..." + systemctl daemon-reload + + # Enable and start Clear Containers proxy service + echo "Enabling and starting Clear Containers proxy service..." + systemctl enable cc-proxy + systemctl start cc-proxy + + # CRIO has only been tested with the azure plugin + configAzureNetworkPolicy + setKubeletOpts " --container-runtime=remote --container-runtime-endpoint=/var/run/crio.sock" + setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro" +} + +# Install Go from source +function installGo() { + export GO_SRC=/usr/local/go + export GOPATH="${HOME}/.go" + + # Remove any old version of Go + if [[ -d "$GO_SRC" ]]; then + rm -rf "$GO_SRC" + fi + + # Remove any old GOPATH + if [[ -d "$GOPATH" ]]; then + rm -rf "$GOPATH" + fi + + # Get the latest Go version + GO_VERSION=$(curl -sSL "https://golang.org/VERSION?m=text") + + echo "Installing Go version $GO_VERSION..." + + # subshell + ( + curl -sSL "https://storage.googleapis.com/golang/${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -v -C /usr/local -xz + ) + + # Set GOPATH and update PATH + echo "Setting GOPATH and updating PATH" + export PATH="${GO_SRC}/bin:${PATH}:${GOPATH}/bin" +} + +# Build and install runc +function buildRunc() { + # Clone the runc source + echo "Cloning the runc source..." + mkdir -p "${GOPATH}/src/github.com/opencontainers" + ( + cd "${GOPATH}/src/github.com/opencontainers" + git clone "https://github.com/opencontainers/runc.git" + cd runc + git reset --hard v1.0.0-rc4 + make BUILDTAGS="seccomp apparmor" + make install + ) + + echo "Successfully built and installed runc..." +} + +# Build and install CRI-O +function buildCRIO() { + # Add CRI-O repositories + echo "Adding repositories required for cri-o..." + add-apt-repository -y ppa:projectatomic/ppa + add-apt-repository -y ppa:alexlarsson/flatpak + apt-get update + + # Install CRI-O dependencies + echo "Installing dependencies for CRI-O..." + apt-get install --no-install-recommends -y \ + btrfs-tools \ + gcc \ + git \ + libapparmor-dev \ + libassuan-dev \ + libc6-dev \ + libdevmapper-dev \ + libglib2.0-dev \ + libgpg-error-dev \ + libgpgme11-dev \ + libostree-dev \ + libseccomp-dev \ + libselinux1-dev \ + make \ + pkg-config \ + skopeo-containers + + installGo; + + # Install md2man + go get github.com/cpuguy83/go-md2man + + # Fix for templates dependency + ( + go get -u github.com/docker/docker/daemon/logger/templates + cd "${GOPATH}/src/github.com/docker/docker" + mkdir -p utils + cp -r daemon/logger/templates utils/ + ) + + buildRunc; + + # Clone the CRI-O source + echo "Cloning the CRI-O source..." + mkdir -p "${GOPATH}/src/github.com/kubernetes-incubator" + ( + cd "${GOPATH}/src/github.com/kubernetes-incubator" + git clone "https://github.com/kubernetes-incubator/cri-o.git" + cd cri-o + git reset --hard v1.0.0 + make BUILDTAGS="seccomp apparmor" + make install + make install.config + make install.systemd + ) + + echo "Successfully built and installed CRI-O..." + + # Cleanup the temporary directory + rm -vrf "$tmpd" + + # Cleanup the Go install + rm -vrf "$GO_SRC" "$GOPATH" + + setupCRIO; +} + +# Setup CRI-O +function setupCRIO() { + # Configure CRI-O + echo "Configuring CRI-O..." + + # Configure crio systemd service file + SYSTEMD_CRI_O_SERVICE_FILE="/usr/local/lib/systemd/system/crio.service" + sed -i 's#ExecStart=/usr/local/bin/crio#ExecStart=/usr/local/bin/crio -log-level debug#' "$SYSTEMD_CRI_O_SERVICE_FILE" + + # Configure /etc/crio/crio.conf + CRI_O_CONFIG="/etc/crio/crio.conf" + sed -i 's#storage_driver = ""#storage_driver = "devicemapper"#' "$CRI_O_CONFIG" + sed -i 's#storage_option = \[#storage_option = \["dm.directlvm_device=/dev/sdc", "dm.thinp_percent=95", "dm.thinp_metapercent=1", "dm.thinp_autoextend_threshold=80", "dm.thinp_autoextend_percent=20", "dm.directlvm_device_force=true"#' "$CRI_O_CONFIG" + sed -i 's#runtime = "/usr/bin/runc"#runtime = "/usr/local/sbin/runc"#' "$CRI_O_CONFIG" + sed -i 's#runtime_untrusted_workload = ""#runtime_untrusted_workload = "/usr/bin/cc-runtime"#' "$CRI_O_CONFIG" + sed -i 's#default_workload_trust = "trusted"#default_workload_trust = "untrusted"#' "$CRI_O_CONFIG" + + # Load systemd changes + echo "Loading changes to systemd service files..." + systemctl daemon-reload +} + +function ensureCRIO() { + if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then + # Make sure we can nest virtualization + if grep -q vmx /proc/cpuinfo; then + # Enable and start cri-o service + # Make sure this is done after networking plugins are installed + echo "Enabling and starting cri-o service..." + systemctl enable crio crio-shutdown + systemctl start crio + fi + fi +} + function systemctlEnableAndCheck() { systemctl enable $1 systemctl is-enabled $1 @@ -453,28 +644,39 @@ users: set -x } +ensureRunCommandCompleted +echo `date`,`hostname`, RunCmdCompleted>>/opt/m + +if [[ $OS == $UBUNTU_OS_NAME ]]; then + # make sure walinuxagent doesn't get updated in the middle of running this script + apt-mark hold walinuxagent +fi + # master and node echo `date`,`hostname`, EnsureDockerStart>>/opt/m ensureDocker echo `date`,`hostname`, configNetworkPolicyStart>>/opt/m configNetworkPolicy -echo `date`,`hostname`, setMaxPodsStart>>/opt/m +if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then + # Ensure we can nest virtualization + if grep -q vmx /proc/cpuinfo; then + echo `date`,`hostname`, installClearContainersRuntimeStart>>/opt/m + installClearContainersRuntime + echo `date`,`hostname`, buildCRIOStart>>/opt/m + buildCRIO + fi +fi +echo `date`,`hostname`, setMaxPodsStart>>/opt/m setMaxPods ${MAX_PODS} +echo `date`,`hostname`, ensureCRIOStart>>/opt/m +ensureCRIO echo `date`,`hostname`, ensureKubeletStart>>/opt/m ensureKubelet echo `date`,`hostname`, extractKubctlStart>>/opt/m extractKubectl echo `date`,`hostname`, ensureJournalStart>>/opt/m ensureJournal -echo `date`,`hostname`, ensureJournalDone>>/opt/m - -ensureRunCommandCompleted -echo `date`,`hostname`, RunCmdCompleted>>/opt/m - -if [[ $OS == $UBUNTU_OS_NAME ]]; then - # make sure walinuxagent doesn't get updated in the middle of running this script - apt-mark hold walinuxagent -fi +echo `date`,`hostname`, ensureJournalDone>>/opt/m # master only if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then @@ -504,4 +706,4 @@ fi echo `date`,`hostname`, endscript>>/opt/m -mkdir -p /opt/azure/containers && touch /opt/azure/containers/provision.complete \ No newline at end of file +mkdir -p /opt/azure/containers && touch /opt/azure/containers/provision.complete diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index bed18bb84b..947dacf380 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -86,6 +86,7 @@ "kubernetesKubeDNSSpec": "[parameters('kubernetesKubeDNSSpec')]", "kubernetesDNSMasqSpec": "[parameters('kubernetesDNSMasqSpec')]", "networkPolicy": "[parameters('networkPolicy')]", + "containerRuntime": "[parameters('containerRuntime')]", "cniPluginsURL":"[parameters('cniPluginsURL')]", "vnetCniLinuxPluginsURL":"[parameters('vnetCniLinuxPluginsURL')]", "vnetCniWindowsPluginsURL":"[parameters('vnetCniWindowsPluginsURL')]", @@ -171,7 +172,7 @@ {{end}} "provisionScript": "{{GetKubernetesB64Provision}}", "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", - "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('servicePrincipalClientSecret'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' MAX_PODS=',variables('maxPods'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'))]", + "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('servicePrincipalClientSecret'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' MAX_PODS=',variables('maxPods'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',variables('containerRuntime'))]", {{if not IsHostedMaster}} "provisionScriptParametersMaster": "[concat('APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index fe693177d1..e29a09e074 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -506,6 +506,17 @@ ], "type": "string" }, + "containerRuntime": { + "defaultValue": "{{.OrchestratorProfile.KubernetesConfig.ContainerRuntime}}", + "metadata": { + "description": "The container runtime to use (docker|clear-containers)" + }, + "allowedValues": [ + "docker", + "clear-containers" + ], + "type": "string" + }, "cniPluginsURL": { "defaultValue": "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-latest.tgz", "type": "string" diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index f9a7be62f6..e780cd265b 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -44,6 +44,8 @@ const ( DefaultNetworkPolicy = NetworkPolicyNone // DefaultNetworkPolicyWindows defines the network policy to use by default for clusters with Windows agent pools DefaultNetworkPolicyWindows = NetworkPolicyNone + // DefaultContainerRuntime is docker + DefaultContainerRuntime = "docker" // DefaultKubernetesNodeStatusUpdateFrequency is 10s, see --node-status-update-frequency at https://kubernetes.io/docs/admin/kubelet/ DefaultKubernetesNodeStatusUpdateFrequency = "10s" // DefaultKubernetesHardEvictionThreshold is memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%, see --eviction-hard at https://kubernetes.io/docs/admin/kubelet/ diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 596590d6ad..cd9c640778 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -315,6 +315,9 @@ func setOrchestratorDefaults(cs *api.ContainerService) { o.KubernetesConfig.NetworkPolicy = DefaultNetworkPolicy } } + if o.KubernetesConfig.ContainerRuntime == "" { + o.KubernetesConfig.ContainerRuntime = DefaultContainerRuntime + } if o.KubernetesConfig.ClusterSubnet == "" { if o.IsAzureCNI() { // When VNET integration is enabled, all masters, agents and pods share the same large subnet. diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 3c42bd8bb0..a5a2945646 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -612,6 +612,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s } addValue(parametersMap, "dockerBridgeCidr", properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet) addValue(parametersMap, "networkPolicy", properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy) + addValue(parametersMap, "containerRuntime", properties.OrchestratorProfile.KubernetesConfig.ContainerRuntime) addValue(parametersMap, "cniPluginsURL", cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL) addValue(parametersMap, "vnetCniLinuxPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL) addValue(parametersMap, "vnetCniWindowsPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL) diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 3f54f8c729..025a4f22f8 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -593,6 +593,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.DNSServiceIP = vlabs.DNSServiceIP api.ServiceCIDR = vlabs.ServiceCidr api.NetworkPolicy = vlabs.NetworkPolicy + api.ContainerRuntime = vlabs.ContainerRuntime api.MaxPods = vlabs.MaxPods api.DockerBridgeSubnet = vlabs.DockerBridgeSubnet api.CloudProviderBackoff = vlabs.CloudProviderBackoff diff --git a/pkg/api/types.go b/pkg/api/types.go index e2193508cb..4a137a55da 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -223,6 +223,7 @@ type KubernetesConfig struct { KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` ClusterSubnet string `json:"clusterSubnet,omitempty"` NetworkPolicy string `json:"networkPolicy,omitempty"` + ContainerRuntime string `json:"containerRuntime,omitempty"` MaxPods int `json:"maxPods,omitempty"` DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` DNSServiceIP string `json:"dnsServiceIP,omitempty"` diff --git a/pkg/api/vlabs/const.go b/pkg/api/vlabs/const.go index aa3cbb3bc8..7a0ad88a44 100644 --- a/pkg/api/vlabs/const.go +++ b/pkg/api/vlabs/const.go @@ -68,9 +68,12 @@ const ( ManagedDisks = "ManagedDisks" ) -// Network policy var ( + // NetworkPolicyValues holds the valid values for a network policy NetworkPolicyValues = [...]string{"", "none", "azure", "calico"} + + // ContainerRuntimeValues holds the valid values for container runtimes + ContainerRuntimeValues = [...]string{"", "docker", "clear-containers"} ) // Kubernetes configuration diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 7bad4798f8..ac1b66b9e7 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -233,6 +233,7 @@ type KubernetesConfig struct { DNSServiceIP string `json:"dnsServiceIP,omitempty"` ServiceCidr string `json:"serviceCidr,omitempty"` NetworkPolicy string `json:"networkPolicy,omitempty"` + ContainerRuntime string `json:"containerRuntime,omitempty"` MaxPods int `json:"maxPods,omitempty"` DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` CloudProviderConfig diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 5ddcd18a47..5af23ac37b 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -315,6 +315,9 @@ func (a *Properties) Validate(isUpdate bool) error { if e := a.validateNetworkPolicy(); e != nil { return e } + if e := a.validateContainerRuntime(); e != nil { + return e + } if e := a.MasterProfile.Validate(); e != nil { return e } @@ -672,6 +675,38 @@ func (a *Properties) validateNetworkPolicy() error { return nil } +func (a *Properties) validateContainerRuntime() error { + var containerRuntime string + + switch a.OrchestratorProfile.OrchestratorType { + case Kubernetes: + if a.OrchestratorProfile.KubernetesConfig != nil { + containerRuntime = a.OrchestratorProfile.KubernetesConfig.ContainerRuntime + } + default: + return nil + } + + // Check ContainerRuntime has a valid value. + valid := false + for _, runtime := range ContainerRuntimeValues { + if containerRuntime == runtime { + valid = true + break + } + } + if !valid { + return fmt.Errorf("unknown containerRuntime %q specified", containerRuntime) + } + + // Make sure we don't use clear containers on windows. + if containerRuntime == "clear-containers" && a.HasWindows() { + return fmt.Errorf("containerRuntime %q is not supporting windows agents", containerRuntime) + } + + return nil +} + func validateName(name string, label string) error { if name == "" { return fmt.Errorf("%s must be a non-empty value", label) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 0e92f5cc3f..893cc873b6 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -512,3 +512,39 @@ func getK8sDefaultProperties() *Properties { }, } } + +func Test_Properties_ValidateContainerRuntime(t *testing.T) { + p := &Properties{} + p.OrchestratorProfile = &OrchestratorProfile{} + p.OrchestratorProfile.OrchestratorType = Kubernetes + + for _, policy := range ContainerRuntimeValues { + p.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{} + p.OrchestratorProfile.KubernetesConfig.NetworkPolicy = policy + if err := p.validateContainerRuntime(); err != nil { + t.Errorf( + "should not error on containerRuntime=\"%s\"", + policy, + ) + } + } + + p.OrchestratorProfile.KubernetesConfig.ContainerRuntime = "not-existing" + if err := p.validateContainerRuntime(); err == nil { + t.Errorf( + "should error on invalid containerRuntime", + ) + } + + p.OrchestratorProfile.KubernetesConfig.ContainerRuntime = "clear-containers" + p.AgentPoolProfiles = []*AgentPoolProfile{ + { + OSType: Windows, + }, + } + if err := p.validateContainerRuntime(); err == nil { + t.Errorf( + "should error on clear-containers for windows clusters", + ) + } +}