Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

Run kubelet outside container #2584

Merged
merged 10 commits into from
Apr 5, 2018
Merged

Run kubelet outside container #2584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
40e432e
Run kubelet outside container
Apr 3, 2018
4600ae0
restore kubelet 1.5 flags
Apr 3, 2018
c413f4d
hyperkubedir and chmod in one line
Apr 4, 2018
b90fffc
remove 1.5
Apr 4, 2018
57eaf25
install kubelet dependencies
Apr 4, 2018
b69bb99
fix syntax
Apr 4, 2018
f3de172
move set -e
Apr 4, 2018
5e3e80b
remove kubelet DOCKER_OPTS
Apr 4, 2018
1fab66b
unblock circleci
Apr 4, 2018
9826c81
increase timeout
Apr 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 0 additions & 45 deletions parts/k8s/artifacts/1.5/kuberneteskubelet.service
View file
Open in desktop

This file was deleted.

27 changes: 2 additions & 25 deletions parts/k8s/artifacts/kuberneteskubelet.service
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
[Unit]
Description=Kubelet
Requires=docker.service
After=docker.service

[Service]
Restart=always
Expand All @@ -17,28 +15,7 @@ ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
ExecStartPre=/sbin/sysctl -w net.ipv4.tcp_retries2=8
ExecStartPre=-/sbin/ebtables -t nat --list
ExecStartPre=-/sbin/iptables -t nat --list
ExecStart=/usr/bin/docker run \
--net=host \
--pid=host \
--privileged \
--rm \
--volume=/:/rootfs:ro,shared \
--volume=/dev:/dev \
--volume=/sys:/sys:ro \
--volume=/var/run:/var/run:rw \
--volume=/var/lib/cni/:/var/lib/cni:rw \
--volume=/sbin/apparmor_parser/:/sbin/apparmor_parser:rw \
--volume=/var/lib/docker/:/var/lib/docker:rw,shared \
--volume=/var/lib/containers/:/var/lib/containers:rw \
--volume=/var/lib/kubelet/:/var/lib/kubelet:rw,shared \
--volume=/var/log:/var/log:rw \
--volume=/etc/kubernetes/:/etc/kubernetes:ro \
--volume=/srv/kubernetes/:/srv/kubernetes:ro $DOCKER_OPTS \
--volume=/var/lib/waagent/ManagedIdentity-Settings:/var/lib/waagent/ManagedIdentity-Settings:ro \
--volume=/etc/kubernetes/volumeplugins:/etc/kubernetes/volumeplugins:rw \
${KUBELET_IMAGE} \
/hyperkube kubelet \
--containerized \
ExecStart=/usr/local/bin/kubelet \
--enable-server \
--node-labels="${KUBELET_NODE_LABELS}" \
--v=2 \
Expand All @@ -47,4 +24,4 @@ ExecStart=/usr/bin/docker run \
${KUBELET_REGISTER_NODE} ${KUBELET_REGISTER_WITH_TAINTS}

[Install]
WantedBy=multi-user.target
WantedBy=multi-user.target
24 changes: 13 additions & 11 deletions parts/k8s/kubernetesagentcustomdata.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,12 @@ write_files:
name: localclustercontext
current-context: localclustercontext

- path: "/etc/systemd/system/kubectl-extract.service"
- path: "/etc/systemd/system/hyperkube-extract.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=Kubectl extraction
Description=kubectl and kubelet extraction
Requires=docker.service
After=docker.service
{{if .IsCoreOS}}
Expand All @@ -92,15 +92,17 @@ write_files:
TimeoutStartSec=0
Restart=on-failure
RestartSec=5s
ExecStartPre=/bin/mkdir -p /tmp/kubectldir
ExecStartPre=/bin/mkdir -p /tmp/hyperkubedir
ExecStartPre=/usr/bin/docker pull {{WrapAsVariable "kubernetesHyperkubeSpec"}}
ExecStartPre=/usr/bin/docker run --rm -v /tmp/kubectldir:/opt/kubectldir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/kubectldir/"
ExecStartPre=/usr/bin/docker run --rm -v /tmp/hyperkubedir:/opt/hyperkubedir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/hyperkubedir/"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Besides the extraction, dependencies should also be installed. e.g. on ubuntu, kubelet depends on iptables (>= 1.4.21), kubernetes-cni (= 0.6.0), iproute2, socat, util-linux, mount, ebtables, ethtool, init-system-helpers (>= 1.18~)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @feiskyer. Just checked and all of these are already being installed except for socat. I will add a step to install them if they are missing so we are safe. I can't find a package kubernetes-cni however, do you have more info on that one?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the cni plugin, I think we are already installing it at

acs-engine/parts/k8s/kubernetesmastercustomscript.sh

Line 217 in b463cd0

retrycmd_get_tarball 60 1 $CONTAINERNETWORKING_CNI_TGZ_TMP ${CNI_PLUGINS_URL}

{{if .IsCoreOS}}
ExecStartPre=/bin/mv /tmp/kubectldir/hyperkube /opt/kubectl
ExecStart=/bin/chmod a+x /opt/kubectl
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /opt/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /opt/kubectl
ExecStart=/bin/chmod a+x /opt/kubelet /opt/kubectl
{{else}}
ExecStartPre=/bin/mv /tmp/kubectldir/hyperkube /usr/local/bin/kubectl
ExecStart=/bin/chmod a+x /usr/local/bin/kubectl
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /usr/local/bin/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /usr/local/bin/kubectl
ExecStart=/bin/chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
{{end}}

[Install]
Expand All @@ -117,7 +119,6 @@ write_files:
{{end}}
KUBELET_CONFIG={{GetKubeletConfigKeyVals .KubernetesConfig }}
KUBELET_IMAGE={{WrapAsVariable "kubernetesHyperkubeSpec"}}
DOCKER_OPTS=
KUBELET_REGISTER_SCHEDULABLE=true
KUBELET_NODE_LABELS={{GetAgentKubernetesLabels . "',variables('labelResourceGroup'),'"}}

Expand All @@ -128,7 +129,9 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER
owner: "root"
content: |
#!/bin/bash

source /opt/azure/containers/provision_source.sh
apt_get_update
retrycmd_if_failure 20 10 300 apt-get install -y iptables iproute2 socat util-linux mount ebtables ethtool init-system-helpers
{{if IsAzureCNI}}
# SNAT outbound traffic from pods to destinations outside of VNET.
iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE
Expand Down Expand Up @@ -178,7 +181,6 @@ runcmd:
# the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run
- set -x
- . /opt/azure/containers/provision_source.sh
- apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0 ] && break || sleep 1; done; echo Executed apt-get update $i times; }
- retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53
- retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443
- apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}}
Expand Down
24 changes: 13 additions & 11 deletions parts/k8s/kubernetesmastercustomdata.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,12 @@ MASTER_MANIFESTS_CONFIG_PLACEHOLDER

MASTER_ADDONS_CONFIG_PLACEHOLDER

- path: "/etc/systemd/system/kubectl-extract.service"
- path: "/etc/systemd/system/hyperkube-extract.service"
permissions: "0644"
owner: "root"
content: |
[Unit]
Description=Kubectl extraction
Description=kubectl and kubelet extraction
Requires=docker.service
After=docker.service
{{if .MasterProfile.IsCoreOS}}
Expand All @@ -133,15 +133,17 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER
TimeoutStartSec=0
Restart=on-failure
RestartSec=5s
ExecStartPre=/bin/mkdir -p /tmp/kubectldir
ExecStartPre=/bin/mkdir -p /tmp/hyperkubedir
ExecStartPre=/usr/bin/docker pull {{WrapAsVariable "kubernetesHyperkubeSpec"}}
ExecStartPre=/usr/bin/docker run --rm -v /tmp/kubectldir:/opt/kubectldir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/kubectldir/"
ExecStartPre=/usr/bin/docker run --rm -v /tmp/hyperkubedir:/opt/hyperkubedir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/hyperkubedir/"
{{if .MasterProfile.IsCoreOS}}
ExecStartPre=/bin/mv /tmp/kubectldir/hyperkube /opt/kubectl
ExecStart=/bin/chmod a+x /opt/kubectl
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /opt/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /opt/kubectl
ExecStart=/bin/chmod a+x /opt/kubelet /opt/kubectl
{{else}}
ExecStartPre=/bin/mv /tmp/kubectldir/hyperkube /usr/local/bin/kubectl
ExecStart=/bin/chmod a+x /usr/local/bin/kubectl
ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /usr/local/bin/kubelet
ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /usr/local/bin/kubectl
ExecStart=/bin/chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl
{{end}}

[Install]
Expand All @@ -158,7 +160,6 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER
{{end}}
KUBELET_CONFIG={{GetKubeletConfigKeyVals .MasterProfile.KubernetesConfig}}
KUBELET_IMAGE={{WrapAsVariable "kubernetesHyperkubeSpec"}}
DOCKER_OPTS=
KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}}
{{if IsKubernetesVersionGe "1.6.0"}}
{{if HasLinuxAgents}}
Expand All @@ -176,6 +177,9 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
owner: "root"
content: |
#!/bin/bash
source /opt/azure/containers/provision_source.sh
apt_get_update
retrycmd_if_failure 20 10 300 apt-get install -y iptables iproute2 socat util-linux mount ebtables ethtool init-system-helpers
set -e
{{if gt .MasterProfile.Count 1}}
# Azure does not support two LoadBalancers(LB) sharing the same nic and backend port.
Expand Down Expand Up @@ -368,8 +372,6 @@ runcmd:
# the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run
- set -x
- . /opt/azure/containers/provision_source.sh
- ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done }
- apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0 ] && break || sleep 1; done; echo Executed apt-get update $i times; }
- retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53
- retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443
- ensure_etcd_ready
Expand Down
19 changes: 6 additions & 13 deletions parts/k8s/kubernetesmastercustomscript.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,10 +199,6 @@ function setKubeletOpts () {
sed -i "s#^KUBELET_OPTS=.*#KUBELET_OPTS=${1}#" /etc/default/kubelet
}

function setDockerOpts () {
sed -i "s#^DOCKER_OPTS=.*#DOCKER_OPTS=${1}#" /etc/default/kubelet
}

function configAzureNetworkPolicy() {
CNI_CONFIG_DIR=/etc/cni/net.d
mkdir -p $CNI_CONFIG_DIR
Expand All @@ -226,7 +222,6 @@ function configAzureNetworkPolicy() {

function configCNINetworkPolicy() {
setNetworkPlugin cni
setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro"
}

function configNetworkPolicy() {
Expand All @@ -237,7 +232,6 @@ function configNetworkPolicy() {
else
# No policy, defaults to kubenet.
setNetworkPlugin kubenet
setDockerOpts ""
fi
}

Expand Down Expand Up @@ -266,7 +260,6 @@ function installClearContainersRuntime() {

setNetworkPlugin cni
setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
setDockerOpts " --volume=/etc/cni/:/etc/cni:ro --volume=/opt/cni/:/opt/cni:ro --volume=/var/lib/containerd:/var/lib/containerd:ro"
}

function installContainerd() {
Expand Down Expand Up @@ -356,19 +349,19 @@ function ensureDocker() {
}

function ensureKubelet() {
retrycmd_if_failure 100 1 60 docker pull $HYPERKUBE_URL
systemctlEnableAndCheck kubelet
# only start if a reboot is not required
if ! $REBOOTREQUIRED; then
systemctl restart kubelet
fi
}

function extractKubectl(){
systemctlEnableAndCheck kubectl-extract
function extractHyperkube(){
retrycmd_if_failure 100 1 60 docker pull $HYPERKUBE_URL
systemctlEnableAndCheck hyperkube-extract
# only start if a reboot is not required
if ! $REBOOTREQUIRED; then
systemctl restart kubectl-extract
systemctl restart hyperkube-extract
fi
}

Expand Down Expand Up @@ -568,10 +561,10 @@ echo `date`,`hostname`, setMaxPodsStart>>/opt/m
setMaxPods ${MAX_PODS}
echo `date`,`hostname`, ensureContainerdStart>>/opt/m
ensureContainerd
echo `date`,`hostname`, extractHyperkubeStart>>/opt/m
extractHyperkube
echo `date`,`hostname`, ensureKubeletStart>>/opt/m
ensureKubelet
echo `date`,`hostname`, extractKubctlStart>>/opt/m
extractKubectl
echo `date`,`hostname`, ensureJournalStart>>/opt/m
ensureJournal
echo `date`,`hostname`, ensureJournalDone>>/opt/m
Expand Down
4 changes: 3 additions & 1 deletion parts/k8s/kubernetesprovisionsource.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@

retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; }
retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; }
retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url -o $tarball; sleep $wait; done; }
retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url -o $tarball; sleep $wait; done; }
ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#2596 :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh yeah I just moved it out because I moved out apt_get_update so I thought might as well but we should generalize it in another PR

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I increased it to 5 minutes... Should be plenty enough

apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0 ] && break || sleep 1; done; echo Executed apt-get update $i times; }