From e6b6798b01387d96b8b642e152baca40410ce7a8 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 7 Feb 2018 09:01:53 -0800 Subject: [PATCH 1/6] more set -x --- parts/k8s/kubernetesmastercustomscript.sh | 26 +++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index ab4a747666..ba77f691a0 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -23,6 +23,7 @@ # KUBECONFIG_KEY ETCD_SERVER_CERTIFICATE ETCD_SERVER_PRIVATE_KEY ETCD_CLIENT_CERTIFICATE ETCD_CLIENT_PRIVATE_KEY # ETCD_PEER_CERTIFICATES ETCD_PEER_PRIVATE_KEYS ADMINUSER MASTER_INDEX +set -x # Capture Interesting Network Stuffs during provision packetCaptureProvision() { tcpdump -G 600 -W 1 -n -vv -w /var/log/azure/dnsdump.pcap -Z root -i eth0 udp port 53 > /dev/null 2>&1 & @@ -40,8 +41,10 @@ COREOS_OS_NAME="COREOS" KUBECTL=/usr/local/bin/kubectl DOCKER=/usr/bin/docker +set +x ETCD_PEER_CERT=$(echo ${ETCD_PEER_CERTIFICATES} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${MASTER_INDEX}+1))) ETCD_PEER_KEY=$(echo ${ETCD_PEER_PRIVATE_KEYS} | cut -d'[' -f 2 | cut -d']' -f 1 | cut -d',' -f $((${MASTER_INDEX}+1))) +set -x # CoreOS: /usr is read-only; therefore kubectl is installed at /opt/kubectl # Details on install at kubernetetsmastercustomdataforcoreos.yml @@ -90,49 +93,52 @@ if [[ ! -z "${MASTER_NODE}" ]]; then touch "${APISERVER_PRIVATE_KEY_PATH}" chmod 0600 "${APISERVER_PRIVATE_KEY_PATH}" chown root:root "${APISERVER_PRIVATE_KEY_PATH}" - echo "${APISERVER_PRIVATE_KEY}" | base64 --decode > "${APISERVER_PRIVATE_KEY_PATH}" CA_PRIVATE_KEY_PATH="/etc/kubernetes/certs/ca.key" touch "${CA_PRIVATE_KEY_PATH}" chmod 0600 "${CA_PRIVATE_KEY_PATH}" chown root:root "${CA_PRIVATE_KEY_PATH}" - echo "${CA_PRIVATE_KEY}" | base64 --decode > "${CA_PRIVATE_KEY_PATH}" ETCD_SERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdserver.key" touch "${ETCD_SERVER_PRIVATE_KEY_PATH}" chmod 0600 "${ETCD_SERVER_PRIVATE_KEY_PATH}" chown etcd:etcd "${ETCD_SERVER_PRIVATE_KEY_PATH}" - echo "${ETCD_SERVER_PRIVATE_KEY}" | base64 --decode > "${ETCD_SERVER_PRIVATE_KEY_PATH}" ETCD_CLIENT_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdclient.key" touch "${ETCD_CLIENT_PRIVATE_KEY_PATH}" chmod 0600 "${ETCD_CLIENT_PRIVATE_KEY_PATH}" chown root:root "${ETCD_CLIENT_PRIVATE_KEY_PATH}" - echo "${ETCD_CLIENT_PRIVATE_KEY}" | base64 --decode > "${ETCD_CLIENT_PRIVATE_KEY_PATH}" ETCD_PEER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.key" touch "${ETCD_PEER_PRIVATE_KEY_PATH}" chmod 0600 "${ETCD_PEER_PRIVATE_KEY_PATH}" chown etcd:etcd "${ETCD_PEER_PRIVATE_KEY_PATH}" - echo "${ETCD_PEER_KEY}" | base64 --decode > "${ETCD_PEER_PRIVATE_KEY_PATH}" ETCD_SERVER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdserver.crt" touch "${ETCD_SERVER_CERTIFICATE_PATH}" chmod 0644 "${ETCD_SERVER_CERTIFICATE_PATH}" chown root:root "${ETCD_SERVER_CERTIFICATE_PATH}" - echo "${ETCD_SERVER_CERTIFICATE}" | base64 --decode > "${ETCD_SERVER_CERTIFICATE_PATH}" ETCD_CLIENT_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdclient.crt" touch "${ETCD_CLIENT_CERTIFICATE_PATH}" chmod 0644 "${ETCD_CLIENT_CERTIFICATE_PATH}" chown root:root "${ETCD_CLIENT_CERTIFICATE_PATH}" - echo "${ETCD_CLIENT_CERTIFICATE}" | base64 --decode > "${ETCD_CLIENT_CERTIFICATE_PATH}" ETCD_PEER_CERTIFICATE_PATH="/etc/kubernetes/certs/etcdpeer${MASTER_INDEX}.crt" touch "${ETCD_PEER_CERTIFICATE_PATH}" chmod 0644 "${ETCD_PEER_CERTIFICATE_PATH}" chown root:root "${ETCD_PEER_CERTIFICATE_PATH}" + + set +x + echo "${APISERVER_PRIVATE_KEY}" | base64 --decode > "${APISERVER_PRIVATE_KEY_PATH}" + echo "${CA_PRIVATE_KEY}" | base64 --decode > "${CA_PRIVATE_KEY_PATH}" + echo "${ETCD_SERVER_PRIVATE_KEY}" | base64 --decode > "${ETCD_SERVER_PRIVATE_KEY_PATH}" + echo "${ETCD_CLIENT_PRIVATE_KEY}" | base64 --decode > "${ETCD_CLIENT_PRIVATE_KEY_PATH}" + echo "${ETCD_PEER_KEY}" | base64 --decode > "${ETCD_PEER_PRIVATE_KEY_PATH}" + echo "${ETCD_SERVER_CERTIFICATE}" | base64 --decode > "${ETCD_SERVER_CERTIFICATE_PATH}" + echo "${ETCD_CLIENT_CERTIFICATE}" | base64 --decode > "${ETCD_CLIENT_CERTIFICATE_PATH}" echo "${ETCD_PEER_CERT}" | base64 --decode > "${ETCD_PEER_CERTIFICATE_PATH}" + set -x echo `date`,`hostname`, finishedGettingEtcdCerts>>/opt/m else @@ -143,18 +149,20 @@ KUBELET_PRIVATE_KEY_PATH="/etc/kubernetes/certs/client.key" touch "${KUBELET_PRIVATE_KEY_PATH}" chmod 0600 "${KUBELET_PRIVATE_KEY_PATH}" chown root:root "${KUBELET_PRIVATE_KEY_PATH}" -echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" APISERVER_PUBLIC_KEY_PATH="/etc/kubernetes/certs/apiserver.crt" touch "${APISERVER_PUBLIC_KEY_PATH}" chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}" chown root:root "${APISERVER_PUBLIC_KEY_PATH}" -echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" AZURE_JSON_PATH="/etc/kubernetes/azure.json" touch "${AZURE_JSON_PATH}" chmod 0600 "${AZURE_JSON_PATH}" chown root:root "${AZURE_JSON_PATH}" + +set +x +echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" +echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" cat << EOF > "${AZURE_JSON_PATH}" { "cloud":"${TARGET_ENVIRONMENT}", From 69749b37f8621ad1c01dd5a3bc1a9fd1fbccd101 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 7 Feb 2018 09:03:13 -0800 Subject: [PATCH 2/6] send ps to background --- parts/k8s/kubernetesmastercustomscript.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index ba77f691a0..723a5b7eb5 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -751,4 +751,4 @@ fi echo `date`,`hostname`, endscript>>/opt/m mkdir -p /opt/azure/containers && touch /opt/azure/containers/provision.complete -ps auxfww > /opt/azure/provision-ps.log +ps auxfww > /opt/azure/provision-ps.log & From c168f9a0c92ee193d9888f55742dd0a0fe2e5b2a Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 7 Feb 2018 09:40:01 -0800 Subject: [PATCH 3/6] timestamps --- parts/k8s/kubernetesmastercustomscript.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 723a5b7eb5..e8af284219 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -89,6 +89,7 @@ if [[ ! -z "${MASTER_NODE}" ]]; then echo "etcd user exists" fi + echo `date`,`hostname`, beginGettingEtcdCerts>>/opt/m APISERVER_PRIVATE_KEY_PATH="/etc/kubernetes/certs/apiserver.key" touch "${APISERVER_PRIVATE_KEY_PATH}" chmod 0600 "${APISERVER_PRIVATE_KEY_PATH}" @@ -140,7 +141,7 @@ if [[ ! -z "${MASTER_NODE}" ]]; then echo "${ETCD_PEER_CERT}" | base64 --decode > "${ETCD_PEER_CERTIFICATE_PATH}" set -x - echo `date`,`hostname`, finishedGettingEtcdCerts>>/opt/m + echo `date`,`hostname`, endGettingEtcdCerts>>/opt/m else echo "skipping master node provision operations, this is an agent node" fi From 6f4c3012ecc6698b3fa4b247a1c46a01cb913625 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 7 Feb 2018 10:01:05 -0800 Subject: [PATCH 4/6] adding certs dependency in cloud-init --- parts/k8s/kubernetesmastercustomdata.yml | 2 ++ parts/k8s/kubernetesmastercustomscript.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 7681e15d52..d295c8439b 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -348,6 +348,8 @@ runcmd: - retrycmd_if_failure() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } - retrycmd_if_failure 120 1 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 - retrycmd_if_failure 120 1 nc -zw1 aptdocker.azureedge.net 443 +- ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done } +- ensure_etcd_ready - /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1 - apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}} - /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file={{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} --peer-key-file={{WrapAsVerbatim "variables('etcdPeerKeyFilepath')[copyIndex(variables('masterOffset'))]"}} --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',https://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster "{{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk"" --initial-cluster-state "new" | tee -a /etc/default/etcd diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index e8af284219..d74b04ef19 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -142,6 +142,7 @@ if [[ ! -z "${MASTER_NODE}" ]]; then set -x echo `date`,`hostname`, endGettingEtcdCerts>>/opt/m + mkdir -p /opt/azure/containers && touch /opt/azure/containers/certs.ready else echo "skipping master node provision operations, this is an agent node" fi From f299d8da7650f1e200006142f071f92ad5a2df2d Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 7 Feb 2018 10:07:01 -0800 Subject: [PATCH 5/6] rationalize etcd certs dep --- parts/k8s/kubernetesmastercustomdata.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index d295c8439b..ab073081d3 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -357,7 +357,7 @@ runcmd: - /bin/chown -R etcd:etcd /var/lib/etcddisk - systemctl stop etcd - systemctl daemon-reload -- retrycmd_if_failure 60 3 cat {{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} +- ensure_etcd_ready - systemctl restart etcd - MEMBER="$(sudo etcdctl member list | grep -E {{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}} | cut -d{{WrapAsVariable "singleQuote"}}:{{WrapAsVariable "singleQuote"}} -f 1)" - sudo etcdctl member update ${MEMBER} {{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}} From b2292d034e95245a0a363722ccf441aa719609da Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 7 Feb 2018 11:28:49 -0800 Subject: [PATCH 6/6] extra ensure_etcd_ready --- parts/k8s/kubernetesmastercustomdata.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index ab073081d3..dab9a6c8ff 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -349,7 +349,6 @@ runcmd: - retrycmd_if_failure 120 1 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 - retrycmd_if_failure 120 1 nc -zw1 aptdocker.azureedge.net 443 - ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done } -- ensure_etcd_ready - /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1 - apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}} - /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file={{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} --peer-key-file={{WrapAsVerbatim "variables('etcdPeerKeyFilepath')[copyIndex(variables('masterOffset'))]"}} --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',https://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster "{{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk"" --initial-cluster-state "new" | tee -a /etc/default/etcd