gocli: Add wait for cri-o service to be ready after proxy settings ap…

…plied (#1303)

* gocli: Add wait for cri-o service to be ready after proxy settings applied

There was previously a wait for the cri-o service to be ready[1] after
applying the docker proxy settings but this was not included with the
introduction of the opts package[2].

This can lead to kubeadm failing its preflight checks as cri-o is not ready[3]

[1] https://github.com/kubevirt/kubevirtci/blob/03aa94e9da974b0e37dc8f180a4f700d47020c08/cluster-provision/k8s/1.31/nodes.sh#L43
[2] #1217
[3] https://prow.ci.kubevirt.io/view/gs/kubevirt-prow/pr-logs/pull/kubevirt_kubevirt/13020/pull-kubevirt-e2e-k8s-1.31-sig-network/1843292337933914112#1:build-log.txt%3A343

Signed-off-by: Brian Carey <[email protected]>

* gocli: Add wait for crio service to node opts

Signed-off-by: Brian Carey <[email protected]>

---------

Signed-off-by: Brian Carey <[email protected]>

cluster-provision/gocli/cmd/run.go

@@ -68,6 +68,11 @@ EOT
 
 systemctl daemon-reload
 systemctl restart crio.service
+
+while [[ systemctl status crio | grep active | wc -l -eq 0 ]]
+do
+    sleep 2
+done
 EOF
 `
 	etcdDataDir         = "/var/lib/etcd"

cluster-provision/gocli/opts/node01/node01.go

@@ -48,6 +48,7 @@ func (n *node01Provisioner) Exec() error {
 		`timeout=30; interval=5; while ! hostnamectl | grep Transient; do echo "Waiting for dhclient to set the hostname from dnsmasq"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`,
 		"swapoff -a",
 		"until ip address show dev eth0 | grep global | grep inet6; do sleep 1; done",
+		`timeout=60; interval=5; while ! systemctl status crio | grep active; do echo "Waiting for cri-o service to be ready"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`,
 		kubeadmInitCmd,
 		`kubectl --kubeconfig=/etc/kubernetes/admin.conf patch deployment coredns -n kube-system -p "$(cat /provision/kubeadm-patches/add-security-context-deployment-patch.yaml)"`,
 		`kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f ` + cniManifest,

cluster-provision/gocli/opts/node01/testconfig.go

@@ -8,6 +8,7 @@ func AddExpectCalls(sshClient *kubevirtcimocks.MockSSHClient) {
 		`timeout=30; interval=5; while ! hostnamectl | grep Transient; do echo "Waiting for dhclient to set the hostname from dnsmasq"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`,
 		"swapoff -a",
 		"until ip address show dev eth0 | grep global | grep inet6; do sleep 1; done",
+		`timeout=60; interval=5; while ! systemctl status crio | grep active; do echo "Waiting for cri-o service to be ready"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`,
 		`kubeadm init --config /etc/kubernetes/kubeadm.conf -v5`,
 		`kubectl --kubeconfig=/etc/kubernetes/admin.conf patch deployment coredns -n kube-system -p "$(cat /provision/kubeadm-patches/add-security-context-deployment-patch.yaml)"`,
 		`kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f /provision/cni.yaml`,

cluster-provision/gocli/opts/nodes/nodes.go

@@ -40,6 +40,7 @@ func (n *nodesProvisioner) Exec() error {
 		"systemctl daemon-reload &&  service kubelet restart",
 		"swapoff -a",
 		"until ip address show dev eth0 | grep global | grep inet6; do sleep 1; done",
+		`timeout=60; interval=5; while ! systemctl status crio | grep active; do echo "Waiting for cri-o service to be ready"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`,
 		"kubeadm join --token abcdef.1234567890123456 " + controlPlaneIP + ":6443 --ignore-preflight-errors=all --discovery-token-unsafe-skip-ca-verification=true",
 		"mkdir -p /var/lib/rook",
 		"chcon -t container_file_t /var/lib/rook",

cluster-provision/gocli/opts/nodes/testconfig.go

@@ -10,6 +10,7 @@ func AddExpectCalls(sshClient *kubevirtcimocks.MockSSHClient) {
 		"systemctl daemon-reload &&  service kubelet restart",
 		"swapoff -a",
 		"until ip address show dev eth0 | grep global | grep inet6; do sleep 1; done",
+		`timeout=60; interval=5; while ! systemctl status crio | grep active; do echo "Waiting for cri-o service to be ready"; sleep $interval; timeout=$((timeout - interval)); [ $timeout -le 0 ] && exit 1; done`,
 		"kubeadm join --token abcdef.1234567890123456 192.168.66.101:6443 --ignore-preflight-errors=all --discovery-token-unsafe-skip-ca-verification=true",
 		"mkdir -p /var/lib/rook",
 		"chcon -t container_file_t /var/lib/rook",