fix(maestro): update pod readiness checks

These changes add a new check in the maestro that the MySQL pod in the bookwarehouse namespace is ready. It also updates `WaitForPodToBeReady()` to check the `Ready` condition on the Pod which is more complete than the existing check that each container in the Pod is ready. Signed-off-by: Jon Huhn <[email protected]>
openservicemesh · Nov 1, 2021 · db9613a · db9613a
1 parent 5d5e0a8
commit db9613a
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 14 deletions.
diff --git a/ci/cmd/maestro.go b/ci/cmd/maestro.go
@@ -30,6 +30,7 @@ const (
 	bookstoreV1Label   = "bookstore-v1"
 	bookstoreV2Label   = "bookstore-v2"
 	bookWarehouseLabel = "bookwarehouse"
+	mySQLLabel         = "mysql"
 )
 
 var (
@@ -39,6 +40,7 @@ var (
 	bookstoreV1Selector      = fmt.Sprintf("%s=%s", constants.AppLabel, bookstoreV1Label)
 	bookstoreV2Selector      = fmt.Sprintf("%s=%s", constants.AppLabel, bookstoreV2Label)
 	bookWarehouseSelector    = fmt.Sprintf("%s=%s", constants.AppLabel, bookWarehouseLabel)
+	mySQLSelector            = fmt.Sprintf("%s=%s", constants.AppLabel, mySQLLabel)
 
 	osmNamespace    = utils.GetEnv(maestro.OSMNamespaceEnvVar, "osm-system")
 	bookbuyerNS     = utils.GetEnv(maestro.BookbuyerNamespaceEnvVar, "bookbuyer")
@@ -59,7 +61,7 @@ var (
 )
 
 func main() {
-	log.Debug().Msgf("Looking for: %s/%s, %s/%s, %s/%s, %s/%s, %s/%s", bookBuyerLabel, bookbuyerNS, bookThiefLabel, bookthiefNS, bookstoreV1Label, bookstoreNS, bookstoreV2Label, bookstoreNS, bookWarehouseLabel, bookWarehouseNS)
+	log.Debug().Msgf("Looking for: %s/%s, %s/%s, %s/%s, %s/%s, %s/%s %s/%s", bookBuyerLabel, bookbuyerNS, bookThiefLabel, bookthiefNS, bookstoreV1Label, bookstoreNS, bookstoreV2Label, bookstoreNS, bookWarehouseLabel, bookWarehouseNS, mySQLLabel, bookWarehouseNS)
 
 	kubeClient := maestro.GetKubernetesClient()
 
@@ -159,6 +161,9 @@ func getPodNames(kubeClient kubernetes.Interface) (string, string, string, strin
 	wg.Add(1)
 	go maestro.WaitForPodToBeReady(kubeClient, maxWaitForPod(), bookWarehouseNS, bookWarehouseSelector, &wg)
 
+	wg.Add(1)
+	go maestro.WaitForPodToBeReady(kubeClient, maxWaitForPod(), bookWarehouseNS, mySQLSelector, &wg)
+
 	wg.Wait()
 
 	bookBuyerPodName, err := maestro.GetPodName(kubeClient, bookbuyerNS, bookBuyerSelector)

diff --git a/ci/cmd/maestro/kubernetes_tools.go b/ci/cmd/maestro/kubernetes_tools.go
@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"github.com/Azure/go-autorest/autorest/to"
-	mapset "github.com/deckarep/golang-set"
 	"helm.sh/helm/v3/pkg/action"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -23,10 +22,6 @@ import (
 	"github.com/openservicemesh/osm/pkg/cli"
 )
 
-// We are going to wait for the Pod certain amount of time if it is in one of these statuses
-// See: https://github.com/kubernetes/kubernetes/blob/d0183703cbe715c879cb42db375c7373b7f2b6a1/pkg/kubelet/kubelet_test.go#L1453-L1454
-var statusWorthWaitingFor = mapset.NewSet("ContainerCreating", "PodInitializing")
-
 // GetPodLogs returns pod logs.
 func GetPodLogs(kubeClient kubernetes.Interface, namespace string, podName string, containerName string, timeSince time.Duration) string {
 	sinceTime := metav1.NewTime(time.Now().Add(-timeSince))
@@ -252,16 +247,17 @@ func WaitForPodToBeReady(kubeClient kubernetes.Interface, totalWait time.Duratio
 			os.Exit(1)
 		}
 
-		for _, container := range pod.Status.ContainerStatuses {
-			if container.State.Waiting != nil && statusWorthWaitingFor.Contains(container.State.Waiting.Reason) {
-				fmt.Printf("Pod %s/%s is still initializing; Waiting %+v (%+v/%+v)\n", namespace, podName, WaitForPod, time.Since(startedWaiting), totalWait)
-				time.Sleep(WaitForPod)
+		for _, condition := range pod.Status.Conditions {
+			if condition.Type != corev1.PodReady {
 				continue
 			}
-
-			log.Info().Msgf("Pod %q is ready!", podName)
-			wg.Done()
-			return
+			if condition.Status == corev1.ConditionTrue {
+				log.Info().Msgf("Pod %q is ready!", podName)
+				wg.Done()
+				return
+			}
+			fmt.Printf("Pod %s/%s is still initializing; Waiting %+v (%+v/%+v)\n", namespace, podName, WaitForPod, time.Since(startedWaiting), totalWait)
+			time.Sleep(WaitForPod)
 		}
 	}
 }