connectivity: Ensure Hubble Relay is connected to all the nodes

Sometimes it takes a while for all the nodes to get connected to Hubble
Relay in the multicluster workflow [^1]. If Hubble is enabled, don't
start running connectivity tests until all the nodes are connected to
avoid flow validation failures.

[^1]: https://github.com/cilium/cilium-cli/actions/runs/9664927814/job/26660784590#step:21:1456

Fixes: #2518
Fixes: #2553

Signed-off-by: Michi Mutsuzaki <[email protected]>

connectivity/check/context.go

@@ -653,10 +653,21 @@ func (ct *ConnectivityTest) enableHubbleClient(ctx context.Context) error {
 			ct.Fail("In --flow-validation=strict mode, Hubble must be available to validate flows")
 			return fmt.Errorf("hubble is not available: %w", err)
 		}
-	} else {
-		ct.Infof("Hubble is OK, flows: %d/%d", status.NumFlows, status.MaxFlows)
+		return nil
 	}
 
+	for status.GetNumUnavailableNodes().GetValue() > 0 {
+		ct.Infof("Waiting for %d nodes to become available: %s",
+			status.GetNumUnavailableNodes().GetValue(), status.GetUnavailableNodes())
+		time.Sleep(5 * time.Second)
+		status, err = ct.hubbleClient.ServerStatus(ctx, &observer.ServerStatusRequest{})
+		if err != nil {
+			ct.Fail("Not all nodes became available to Hubble Relay: %w", err)
+			return fmt.Errorf("not all nodes became available to Hubble Relay: %w", err)
+		}
+	}
+	ct.Infof("Hubble is OK, flows: %d/%d, connected nodes: %d, unavailable nodes %d",
+		status.NumFlows, status.MaxFlows, status.GetNumConnectedNodes().GetValue(), status.GetNumUnavailableNodes().GetValue())
 	return nil
 }