From b00838fd48cb6b471e5288706c71780b5dc9f9b6 Mon Sep 17 00:00:00 2001
From: Justin SB <justinsb@google.com>
Date: Fri, 2 Aug 2019 09:19:56 -0400
Subject: [PATCH] Add --wait argument to kops validate

With this argument, kops validate will poll until the timeout expires,
waiting for readiness.  On readiness or on timer expiration, it exits
as if wait was not present.
---
 cmd/kops/validate_cluster.go | 70 +++++++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 25 deletions(-)

diff --git a/cmd/kops/validate_cluster.go b/cmd/kops/validate_cluster.go
index b353df8153ee2..9e08c108fe9c2 100644
--- a/cmd/kops/validate_cluster.go
+++ b/cmd/kops/validate_cluster.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/spf13/cobra"
@@ -46,6 +47,7 @@ func init() {
 
 type ValidateClusterOptions struct {
 	output string
+	wait   time.Duration
 }
 
 func (o *ValidateClusterOptions) InitDefaults() {
@@ -75,6 +77,7 @@ func NewCmdValidateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	}
 
 	cmd.Flags().StringVarP(&options.output, "output", "o", options.output, "Output format. One of json|yaml|table.")
+	cmd.Flags().DurationVar(&options.wait, "wait", options.wait, "If set, will wait for cluster to be ready")
 
 	return cmd
 }
@@ -128,40 +131,57 @@ func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out
 		return nil, fmt.Errorf("Cannot build kubernetes api client for %q: %v", contextName, err)
 	}
 
-	result, err := validation.ValidateCluster(cluster, list, k8sClient)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected error during validation: %v", err)
-	}
+	timeout := time.Now().Add(options.wait)
+	pollInterval := 10 * time.Second
 
-	switch options.output {
-	case OutputTable:
-		if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil {
-			return nil, err
-		}
-
-	case OutputYaml:
-		y, err := yaml.Marshal(result)
+	for {
+		result, err := validation.ValidateCluster(cluster, list, k8sClient)
 		if err != nil {
-			return nil, fmt.Errorf("unable to marshal YAML: %v", err)
-		}
-		if _, err := out.Write(y); err != nil {
-			return nil, fmt.Errorf("error writing to output: %v", err)
+			if time.Now().After(timeout) {
+				return nil, fmt.Errorf("unexpected error during validation: %v", err)
+			} else {
+				klog.Warningf("(will retry): unexpected error during validation: %v", err)
+				time.Sleep(pollInterval)
+				continue
+			}
 		}
 
-	case OutputJSON:
-		j, err := json.Marshal(result)
-		if err != nil {
-			return nil, fmt.Errorf("unable to marshal JSON: %v", err)
+		switch options.output {
+		case OutputTable:
+			if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil {
+				return nil, err
+			}
+
+		case OutputYaml:
+			y, err := yaml.Marshal(result)
+			if err != nil {
+				return nil, fmt.Errorf("unable to marshal YAML: %v", err)
+			}
+			if _, err := out.Write(y); err != nil {
+				return nil, fmt.Errorf("error writing to output: %v", err)
+			}
+
+		case OutputJSON:
+			j, err := json.Marshal(result)
+			if err != nil {
+				return nil, fmt.Errorf("unable to marshal JSON: %v", err)
+			}
+			if _, err := out.Write(j); err != nil {
+				return nil, fmt.Errorf("error writing to output: %v", err)
+			}
+
+		default:
+			return nil, fmt.Errorf("Unknown output format: %q", options.output)
 		}
-		if _, err := out.Write(j); err != nil {
-			return nil, fmt.Errorf("error writing to output: %v", err)
+
+		if options.wait == 0 || len(result.Failures) == 0 {
+			return result, nil
 		}
 
-	default:
-		return nil, fmt.Errorf("Unknown output format: %q", options.output)
+		klog.Warningf("(will retry): cluster not yet healthy")
+		time.Sleep(pollInterval)
 	}
 
-	return result, nil
 }
 
 func validateClusterOutputTable(result *validation.ValidationCluster, cluster *api.Cluster, instanceGroups []api.InstanceGroup, out io.Writer) error {