From 12c50e2caaf1d447239ec8f74c8b186db9ba1849 Mon Sep 17 00:00:00 2001
From: Tobias Schottdorf <tobias.schottdorf@gmail.com>
Date: Thu, 30 Apr 2020 10:35:17 +0200
Subject: [PATCH] roachtest: assert against bogus return from pgurls

See https://github.com/cockroachdb/cockroach/issues/48091.

I have no idea how this can happen despite a bit of sleuthing, but
apparently it does, so the low cost way to home in on the problem
is to sprinkle some assertions.

Release note: None
---
 pkg/cmd/roachprod/install/cluster_synced.go |  9 ++++++++-
 pkg/cmd/roachprod/main.go                   |  6 ++++++
 pkg/cmd/roachtest/cluster.go                | 12 ++++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/pkg/cmd/roachprod/install/cluster_synced.go b/pkg/cmd/roachprod/install/cluster_synced.go
index b776d1ece7d6..bfa8911e61d6 100644
--- a/pkg/cmd/roachprod/install/cluster_synced.go
+++ b/pkg/cmd/roachprod/install/cluster_synced.go
@@ -148,7 +148,14 @@ func (c *SyncedCluster) GetInternalIP(index int) (string, error) {
 			"GetInternalIP: failed to execute hostname on %s:%d:\n(stdout) %s\n(stderr) %s",
 			c.Name, index, stdout.String(), stderr.String())
 	}
-	return strings.TrimSpace(stdout.String()), nil
+	ip := strings.TrimSpace(stdout.String())
+	if ip == "" {
+		return "", errors.Errorf(
+			"empty internal IP returned, stdout:\n%s\nstderr:\n%s",
+			stdout.String(), stderr.String(),
+		)
+	}
+	return ip, nil
 }
 
 // Start TODO(peter): document
diff --git a/pkg/cmd/roachprod/main.go b/pkg/cmd/roachprod/main.go
index 8e31a9a987f9..66ea651cadf4 100644
--- a/pkg/cmd/roachprod/main.go
+++ b/pkg/cmd/roachprod/main.go
@@ -1436,9 +1436,15 @@ var pgurlCmd = &cobra.Command{
 
 		var urls []string
 		for i, ip := range ips {
+			if ip == "" {
+				return errors.Errorf("empty ip: %v", ips)
+			}
 			urls = append(urls, c.Impl.NodeURL(c, ip, c.Impl.NodePort(c, nodes[i])))
 		}
 		fmt.Println(strings.Join(urls, " "))
+		if len(urls) != len(nodes) {
+			return errors.Errorf("have nodes %v, but urls %v from ips %v", nodes, urls, ips)
+		}
 		return nil
 	}),
 }
diff --git a/pkg/cmd/roachtest/cluster.go b/pkg/cmd/roachtest/cluster.go
index adbf4c280352..ec838e519595 100644
--- a/pkg/cmd/roachtest/cluster.go
+++ b/pkg/cmd/roachtest/cluster.go
@@ -2126,8 +2126,20 @@ func (c *cluster) pgURL(ctx context.Context, node nodeListOption, external bool)
 		c.t.Fatal(errors.Wrapf(cmd.err, "failed to get pgurl for nodes: %s", nodes))
 	}
 	urls := strings.Split(strings.TrimSpace(cmd.stdout), " ")
+	if len(urls) != len(node) {
+		c.t.Fatalf(
+			"pgurl for nodes %v got urls %v from stdout:\n%s\nstderr:\n%s",
+			node, urls, cmd.stdout, cmd.stderr,
+		)
+	}
 	for i := range urls {
 		urls[i] = strings.Trim(urls[i], "'")
+		if urls[i] == "" {
+			c.t.Fatalf(
+				"pgurl for nodes %s empty: %v from\nstdout:\n%s\nstderr:\n%s",
+				urls, node, cmd.stdout, cmd.stderr,
+			)
+		}
 	}
 	return urls
 }