diff --git a/pkg/cmd/roachprod/install/cluster_synced.go b/pkg/cmd/roachprod/install/cluster_synced.go index 7a4f38e31580..9da8ad01ece3 100644 --- a/pkg/cmd/roachprod/install/cluster_synced.go +++ b/pkg/cmd/roachprod/install/cluster_synced.go @@ -144,7 +144,8 @@ func (c *SyncedCluster) newSession(i int) (session, error) { if c.IsLocal() { return newLocalSession(), nil } - return newRemoteSession(c.user(i), c.host(i)) + host := c.host(i) + return newRemoteSession(c.user(i), host) } // Stop TODO(peter): document diff --git a/pkg/cmd/roachprod/install/session.go b/pkg/cmd/roachprod/install/session.go index 851be94c4462..119052235835 100644 --- a/pkg/cmd/roachprod/install/session.go +++ b/pkg/cmd/roachprod/install/session.go @@ -17,13 +17,17 @@ package install import ( "context" + "fmt" "io" + "io/ioutil" "os" "os/exec" "path/filepath" "sync" + "time" "github.com/cockroachdb/cockroach/pkg/cmd/roachprod/config" + "github.com/pkg/errors" ) type session interface { @@ -41,13 +45,18 @@ type session interface { type remoteSession struct { *exec.Cmd - cancel func() + cancel func() + logfile string // captures ssh -vvv } func newRemoteSession(user, host string) (*remoteSession, error) { + logfile := filepath.Join(os.TempDir(), fmt.Sprintf("ssh_%s_%s", host, time.Now().Format(time.RFC3339))) args := []string{ user + "@" + host, - "-q", + "-vvv", "-E", logfile, + // NB: -q suppresses -E, at least on OSX. Difficult decisions will have + // to be made if omitting -q leads to annoyance on stdout/stderr. + // "-q", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", // Send keep alives every minute to prevent connections without activity @@ -60,17 +69,26 @@ func newRemoteSession(user, host string) (*remoteSession, error) { args = append(args, sshAuthArgs()...) ctx, cancel := context.WithCancel(context.Background()) cmd := exec.CommandContext(ctx, "ssh", args...) - return &remoteSession{cmd, cancel}, nil + return &remoteSession{cmd, cancel, logfile}, nil +} + +func (s *remoteSession) errWithDebug(err error) error { + if err != nil { + debug, _ := ioutil.ReadFile(s.logfile) + err = errors.Wrapf(err, "ssh verbose log:\n%s\n%s", s.Cmd.Args, debug) + } + return err } func (s *remoteSession) CombinedOutput(cmd string) ([]byte, error) { s.Cmd.Args = append(s.Cmd.Args, cmd) - return s.Cmd.CombinedOutput() + b, err := s.Cmd.CombinedOutput() + return b, s.errWithDebug(err) } func (s *remoteSession) Run(cmd string) error { s.Cmd.Args = append(s.Cmd.Args, cmd) - return s.Cmd.Run() + return s.errWithDebug(s.Cmd.Run()) } func (s *remoteSession) SetStdin(r io.Reader) {