Skip to content

Commit

Permalink
cli: debug zip with timeout, added dump for crdb_internal.gossip_*
Browse files Browse the repository at this point in the history 
Now made command `debug zip` continue past errors with a timeout
(based on peter's timeout commit).

Also dumped information in crdb_internal.gossip_nodes and gossip_liveness
to the output file.

Fixes #23954.

Release note: None
  • Loading branch information
@windchan7
windchan7 committed Apr 4, 2018
1 parent fbdc553 commit 859107a
Show file tree
Hide file tree
Showing 4 changed files with 195 additions and 101 deletions.
2 changes: 2 additions & 0 deletions pkg/cli/cli_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2093,6 +2093,8 @@ writing ` + os.DevNull + `
debug/events
debug/liveness
debug/settings
debug/gossip/liveness
debug/gossip/nodes
debug/nodes/1/status
debug/nodes/1/gossip
debug/nodes/1/stacks
Expand Down
7 changes: 7 additions & 0 deletions pkg/cli/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,13 @@ func cmdTimeoutContext(ctx context.Context) (context.Context, func()) {
return context.WithCancel(ctx)
}

func zipTimeoutContext(ctx context.Context, timeout time.Duration) (context.Context, func()) {
if cliCtx.cmdTimeout != 0 {
return context.WithTimeout(ctx, cliCtx.cmdTimeout)
}
return context.WithTimeout(ctx, timeout)
}

// sqlCtx captures the command-line parameters of the `sql` command.
// Defaults set by InitCLIDefaults() above.
var sqlCtx = struct {
Expand Down
1 change: 1 addition & 0 deletions pkg/cli/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ func init() {
timeoutCmds := []*cobra.Command{
statusNodeCmd,
lsNodesCmd,
debugZipCmd,
// If you add something here, make sure the actual implementation
// of the command uses `cmdTimeoutContext(.)` or it will ignore
// the timeout.
Expand Down
286 changes: 185 additions & 101 deletions pkg/cli/zip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"io"
"os"
"sort"
"time"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -98,6 +99,8 @@ func runDebugZip(cmd *cobra.Command, args []string) error {
const (
base = "debug"
eventsName = base + "/events"
gossipLName = base + "/gossip/liveness"
gossipNName = base + "/gossip/nodes"
livenessName = base + "/liveness"
nodesPrefix = base + "/nodes"
schemaPrefix = base + "/schema"
Expand Down Expand Up @@ -130,157 +133,238 @@ func runDebugZip(cmd *cobra.Command, args []string) error {
z := newZipper(out)
defer z.close()

if events, err := admin.Events(ctx, &serverpb.EventsRequest{}); err != nil {
if err := z.createError(eventsName, err); err != nil {
return err
const timeout = 10 * time.Second

{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if events, err := admin.Events(ctx, &serverpb.EventsRequest{}); err != nil {
if err := z.createError(eventsName, err); err != nil {
return err
}
} else {
if err := z.createJSON(eventsName, events); err != nil {
return err
}
}
} else {
if err := z.createJSON(eventsName, events); err != nil {
return err
}

{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if liveness, err := admin.Liveness(ctx, &serverpb.LivenessRequest{}); err != nil {
if err := z.createError(livenessName, err); err != nil {
return err
}
} else {
if err := z.createJSON(livenessName, liveness); err != nil {
return err
}
}
}

if liveness, err := admin.Liveness(ctx, &serverpb.LivenessRequest{}); err != nil {
if err := z.createError(livenessName, err); err != nil {
return err
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if settings, err := admin.Settings(ctx, &serverpb.SettingsRequest{}); err != nil {
if err := z.createError(settingsName, err); err != nil {
return err
}
} else {
if err := z.createJSON(settingsName, settings); err != nil {
return err
}
}
} else {
if err := z.createJSON(livenessName, liveness); err != nil {
}

{
sqlConn, err := getPasswordAndMakeSQLClient("cockroach sql")
if err != nil {
return err
}
}
defer sqlConn.Close()

queryLiveness := "SELECT * FROM crdb_internal.gossip_liveness;"
queryNodes := "SELECT * FROM crdb_internal.gossip_nodes;"

if settings, err := admin.Settings(ctx, &serverpb.SettingsRequest{}); err != nil {
if err := z.createError(settingsName, err); err != nil {
if err := dumpGossipData(z, sqlConn, queryLiveness, gossipLName); err != nil {
return err
}
} else {
if err := z.createJSON(settingsName, settings); err != nil {
if err := dumpGossipData(z, sqlConn, queryNodes, gossipNName); err != nil {
return err
}
}

if nodes, err := status.Nodes(ctx, &serverpb.NodesRequest{}); err != nil {
if err := z.createError(nodesPrefix, err); err != nil {
return err
}
} else {
for _, node := range nodes.Nodes {
id := fmt.Sprintf("%d", node.Desc.NodeID)
prefix := fmt.Sprintf("%s/%s", nodesPrefix, id)
if err := z.createJSON(prefix+"/status", node); err != nil {
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if nodes, err := status.Nodes(ctx, &serverpb.NodesRequest{}); err != nil {
if err := z.createError(nodesPrefix, err); err != nil {
return err
}

if gossip, err := status.Gossip(ctx, &serverpb.GossipRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/gossip", err); err != nil {
} else {
for _, node := range nodes.Nodes {
id := fmt.Sprintf("%d", node.Desc.NodeID)
prefix := fmt.Sprintf("%s/%s", nodesPrefix, id)
if err := z.createJSON(prefix+"/status", node); err != nil {
return err
}
} else if err := z.createJSON(prefix+"/gossip", gossip); err != nil {
return err
}

if stacks, err := status.Stacks(ctx, &serverpb.StacksRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/stacks", err); err != nil {
return err
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if gossip, err := status.Gossip(ctx, &serverpb.GossipRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/gossip", err); err != nil {
return err
}
} else if err := z.createJSON(prefix+"/gossip", gossip); err != nil {
return err
}
}
} else if err := z.createRaw(prefix+"/stacks", stacks.Data); err != nil {
return err
}

if heap, err := status.Profile(ctx, &serverpb.ProfileRequest{
NodeId: id,
Type: serverpb.ProfileRequest_HEAP,
}); err != nil {
if err := z.createError(prefix+"/heap", err); err != nil {
return err
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if stacks, err := status.Stacks(ctx, &serverpb.StacksRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/stacks", err); err != nil {
return err
}
} else if err := z.createRaw(prefix+"/stacks", stacks.Data); err != nil {
return err
}
}
} else if err := z.createRaw(prefix+"/heap", heap.Data); err != nil {
return err
}

if logs, err := status.LogFilesList(
ctx, &serverpb.LogFilesListRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/logs", err); err != nil {
return err
}
} else {
for _, file := range logs.Files {
name := prefix + "/logs/" + file.Name
entries, err := status.LogFile(
ctx, &serverpb.LogFileRequest{NodeId: id, File: file.Name})
if err != nil {
if err := z.createError(name, err); err != nil {
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if heap, err := status.Profile(ctx, &serverpb.ProfileRequest{
NodeId: id,
Type: serverpb.ProfileRequest_HEAP,
}); err != nil {
if err := z.createError(prefix+"/heap", err); err != nil {
return err
}
continue
}
logOut, err := z.create(name)
if err != nil {
} else if err := z.createRaw(prefix+"/heap", heap.Data); err != nil {
return err
}
for _, e := range entries.Entries {
if err := e.Format(logOut); err != nil {
}

{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if logs, err := status.LogFilesList(
ctx, &serverpb.LogFilesListRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/logs", err); err != nil {
return err
}
} else {
for _, file := range logs.Files {
name := prefix + "/logs/" + file.Name
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
entries, err := status.LogFile(
ctx, &serverpb.LogFileRequest{NodeId: id, File: file.Name})
if err != nil {
if err := z.createError(name, err); err != nil {
return err
}
continue
}
logOut, err := z.create(name)
if err != nil {
return err
}
for _, e := range entries.Entries {
if err := e.Format(logOut); err != nil {
return err
}
}
}
}
}
}

if ranges, err := status.Ranges(ctx, &serverpb.RangesRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/ranges", err); err != nil {
return err
}
} else {
sort.Slice(ranges.Ranges, func(i, j int) bool {
return ranges.Ranges[i].State.Desc.RangeID <
ranges.Ranges[j].State.Desc.RangeID
})
for _, r := range ranges.Ranges {
name := fmt.Sprintf("%s/ranges/%s", prefix, r.State.Desc.RangeID)
if err := z.createJSON(name, r); err != nil {
return err
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if ranges, err := status.Ranges(ctx, &serverpb.RangesRequest{NodeId: id}); err != nil {
if err := z.createError(prefix+"/ranges", err); err != nil {
return err
}
} else {
sort.Slice(ranges.Ranges, func(i, j int) bool {
return ranges.Ranges[i].State.Desc.RangeID <
ranges.Ranges[j].State.Desc.RangeID
})
for _, r := range ranges.Ranges {
name := fmt.Sprintf("%s/ranges/%s", prefix, r.State.Desc.RangeID)
if err := z.createJSON(name, r); err != nil {
return err
}
}
}
}
}
}
}

if databases, err := admin.Databases(ctx, &serverpb.DatabasesRequest{}); err != nil {
if err := z.createError(schemaPrefix, err); err != nil {
return err
}
} else {
for _, dbName := range databases.Databases {
prefix := schemaPrefix + "/" + dbName
database, err := admin.DatabaseDetails(
ctx, &serverpb.DatabaseDetailsRequest{Database: dbName})
if err != nil {
if err := z.createError(prefix, err); err != nil {
return err
}
continue
}
if err := z.createJSON(prefix+"@details", database); err != nil {
{
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
if databases, err := admin.Databases(ctx, &serverpb.DatabasesRequest{}); err != nil {
if err := z.createError(schemaPrefix, err); err != nil {
return err
}

for _, tableName := range database.TableNames {
name := prefix + "/" + tableName
table, err := admin.TableDetails(
ctx, &serverpb.TableDetailsRequest{Database: dbName, Table: tableName})
} else {
for _, dbName := range databases.Databases {
prefix := schemaPrefix + "/" + dbName
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
database, err := admin.DatabaseDetails(
ctx, &serverpb.DatabaseDetailsRequest{Database: dbName})
if err != nil {
if err := z.createError(name, err); err != nil {
if err := z.createError(prefix, err); err != nil {
return err
}
continue
}
if err := z.createJSON(name, table); err != nil {
if err := z.createJSON(prefix+"@details", database); err != nil {
return err
}

for _, tableName := range database.TableNames {
name := prefix + "/" + tableName
ctx, cancel := zipTimeoutContext(context.Background(), timeout)
defer cancel()
table, err := admin.TableDetails(
ctx, &serverpb.TableDetailsRequest{Database: dbName, Table: tableName})
if err != nil {
if err := z.createError(name, err); err != nil {
return err
}
continue
}
if err := z.createJSON(name, table); err != nil {
return err
}
}
}
}
}

return nil
}

func dumpGossipData(z *zipper, conn *sqlConn, query string, name string) error {
w, err := z.create(name)
if err != nil {
return err
}

if err = runQueryAndFormatResults(conn, w, makeQuery(query)); err != nil {
if err := z.createError(name, err); err != nil {
return err
}
}

return nil
}

0 comments on commit 859107a

Please sign in to comment.