Refactor pprof profile capture

command/operator_debug.go

@@ -654,71 +654,60 @@ func (c *OperatorDebugCommand) collectPprof(path, id string, client *api.Client)
 		}
 	}
 
-	bs, err = client.Agent().Trace(opts, nil)
-	if err != nil {
-		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof trace.prof, err: %v", path, err))
-	} else {
-		err := c.writeBytes(path, "trace.prof", bs)
-		if err != nil {
-			c.Ui.Error(err.Error())
-		}
-	}
+	// goroutine debug type 1 = legacy text format for human readable output
+	opts.Debug = 1
+	c.savePprofProfile(path, "goroutine", opts, client)
 
-	bs, err = client.Agent().Lookup("goroutine", opts, nil)
-	if err != nil {
-		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof goroutine.prof, err: %v", path, err))
-	} else {
-		err := c.writeBytes(path, "goroutine.prof", bs)
-		if err != nil {
-			c.Ui.Error(err.Error())
-		}
-	}
+	// goroutine debug type 2 = goroutine stacks in panic format
+	opts.Debug = 2
+	c.savePprofProfile(path, "goroutine", opts, client)
 
-	err = c.savePprofProfile(path, "heap", opts, client)
-	err = c.savePprofProfile(path, "allocs", opts, client)
-	err = c.savePprofProfile(path, "threadcreate", opts, client)
-	err = c.savePprofProfile(path, "block", opts, client)
-	err = c.savePprofProfile(path, "mutex", opts, client)
+	// Reset to pprof binary format
+	opts.Debug = 0
 
-	// Gather goroutine text output - debug type 1
-	// debug type 1 writes the legacy text format for human readable output
-	opts.Debug = 1
-	bs, err = client.Agent().Lookup("goroutine", opts, nil)
-	if err != nil {
-		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof goroutine-debug1.txt, err: %v", path, err))
-	} else {
-		err := c.writeBytes(path, "goroutine-debug1.txt", bs)
-		if err != nil {
-			c.Ui.Error(err.Error())
-		}
+	c.savePprofProfile(path, "goroutine", opts, client)    // Stack traces of all current goroutines
+	c.savePprofProfile(path, "trace", opts, client)        // A trace of execution of the current program
+	c.savePprofProfile(path, "heap", opts, client)         // A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
+	c.savePprofProfile(path, "allocs", opts, client)       // A sampling of all past memory allocations
+	c.savePprofProfile(path, "threadcreate", opts, client) // Stack traces that led to the creation of new OS threads
+
+	// This profile is disabled by default -- Requires runtime.SetBlockProfileRate to enable
+	// c.savePprofProfile(path, "block", opts, client)        // Stack traces that led to blocking on synchronization primitives
+
+	// This profile is disabled by default -- Requires runtime.SetMutexProfileFraction to enable
+	// c.savePprofProfile(path, "mutex", opts, client)        // Stack traces of holders of contended mutexes
+}
+
+// savePprofProfile retrieves a pprof profile and writes to disk
+func (c *OperatorDebugCommand) savePprofProfile(path string, profile string, opts api.PprofOptions, client *api.Client) {
+	fileName := fmt.Sprintf("%s.prof", profile)
+	if opts.Debug > 0 {
+		fileName = fmt.Sprintf("%s-debug%d.txt", profile, opts.Debug)
 	}
 
-	// Gather goroutine text output - debug type 2
-	// When printing the "goroutine" profile, debug=2 means to print the goroutine
-	// stacks in the same form that a Go program uses when dying due to an unrecovered panic.
-	opts.Debug = 2
-	bs, err = client.Agent().Lookup("goroutine", opts, nil)
+	bs, err := retrievePprofProfile(profile, opts, client)
 	if err != nil {
-		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof goroutine-debug2.txt, err: %v", path, err))
-	} else {
-		err := c.writeBytes(path, "goroutine-debug2.txt", bs)
-		if err != nil {
-			c.Ui.Error(err.Error())
-		}
+		c.Ui.Error(fmt.Sprintf("%s: Failed to retrieve pprof %s, err: %s", path, fileName, err.Error()))
 	}
-}
 
-// savePprofProfile collects a pprof profile from the client API and writes to disk
-func (c *OperatorDebugCommand) savePprofProfile(path string, profile string, opts api.PprofOptions, client *api.Client) error {
-	bs, err := client.Agent().Lookup(profile, opts, nil)
+	err = c.writeBytes(path, fileName, bs)
 	if err != nil {
-		return fmt.Errorf("%s: Failed to retrieve pprof profile %s, err: %v", path, profile, err)
+		c.Ui.Error(fmt.Sprintf("%s: Failed to write file %s, err: %s", path, fileName, err.Error()))
 	}
+}
 
-	fileName := fmt.Sprintf("%s.prof", profile)
-	err = c.writeBytes(path, fileName, bs)
+// retrievePprofProfile gets a pprof profile from the node specified in opts using the API client
+func retrievePprofProfile(profile string, opts api.PprofOptions, client *api.Client) (bs []byte, err error) {
+	switch profile {
+	case "cpuprofile":
+		bs, err = client.Agent().CPUProfile(opts, nil)
+	case "trace":
+		bs, err = client.Agent().Trace(opts, nil)
+	default:
+		bs, err = client.Agent().Lookup(profile, opts, nil)
+	}
 
-	return err
+	return bs, err
 }
 
 // collectPeriodic runs for duration, capturing the cluster state every interval. It flushes and stops

command/operator_debug_test.go

@@ -365,11 +365,16 @@ func TestDebug_CapturedFiles(t *testing.T) {
 
 		// Monitor files are only created when selected
 		filepath.Join(path, "server", "leader", "monitor.log"),
+
+		// Pprof profiles
 		filepath.Join(path, "server", "leader", "profile.prof"),
 		filepath.Join(path, "server", "leader", "trace.prof"),
 		filepath.Join(path, "server", "leader", "goroutine.prof"),
 		filepath.Join(path, "server", "leader", "goroutine-debug1.txt"),
 		filepath.Join(path, "server", "leader", "goroutine-debug2.txt"),
+		filepath.Join(path, "server", "leader", "heap.prof"),
+		filepath.Join(path, "server", "leader", "allocs.prof"),
+		filepath.Join(path, "server", "leader", "threadcreate.prof"),
 
 		// Multiple snapshots are collected, 00 is always created
 		filepath.Join(path, "nomad", "0000", "jobs.json"),