From bf8bdc94f8fb9bf5bd4ebf644951f583f5642ba6 Mon Sep 17 00:00:00 2001 From: davemay99 Date: Mon, 5 Oct 2020 22:30:01 -0400 Subject: [PATCH 1/5] Add metrics command / output to debug bundle --- api/operator.go | 19 +++++++ command/commands.go | 5 ++ command/metrics.go | 101 +++++++++++++++++++++++++++++++++ command/metrics_test.go | 79 ++++++++++++++++++++++++++ command/operator_debug.go | 4 ++ command/operator_debug_test.go | 25 ++++++++ 6 files changed, 233 insertions(+) create mode 100644 command/metrics.go create mode 100644 command/metrics_test.go diff --git a/api/operator.go b/api/operator.go index d5bc5d061d5..e84dd1d4cec 100644 --- a/api/operator.go +++ b/api/operator.go @@ -304,3 +304,22 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro } return &reply, qm, nil } + +func (op *Operator) Metrics(q *QueryOptions) (string, error) { + if q == nil { + q = &QueryOptions{} + } + + metricsReader, err := op.c.rawQuery("/v1/metrics", q) + if err != nil { + return "", err + } + + metricsBytes, err := ioutil.ReadAll(metricsReader) + if err != nil { + return "", err + } + + metrics := string(metricsBytes[:]) + return metrics, nil +} diff --git a/command/commands.go b/command/commands.go index 3bb5831bff1..67e61a37fc3 100644 --- a/command/commands.go +++ b/command/commands.go @@ -500,6 +500,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "operator metrics": func() (cli.Command, error) { + return &OperatorMetricsCommand{ + Meta: meta, + }, nil + }, "operator raft": func() (cli.Command, error) { return &OperatorRaftCommand{ Meta: meta, diff --git a/command/metrics.go b/command/metrics.go new file mode 100644 index 00000000000..116f8bd7718 --- /dev/null +++ b/command/metrics.go @@ -0,0 +1,101 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/posener/complete" +) + +//var _ cli.Command = &MetricsCommand{} + +type OperatorMetricsCommand struct { + Meta +} + +func (c *OperatorMetricsCommand) Help() string { + helpText := ` +Usage: nomad operator metrics [options] + +Get Nomad metrics +General Options: + + ` + generalOptionsUsage() + ` + +Metrics Specific Options + + -pretty + Pretty prints the JSON output + + -format + Specify output format (prometheus) +` + + return strings.TrimSpace(helpText) +} + +func (c *OperatorMetricsCommand) Synopsis() string { + return "Retrieve Nomad metrics" +} + +func (c *OperatorMetricsCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-pretty": complete.PredictAnything, + "-format": complete.PredictAnything, + }) +} + +func (c *OperatorMetricsCommand) Name() string { return "metrics" } + +func (c *OperatorMetricsCommand) Run(args []string) int { + var pretty bool + var format string + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&pretty, "pretty", false, "") + flags.StringVar(&format, "format", "", "") + + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing flags: %s", err)) + return 1 + } + + args = flags.Args() + if l := len(args); l != 0 { + c.Ui.Error("This command takes no arguments") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + params := map[string]string{} + + if pretty { + params["pretty"] = "1" + } + + if len(format) > 0 { + params["format"] = format + } + + query := &api.QueryOptions{ + Params: params, + } + + resp, err := client.Operator().Metrics(query) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error getting metrics: %v", err)) + return 1 + } + + c.Ui.Output(resp) + return 0 +} diff --git a/command/metrics_test.go b/command/metrics_test.go new file mode 100644 index 00000000000..fd0c71e5835 --- /dev/null +++ b/command/metrics_test.go @@ -0,0 +1,79 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +var _ cli.Command = &OperatorMetricsCommand{} + +func TestCommand_Metrics_Cases(t *testing.T) { + t.Parallel() + + srv, _, url := testServer(t, false, nil) + defer srv.Shutdown() + + ui := new(cli.MockUi) + cmd := &OperatorMetricsCommand{Meta: Meta{Ui: ui}} + + cases := []struct { + name string + args []string + expectedCode int + expectedOutput string + expectedError string + }{ + { + "pretty print json", + []string{"-address=" + url, "-pretty"}, + 0, + "{", + "", + }, + { + "prometheus format", + []string{"-address=" + url, "-format", "prometheus"}, + 0, + "# HELP", + "", + }, + { + "bad argument", + []string{"-address=" + url, "-foo", "bar"}, + 1, + "Usage: nomad operator metrics", + "flag provided but not defined: -foo", + }, + { + "bad address - no protocol", + []string{"-address=foo"}, + 1, + "", + "Error getting metrics: Get \"/v1/metrics\": unsupported protocol scheme", + }, + { + "bad address - fake host", + []string{"-address=http://foo"}, + 1, + "", + "no such host", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + code := cmd.Run(c.args) + out := ui.OutputWriter.String() + outerr := ui.ErrorWriter.String() + + require.Equalf(t, code, c.expectedCode, "expected exit code %d, got: %d: %s", c.expectedCode, code, outerr) + require.Contains(t, out, c.expectedOutput, "expected output \"%s\", got \"%s\"", c.expectedOutput, out) + require.Containsf(t, outerr, c.expectedError, "expected error \"%s\", got \"%s\"", c.expectedError, outerr) + + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() + }) + } +} diff --git a/command/operator_debug.go b/command/operator_debug.go index d8260f560de..b240724e276 100644 --- a/command/operator_debug.go +++ b/command/operator_debug.go @@ -200,6 +200,7 @@ func (c *OperatorDebugCommand) Run(args []string) int { flags.StringVar(&c.vault.tls.ClientKey, "vault-client-key", os.Getenv("VAULT_CLIENT_KEY"), "") if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing arguments: %q", err)) return 1 } @@ -575,6 +576,9 @@ func (c *OperatorDebugCommand) collectNomad(dir string, client *api.Client) erro vs, _, err := client.CSIVolumes().List(qo) c.writeJSON(dir, "volumes.json", vs, err) + metrics, err := client.Operator().Metrics(qo) + c.writeJSON(dir, "metrics.json", metrics, err) + return nil } diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index 3d18ab51d33..9ff58fe1d5f 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -32,6 +32,29 @@ func TestDebugUtils(t *testing.T) { require.Equal(t, "https://127.0.0.1:8500", e.addr("foo")) } +func TestDebugSuccesses(t *testing.T) { + t.Parallel() + srv, _, _ := testServer(t, false, nil) + defer srv.Shutdown() + + ui := new(cli.MockUi) + cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} + + // NOTE -- duration must be shorter than default 2m to prevent testify from timing out + + // Debug on the leader + code := cmd.Run([]string{"-duration", "250ms", "-server-id", "leader"}) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Starting debugger") + ui.OutputWriter.Reset() + + // Debug on all servers + code = cmd.Run([]string{"-duration", "250ms", "-server-id", "all"}) + require.Equal(t, 0, code) + require.Contains(t, ui.OutputWriter.String(), "Starting debugger") + ui.OutputWriter.Reset() +} + func TestDebugFails(t *testing.T) { t.Parallel() srv, _, _ := testServer(t, false, nil) @@ -111,8 +134,10 @@ func TestDebugCapturedFiles(t *testing.T) { // Multiple snapshots are collected, 00 is always created require.FileExists(t, filepath.Join(path, "nomad", "0000", "jobs.json")) require.FileExists(t, filepath.Join(path, "nomad", "0000", "nodes.json")) + require.FileExists(t, filepath.Join(path, "nomad", "0000", "metrics.json")) // Multiple snapshots are collected, 01 requires two intervals require.FileExists(t, filepath.Join(path, "nomad", "0001", "jobs.json")) require.FileExists(t, filepath.Join(path, "nomad", "0001", "nodes.json")) + require.FileExists(t, filepath.Join(path, "nomad", "0001", "metrics.json")) } From 29dbd2239585cb513bd672e76cb37d50fdf0d5f5 Mon Sep 17 00:00:00 2001 From: davemay99 Date: Tue, 6 Oct 2020 09:16:52 -0400 Subject: [PATCH 2/5] sync vendored modules --- command/metrics.go | 2 -- .../hashicorp/nomad/api/operator.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/command/metrics.go b/command/metrics.go index 116f8bd7718..43805c7e276 100644 --- a/command/metrics.go +++ b/command/metrics.go @@ -8,8 +8,6 @@ import ( "github.com/posener/complete" ) -//var _ cli.Command = &MetricsCommand{} - type OperatorMetricsCommand struct { Meta } diff --git a/vendor/github.com/hashicorp/nomad/api/operator.go b/vendor/github.com/hashicorp/nomad/api/operator.go index d5bc5d061d5..e84dd1d4cec 100644 --- a/vendor/github.com/hashicorp/nomad/api/operator.go +++ b/vendor/github.com/hashicorp/nomad/api/operator.go @@ -304,3 +304,22 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro } return &reply, qm, nil } + +func (op *Operator) Metrics(q *QueryOptions) (string, error) { + if q == nil { + q = &QueryOptions{} + } + + metricsReader, err := op.c.rawQuery("/v1/metrics", q) + if err != nil { + return "", err + } + + metricsBytes, err := ioutil.ReadAll(metricsReader) + if err != nil { + return "", err + } + + metrics := string(metricsBytes[:]) + return metrics, nil +} From e0e3a01404d22b2fd026cb1f6188b12d62780b36 Mon Sep 17 00:00:00 2001 From: davemay99 Date: Tue, 6 Oct 2020 09:47:16 -0400 Subject: [PATCH 3/5] update deprecated syntax per GH-9027 --- command/metrics_test.go | 2 +- command/operator_debug_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/command/metrics_test.go b/command/metrics_test.go index fd0c71e5835..f909d4e6cca 100644 --- a/command/metrics_test.go +++ b/command/metrics_test.go @@ -15,7 +15,7 @@ func TestCommand_Metrics_Cases(t *testing.T) { srv, _, url := testServer(t, false, nil) defer srv.Shutdown() - ui := new(cli.MockUi) + ui := cli.NewMockUi() cmd := &OperatorMetricsCommand{Meta: Meta{Ui: ui}} cases := []struct { diff --git a/command/operator_debug_test.go b/command/operator_debug_test.go index 9ff58fe1d5f..d329deada8d 100644 --- a/command/operator_debug_test.go +++ b/command/operator_debug_test.go @@ -37,7 +37,7 @@ func TestDebugSuccesses(t *testing.T) { srv, _, _ := testServer(t, false, nil) defer srv.Shutdown() - ui := new(cli.MockUi) + ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} // NOTE -- duration must be shorter than default 2m to prevent testify from timing out @@ -60,7 +60,7 @@ func TestDebugFails(t *testing.T) { srv, _, _ := testServer(t, false, nil) defer srv.Shutdown() - ui := new(cli.MockUi) + ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} // Fails incorrect args @@ -98,7 +98,7 @@ func TestDebugCapturedFiles(t *testing.T) { srv, _, url := testServer(t, false, nil) defer srv.Shutdown() - ui := new(cli.MockUi) + ui := cli.NewMockUi() cmd := &OperatorDebugCommand{Meta: Meta{Ui: ui}} code := cmd.Run([]string{ From ec09c593b5d0a347a781dcd128c894dac23bf6fc Mon Sep 17 00:00:00 2001 From: davemay99 Date: Tue, 6 Oct 2020 10:49:15 -0400 Subject: [PATCH 4/5] metrics return bytes instead of string for more flexibility --- api/operator.go | 9 ++++----- command/metrics.go | 4 +++- vendor/github.com/hashicorp/nomad/api/operator.go | 9 ++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/operator.go b/api/operator.go index e84dd1d4cec..de9fbd7fc7f 100644 --- a/api/operator.go +++ b/api/operator.go @@ -305,21 +305,20 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro return &reply, qm, nil } -func (op *Operator) Metrics(q *QueryOptions) (string, error) { +func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { if q == nil { q = &QueryOptions{} } metricsReader, err := op.c.rawQuery("/v1/metrics", q) if err != nil { - return "", err + return nil, err } metricsBytes, err := ioutil.ReadAll(metricsReader) if err != nil { - return "", err + return nil, err } - metrics := string(metricsBytes[:]) - return metrics, nil + return metricsBytes, nil } diff --git a/command/metrics.go b/command/metrics.go index 43805c7e276..5ceb3977866 100644 --- a/command/metrics.go +++ b/command/metrics.go @@ -88,12 +88,14 @@ func (c *OperatorMetricsCommand) Run(args []string) int { Params: params, } - resp, err := client.Operator().Metrics(query) + bs, err := client.Operator().Metrics(query) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting metrics: %v", err)) return 1 } + resp := string(bs[:]) c.Ui.Output(resp) + return 0 } diff --git a/vendor/github.com/hashicorp/nomad/api/operator.go b/vendor/github.com/hashicorp/nomad/api/operator.go index e84dd1d4cec..de9fbd7fc7f 100644 --- a/vendor/github.com/hashicorp/nomad/api/operator.go +++ b/vendor/github.com/hashicorp/nomad/api/operator.go @@ -305,21 +305,20 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro return &reply, qm, nil } -func (op *Operator) Metrics(q *QueryOptions) (string, error) { +func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { if q == nil { q = &QueryOptions{} } metricsReader, err := op.c.rawQuery("/v1/metrics", q) if err != nil { - return "", err + return nil, err } metricsBytes, err := ioutil.ReadAll(metricsReader) if err != nil { - return "", err + return nil, err } - metrics := string(metricsBytes[:]) - return metrics, nil + return metricsBytes, nil } From ff1578f0f3e3c637f0b8411698eca3237b177fbe Mon Sep 17 00:00:00 2001 From: davemay99 Date: Tue, 6 Oct 2020 11:22:10 -0400 Subject: [PATCH 5/5] added comment to operator metrics function --- api/operator.go | 1 + vendor/github.com/hashicorp/nomad/api/operator.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/operator.go b/api/operator.go index de9fbd7fc7f..cbc5e24a190 100644 --- a/api/operator.go +++ b/api/operator.go @@ -305,6 +305,7 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro return &reply, qm, nil } +// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { if q == nil { q = &QueryOptions{} diff --git a/vendor/github.com/hashicorp/nomad/api/operator.go b/vendor/github.com/hashicorp/nomad/api/operator.go index de9fbd7fc7f..cbc5e24a190 100644 --- a/vendor/github.com/hashicorp/nomad/api/operator.go +++ b/vendor/github.com/hashicorp/nomad/api/operator.go @@ -305,6 +305,7 @@ func (op *Operator) LicenseGet(q *QueryOptions) (*LicenseReply, *QueryMeta, erro return &reply, qm, nil } +// Metrics returns a slice of bytes containing metrics, optionally formatted as either json or prometheus func (op *Operator) Metrics(q *QueryOptions) ([]byte, error) { if q == nil { q = &QueryOptions{}