cli: unhide advanced operator raft debugging commands (#11682)

The `nomad operator raft` and `nomad operator snapshot state`
subcommands for inspecting on-disk raft state were hidden and
undocumented. Expose and document these so that advanced operators
have support for these tools.

.changelog/11682.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Made the `operator raft info`, `operator raft logs`, `operator raft state`, and `operator snapshot state` commands visible to command line help.
+```

command/commands.go

@@ -540,17 +540,17 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
-		"operator raft _info": func() (cli.Command, error) {
+		"operator raft info": func() (cli.Command, error) {
 			return &OperatorRaftInfoCommand{
 				Meta: meta,
 			}, nil
 		},
-		"operator raft _logs": func() (cli.Command, error) {
+		"operator raft logs": func() (cli.Command, error) {
 			return &OperatorRaftLogsCommand{
 				Meta: meta,
 			}, nil
 		},
-		"operator raft _state": func() (cli.Command, error) {
+		"operator raft state": func() (cli.Command, error) {
 			return &OperatorRaftStateCommand{
 				Meta: meta,
 			}, nil
@@ -571,7 +571,7 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
-		"operator snapshot _state": func() (cli.Command, error) {
+		"operator snapshot state": func() (cli.Command, error) {
 			return &OperatorSnapshotStateCommand{
 				Meta: meta,
 			}, nil

command/operator_raft.go

@@ -26,7 +26,22 @@ Usage: nomad operator raft <subcommand> [options]
 
       $ nomad operator raft remove-peer -peer-address "IP:Port"
 
+  Display info about the raft logs in the data directory:
+
+      $ nomad operator raft info /var/nomad/data
+
+  Display the log entries persisted in data dir in JSON format.
+
+      $ nomad operator raft logs /var/nomad/data
+
+  Display the server state obtained by replaying raft log entries
+  persisted in data dir in JSON format.
+
+      $ nomad operator raft state /var/nomad/data
+
   Please see the individual subcommand help for detailed usage information.
+
+
 `
 	return strings.TrimSpace(helpText)
 }

command/operator_raft_info.go

@@ -14,14 +14,16 @@ type OperatorRaftInfoCommand struct {
 
 func (c *OperatorRaftInfoCommand) Help() string {
 	helpText := `
-Usage: nomad operator raft _info <path to nomad data dir>
+Usage: nomad operator raft info <path to nomad data dir>
 
-  Displays info about the raft logs in the data directory.
+  Displays summary information about the raft logs in the data directory.
+
+  This command requires file system permissions to access the data directory on
+  disk. The Nomad server locks access to the data directory, so this command
+  cannot be run on a data directory that is being used by a running Nomad server.
 
   This is a low-level debugging tool and not subject to Nomad's usual backward
   compatibility guarantees.
-
-  If ACLs are enabled, this command requires a management token.
 `
 	return strings.TrimSpace(helpText)
 }
@@ -38,7 +40,7 @@ func (c *OperatorRaftInfoCommand) Synopsis() string {
 	return "Display info of the raft log"
 }
 
-func (c *OperatorRaftInfoCommand) Name() string { return "operator raft _info" }
+func (c *OperatorRaftInfoCommand) Name() string { return "operator raft info" }
 
 func (c *OperatorRaftInfoCommand) Run(args []string) int {
 	if len(args) != 1 {

command/operator_raft_logs.go

@@ -16,14 +16,18 @@ type OperatorRaftLogsCommand struct {
 
 func (c *OperatorRaftLogsCommand) Help() string {
 	helpText := `
-Usage: nomad operator raft _logs <path to nomad data dir>
+Usage: nomad operator raft logs <path to nomad data dir>
 
-  Display the log entries persisted in data dir in json form.
+  Display the log entries persisted in the Nomad data directory in JSON
+  format.
+
+  This command requires file system permissions to access the data directory on
+  disk. The Nomad server locks access to the data directory, so this command
+  cannot be run on a data directory that is being used by a running Nomad server.
 
   This is a low-level debugging tool and not subject to Nomad's usual backward
   compatibility guarantees.
 
-  If ACLs are enabled, this command requires a management token.
 `
 	return strings.TrimSpace(helpText)
 }
@@ -40,7 +44,7 @@ func (c *OperatorRaftLogsCommand) Synopsis() string {
 	return "Display raft log content"
 }
 
-func (c *OperatorRaftLogsCommand) Name() string { return "operator raft _info" }
+func (c *OperatorRaftLogsCommand) Name() string { return "operator raft logs" }
 
 func (c *OperatorRaftLogsCommand) Run(args []string) int {
 	if len(args) != 1 {

command/operator_raft_state.go

@@ -16,15 +16,18 @@ type OperatorRaftStateCommand struct {
 
 func (c *OperatorRaftStateCommand) Help() string {
 	helpText := `
-Usage: nomad operator raft _state <path to nomad data dir>
+Usage: nomad operator raft state <path to nomad data dir>
 
-  Display the server state obtained by replaying raft log entries persisted in data dir in json form.
+  Display the server state obtained by replaying raft log entries persisted in
+  the Nomad data directory in JSON format.
+
+  This command requires file system permissions to access the data directory on
+  disk. The Nomad server locks access to the data directory, so this command
+  cannot be run on a data directory that is being used by a running Nomad server.
 
   This is a low-level debugging tool and not subject to Nomad's usual backward
   compatibility guarantees.
 
-  If ACLs are enabled, this command requires a management token.
-
 Options:
 
   -last-index=<last_index>
@@ -47,7 +50,7 @@ func (c *OperatorRaftStateCommand) Synopsis() string {
 	return "Display raft server state"
 }
 
-func (c *OperatorRaftStateCommand) Name() string { return "operator raft _state" }
+func (c *OperatorRaftStateCommand) Name() string { return "operator raft state" }
 
 func (c *OperatorRaftStateCommand) Run(args []string) int {
 	var fLastIdx int64

command/operator_snapshot_state.go

@@ -16,13 +16,13 @@ type OperatorSnapshotStateCommand struct {
 
 func (c *OperatorSnapshotStateCommand) Help() string {
 	helpText := `
-Usage: nomad operator snapshot _state <file>
+Usage: nomad operator snapshot state <file>
 
   Displays a JSON representation of state in the snapshot.
 
   To inspect the file "backup.snap":
 
-    $ nomad operator snapshot _state backup.snap
+    $ nomad operator snapshot state backup.snap
 `
 	return strings.TrimSpace(helpText)
 }
@@ -39,7 +39,7 @@ func (c *OperatorSnapshotStateCommand) Synopsis() string {
 	return "Displays information about a Nomad snapshot file"
 }
 
-func (c *OperatorSnapshotStateCommand) Name() string { return "operator snapshot _state" }
+func (c *OperatorSnapshotStateCommand) Name() string { return "operator snapshot state" }
 
 func (c *OperatorSnapshotStateCommand) Run(args []string) int {
 	// Check that we either got no filename or exactly one.

website/content/docs/commands/operator/raft-info.mdx

@@ -0,0 +1,39 @@
+---
+layout: docs
+page_title: 'Commands: operator raft info'
+description: |
+  Display Raft server state.
+---
+
+# Command: operator raft info
+
+The `raft info` command is used to display summary information about the
+raft logs persisted in the Nomad [data directory].
+
+This command requires file system permissions to access the data
+directory on disk. The Nomad server locks access to the data
+directory, so this command cannot be run on a data directory that is
+being used by a running Nomad server.
+
+~> **Warning:** This is a low-level debugging tool and not subject to
+  Nomad's usual backward compatibility guarantees.
+
+## Usage
+
+```plaintext
+nomad operator raft info <path to data dir>
+```
+
+## Examples
+
+An example output is as follows:
+
+```shell-session
+$ sudo nomad operator raft info /var/nomad/data
+path:        /var/nomad/data/server/raft/raft.db
+length:      10
+first index: 1
+last index:  10
+```
+
+[data directory]: /docs/configuration#data_dir

website/content/docs/commands/operator/raft-list-peers.mdx

@@ -7,7 +7,7 @@ description: |
 
 # Command: operator raft list-peers
 
-The Raft list-peers command is used to display the current Raft peer
+The `raft list-peers` command is used to display the current Raft peer
 configuration.
 
 See the [Outage Recovery] guide for some examples of how this command is used.

website/content/docs/commands/operator/raft-logs.mdx

@@ -0,0 +1,38 @@
+---
+layout: docs
+page_title: 'Commands: operator raft logs'
+description: |
+  Display Raft server state.
+---
+
+# Command: operator raft logs
+
+The `raft logs` command is used to display the log entries persisted in
+the Nomad [data directory] in JSON format.
+
+This command requires file system permissions to access the data
+directory on disk. The Nomad server locks access to the data
+directory, so this command cannot be run on a data directory that is
+being used by a running Nomad server.
+
+~> **Warning:** This is a low-level debugging tool and not subject to
+  Nomad's usual backward compatibility guarantees.
+
+## Usage
+
+```plaintext
+nomad operator raft logs [options] <path to data dir>
+```
+
+## Examples
+
+The output of this command can be very large, so it's recommended that
+you redirect the output to a file for later examination with other
+tools.
+
+```shell-session
+$ sudo nomad operator raft logs /var/nomad/data > ~/raft-logs.json
+$ jq . < ~/raft-logs.json
+```
+
+[data directory]: /docs/configuration#data_dir

website/content/docs/commands/operator/raft-state.mdx

@@ -0,0 +1,46 @@
+---
+layout: docs
+page_title: 'Commands: operator raft state'
+description: |
+  Display Raft server state.
+---
+
+# Command: operator raft state
+
+The `raft state` command is used to display the server state obtained by
+replaying raft log entries persisted in the Nomad [data directory] in
+JSON format.
+
+This command requires file system permissions to access the data
+directory on disk. The Nomad server locks access to the data
+directory, so this command cannot be run on a data directory that is
+being used by a running Nomad server.
+
+~> **Warning:** This is a low-level debugging tool and not subject to
+  Nomad's usual backward compatibility guarantees.
+
+## Usage
+
+```plaintext
+nomad operator raft state [options] <path to data dir>
+```
+
+## Raft State Options
+
+- `-last-index=<last_index>`: Set the last log index to be applied, to
+  drop spurious log entries not properly committed. If the
+  `last_index` option is zero or negative, it's treated as an offset
+  from the last index seen in raft.
+
+## Examples
+
+The output of this command can be very large, so it's recommended that
+you redirect the output to a file for later examination with other
+tools.
+
+```shell-session
+$ sudo nomad operator raft state /var/nomad/data > ~/raft-state.json
+$ jq . < ~/raft-state.json
+```
+
+[data directory]: /docs/configuration#data_dir

website/content/docs/commands/operator/snapshot-state.mdx

@@ -0,0 +1,30 @@
+---
+layout: docs
+page_title: 'Commands: operator snapshot state'
+description: |
+  Displays a JSON representation of a Raft snapshot.
+---
+
+# Command: operator snapshot state
+
+Displays a JSON representation of state in a raft snapshot on disk.
+
+~> **Warning:** This is a low-level debugging tool and not subject to
+  Nomad's usual backward compatibility guarantees.
+
+## Usage
+
+```plaintext
+nomad operator snapshot state <file>
+```
+
+## Examples
+
+The output of this command can be very large, so it's recommended that
+you redirect the output to a file for later examination with other
+tools.
+
+```shell-session
+$ nomad operator snapshot state backup.snap > ~/raft-state.json
+$ jq . < ~/raft-state.json
+```

website/data/docs-nav-data.json

@@ -548,14 +548,26 @@
             "title": "metrics",
             "path": "commands/operator/metrics"
           },
+          {
+            "title": "raft info",
+            "path": "commands/operator/raft-info"
+          },
           {
             "title": "raft list-peers",
             "path": "commands/operator/raft-list-peers"
           },
+          {
+            "title": "raft logs",
+            "path": "commands/operator/raft-logs"
+          },
           {
             "title": "raft remove-peer",
             "path": "commands/operator/raft-remove-peer"
           },
+          {
+            "title": "raft state",
+            "path": "commands/operator/raft-state"
+          },
           {
             "title": "snapshot agent",
             "path": "commands/operator/snapshot-agent"
@@ -571,6 +583,10 @@
           {
             "title": "snapshot save",
             "path": "commands/operator/snapshot-save"
+          },
+          {
+            "title": "snapshot state",
+            "path": "commands/operator/snapshot-state"
           }
         ]
       },