diff --git a/api/operator.go b/api/operator.go new file mode 100644 index 00000000000..dcf65b3321f --- /dev/null +++ b/api/operator.go @@ -0,0 +1,81 @@ +package api + +// Operator can be used to perform low-level operator tasks for Nomad. +type Operator struct { + c *Client +} + +// Operator returns a handle to the operator endpoints. +func (c *Client) Operator() *Operator { + return &Operator{c} +} + +// RaftServer has information about a server in the Raft configuration. +type RaftServer struct { + // ID is the unique ID for the server. These are currently the same + // as the address, but they will be changed to a real GUID in a future + // release of Nomad. + ID string + + // Node is the node name of the server, as known by Nomad, or this + // will be set to "(unknown)" otherwise. + Node string + + // Address is the IP:port of the server, used for Raft communications. + Address string + + // Leader is true if this server is the current cluster leader. + Leader bool + + // Voter is true if this server has a vote in the cluster. This might + // be false if the server is staging and still coming online, or if + // it's a non-voting server, which will be added in a future release of + // Nomad. + Voter bool +} + +// RaftConfigration is returned when querying for the current Raft configuration. +type RaftConfiguration struct { + // Servers has the list of servers in the Raft configuration. + Servers []*RaftServer + + // Index has the Raft index of this configuration. + Index uint64 +} + +// RaftGetConfiguration is used to query the current Raft peer set. +func (op *Operator) RaftGetConfiguration(q *QueryOptions) (*RaftConfiguration, error) { + r := op.c.newRequest("GET", "/v1/operator/raft/configuration") + r.setQueryOptions(q) + _, resp, err := requireOK(op.c.doRequest(r)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var out RaftConfiguration + if err := decodeBody(resp, &out); err != nil { + return nil, err + } + return &out, nil +} + +// RaftRemovePeerByAddress is used to kick a stale peer (one that it in the Raft +// quorum but no longer known to Serf or the catalog) by address in the form of +// "IP:port". +func (op *Operator) RaftRemovePeerByAddress(address string, q *WriteOptions) error { + r := op.c.newRequest("DELETE", "/v1/operator/raft/peer") + r.setWriteOptions(q) + + // TODO (alexdadgar) Currently we made address a query parameter. Once + // IDs are in place this will be DELETE /v1/operator/raft/peer/. + r.params.Set("address", string(address)) + + _, resp, err := requireOK(op.c.doRequest(r)) + if err != nil { + return err + } + + resp.Body.Close() + return nil +} diff --git a/api/operator_test.go b/api/operator_test.go new file mode 100644 index 00000000000..4db2c4c4514 --- /dev/null +++ b/api/operator_test.go @@ -0,0 +1,36 @@ +package api + +import ( + "strings" + "testing" +) + +func TestOperator_RaftGetConfiguration(t *testing.T) { + c, s := makeClient(t, nil, nil) + defer s.Stop() + + operator := c.Operator() + out, err := operator.RaftGetConfiguration(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter { + t.Fatalf("bad: %v", out) + } +} + +func TestOperator_RaftRemovePeerByAddress(t *testing.T) { + c, s := makeClient(t, nil, nil) + defer s.Stop() + + // If we get this error, it proves we sent the address all the way + // through. + operator := c.Operator() + err := operator.RaftRemovePeerByAddress("nope", nil) + if err == nil || !strings.Contains(err.Error(), + "address \"nope\" was not found in the Raft configuration") { + t.Fatalf("err: %v", err) + } +} diff --git a/command/agent/http.go b/command/agent/http.go index 5a31f7d4e29..e111baea84a 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -170,6 +170,8 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/status/leader", s.wrap(s.StatusLeaderRequest)) s.mux.HandleFunc("/v1/status/peers", s.wrap(s.StatusPeersRequest)) + s.mux.HandleFunc("/v1/operator/", s.wrap(s.OperatorRequest)) + s.mux.HandleFunc("/v1/system/gc", s.wrap(s.GarbageCollectRequest)) s.mux.HandleFunc("/v1/system/reconcile/summaries", s.wrap(s.ReconcileJobSummaries)) diff --git a/command/agent/operator_endpoint.go b/command/agent/operator_endpoint.go new file mode 100644 index 00000000000..0a898de4b51 --- /dev/null +++ b/command/agent/operator_endpoint.go @@ -0,0 +1,69 @@ +package agent + +import ( + "net/http" + "strings" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/raft" +) + +func (s *HTTPServer) OperatorRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + path := strings.TrimPrefix(req.URL.Path, "/v1/operator/raft/") + switch { + case strings.HasPrefix(path, "configuration"): + return s.OperatorRaftConfiguration(resp, req) + case strings.HasPrefix(path, "peer"): + return s.OperatorRaftPeer(resp, req) + default: + return nil, CodedError(404, ErrInvalidMethod) + } +} + +// OperatorRaftConfiguration is used to inspect the current Raft configuration. +// This supports the stale query mode in case the cluster doesn't have a leader. +func (s *HTTPServer) OperatorRaftConfiguration(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != "GET" { + resp.WriteHeader(http.StatusMethodNotAllowed) + return nil, nil + } + + var args structs.GenericRequest + if done := s.parse(resp, req, &args.Region, &args.QueryOptions); done { + return nil, nil + } + + var reply structs.RaftConfigurationResponse + if err := s.agent.RPC("Operator.RaftGetConfiguration", &args, &reply); err != nil { + return nil, err + } + + return reply, nil +} + +// OperatorRaftPeer supports actions on Raft peers. Currently we only support +// removing peers by address. +func (s *HTTPServer) OperatorRaftPeer(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != "DELETE" { + resp.WriteHeader(http.StatusMethodNotAllowed) + return nil, nil + } + + var args structs.RaftPeerByAddressRequest + s.parseRegion(req, &args.Region) + + params := req.URL.Query() + if _, ok := params["address"]; ok { + args.Address = raft.ServerAddress(params.Get("address")) + } else { + resp.WriteHeader(http.StatusBadRequest) + resp.Write([]byte("Must specify ?address with IP:port of peer to remove")) + return nil, nil + } + + var reply struct{} + if err := s.agent.RPC("Operator.RaftRemovePeerByAddress", &args, &reply); err != nil { + return nil, err + } + return nil, nil +} diff --git a/command/agent/operator_endpoint_test.go b/command/agent/operator_endpoint_test.go new file mode 100644 index 00000000000..5b5de7d8233 --- /dev/null +++ b/command/agent/operator_endpoint_test.go @@ -0,0 +1,58 @@ +package agent + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestHTTP_OperatorRaftConfiguration(t *testing.T) { + httpTest(t, nil, func(s *TestServer) { + body := bytes.NewBuffer(nil) + req, err := http.NewRequest("GET", "/v1/operator/raft/configuration", body) + if err != nil { + t.Fatalf("err: %v", err) + } + + resp := httptest.NewRecorder() + obj, err := s.Server.OperatorRaftConfiguration(resp, req) + if err != nil { + t.Fatalf("err: %v", err) + } + if resp.Code != 200 { + t.Fatalf("bad code: %d", resp.Code) + } + out, ok := obj.(structs.RaftConfigurationResponse) + if !ok { + t.Fatalf("unexpected: %T", obj) + } + if len(out.Servers) != 1 || + !out.Servers[0].Leader || + !out.Servers[0].Voter { + t.Fatalf("bad: %v", out) + } + }) +} + +func TestHTTP_OperatorRaftPeer(t *testing.T) { + httpTest(t, nil, func(s *TestServer) { + body := bytes.NewBuffer(nil) + req, err := http.NewRequest("DELETE", "/v1/operator/raft/peer?address=nope", body) + if err != nil { + t.Fatalf("err: %v", err) + } + + // If we get this error, it proves we sent the address all the + // way through. + resp := httptest.NewRecorder() + _, err = s.Server.OperatorRaftPeer(resp, req) + if err == nil || !strings.Contains(err.Error(), + "address \"nope\" was not found in the Raft configuration") { + t.Fatalf("err: %v", err) + } + }) +} diff --git a/command/job_dispatch.go b/command/job_dispatch.go index 702fa4a97db..b53567509e1 100644 --- a/command/job_dispatch.go +++ b/command/job_dispatch.go @@ -33,11 +33,11 @@ General Options: Dispatch Options: -meta = - Meta takes a key/value pair seperated by "=". The metadata key will be - merged into the job's metadata. The job may define a default value for the - key which is overriden when dispatching. The flag can be provided more than - once to inject multiple metadata key/value pairs. Arbitrary keys are not - allowed. The parameterized job must allow the key to be merged. + Meta takes a key/value pair seperated by "=". The metadata key will be + merged into the job's metadata. The job may define a default value for the + key which is overriden when dispatching. The flag can be provided more than + once to inject multiple metadata key/value pairs. Arbitrary keys are not + allowed. The parameterized job must allow the key to be merged. -detach Return immediately instead of entering monitor mode. After job dispatch, diff --git a/command/operator.go b/command/operator.go new file mode 100644 index 00000000000..db5ed3574a3 --- /dev/null +++ b/command/operator.go @@ -0,0 +1,32 @@ +package command + +import ( + "strings" + + "github.com/mitchellh/cli" +) + +type OperatorCommand struct { + Meta +} + +func (f *OperatorCommand) Help() string { + helpText := ` +Usage: nomad operator [options] + + Provides cluster-level tools for Nomad operators, such as interacting with + the Raft subsystem. NOTE: Use this command with extreme caution, as improper + use could lead to a Nomad outage and even loss of data. + + Run nomad operator with no arguments for help on that subcommand. +` + return strings.TrimSpace(helpText) +} + +func (f *OperatorCommand) Synopsis() string { + return "Provides cluster-level tools for Nomad operators" +} + +func (f *OperatorCommand) Run(args []string) int { + return cli.RunResultHelp +} diff --git a/command/operator_raft.go b/command/operator_raft.go new file mode 100644 index 00000000000..450988e0f3e --- /dev/null +++ b/command/operator_raft.go @@ -0,0 +1,30 @@ +package command + +import ( + "strings" + + "github.com/mitchellh/cli" +) + +type OperatorRaftCommand struct { + Meta +} + +func (c *OperatorRaftCommand) Help() string { + helpText := ` +Usage: nomad operator raft [options] + +The Raft operator command is used to interact with Nomad's Raft subsystem. The +command can be used to verify Raft peers or in rare cases to recover quorum by +removing invalid peers. +` + return strings.TrimSpace(helpText) +} + +func (c *OperatorRaftCommand) Synopsis() string { + return "Provides access to the Raft subsystem" +} + +func (c *OperatorRaftCommand) Run(args []string) int { + return cli.RunResultHelp +} diff --git a/command/operator_raft_list.go b/command/operator_raft_list.go new file mode 100644 index 00000000000..9c7bac8b6a1 --- /dev/null +++ b/command/operator_raft_list.go @@ -0,0 +1,82 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/ryanuber/columnize" +) + +type OperatorRaftListCommand struct { + Meta +} + +func (c *OperatorRaftListCommand) Help() string { + helpText := ` +Usage: nomad operator raft list-peers [options] + +Displays the current Raft peer configuration. + +General Options: + + ` + generalOptionsUsage() + ` + +List Peers Options: + + -stale=[true|false] + The -stale argument defaults to "false" which means the leader provides the + result. If the cluster is in an outage state without a leader, you may need + to set -stale to "true" to get the configuration from a non-leader server. +` + return strings.TrimSpace(helpText) +} + +func (c *OperatorRaftListCommand) Synopsis() string { + return "Display the current Raft peer configuration" +} + +func (c *OperatorRaftListCommand) Run(args []string) int { + var stale bool + + flags := c.Meta.FlagSet("raft", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + flags.BoolVar(&stale, "stale", false, "") + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err)) + return 1 + } + + // Set up a client. + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + operator := client.Operator() + + // Fetch the current configuration. + q := &api.QueryOptions{ + AllowStale: stale, + } + reply, err := operator.RaftGetConfiguration(q) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to retrieve raft configuration: %v", err)) + return 1 + } + + // Format it as a nice table. + result := []string{"Node|ID|Address|State|Voter"} + for _, s := range reply.Servers { + state := "follower" + if s.Leader { + state = "leader" + } + result = append(result, fmt.Sprintf("%s|%s|%s|%s|%v", + s.Node, s.ID, s.Address, state, s.Voter)) + } + c.Ui.Output(columnize.SimpleFormat(result)) + + return 0 +} diff --git a/command/operator_raft_list_test.go b/command/operator_raft_list_test.go new file mode 100644 index 00000000000..b96b66f504f --- /dev/null +++ b/command/operator_raft_list_test.go @@ -0,0 +1,30 @@ +package command + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Raft_ListPeers_Implements(t *testing.T) { + var _ cli.Command = &OperatorRaftListCommand{} +} + +func TestOperator_Raft_ListPeers(t *testing.T) { + s, _, addr := testServer(t, nil) + defer s.Stop() + + ui := new(cli.MockUi) + c := &OperatorRaftListCommand{Meta: Meta{Ui: ui}} + args := []string{"-address=" + addr} + + code := c.Run(args) + if code != 0 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + output := strings.TrimSpace(ui.OutputWriter.String()) + if !strings.Contains(output, "leader") { + t.Fatalf("bad: %s", output) + } +} diff --git a/command/operator_raft_remove.go b/command/operator_raft_remove.go new file mode 100644 index 00000000000..bb5789cfde5 --- /dev/null +++ b/command/operator_raft_remove.go @@ -0,0 +1,79 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" +) + +type OperatorRaftRemoveCommand struct { + Meta +} + +func (c *OperatorRaftRemoveCommand) Help() string { + helpText := ` +Usage: nomad operator raft remove-peer [options] + +Remove the Nomad server with given -peer-address from the Raft configuration. + +There are rare cases where a peer may be left behind in the Raft quorum even +though the server is no longer present and known to the cluster. This command +can be used to remove the failed server so that it is no longer affects the Raft +quorum. If the server still shows in the output of the "nomad server-members" +command, it is preferable to clean up by simply running "nomad +server-force-leave" instead of this command. + +General Options: + + ` + generalOptionsUsage() + ` + +Remove Peer Options: + + -peer-address="IP:port" + Remove a Nomad server with given address from the Raft configuration. +` + return strings.TrimSpace(helpText) +} + +func (c *OperatorRaftRemoveCommand) Synopsis() string { + return "Remove a Nomad server from the Raft configuration" +} + +func (c *OperatorRaftRemoveCommand) Run(args []string) int { + var peerAddress string + + flags := c.Meta.FlagSet("raft", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + flags.StringVar(&peerAddress, "peer-address", "", "") + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err)) + return 1 + } + + // Set up a client. + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + operator := client.Operator() + + // TODO (alexdadgar) Once we expose IDs, add support for removing + // by ID, add support for that. + if len(peerAddress) == 0 { + c.Ui.Error(fmt.Sprintf("an address is required for the peer to remove")) + return 1 + } + + // Try to kick the peer. + w := &api.WriteOptions{} + if err := operator.RaftRemovePeerByAddress(peerAddress, w); err != nil { + c.Ui.Error(fmt.Sprintf("Failed to remove raft peer: %v", err)) + return 1 + } + c.Ui.Output(fmt.Sprintf("Removed peer with address %q", peerAddress)) + + return 0 +} diff --git a/command/operator_raft_remove_test.go b/command/operator_raft_remove_test.go new file mode 100644 index 00000000000..a5954d03f8b --- /dev/null +++ b/command/operator_raft_remove_test.go @@ -0,0 +1,32 @@ +package command + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Raft_RemovePeers_Implements(t *testing.T) { + var _ cli.Command = &OperatorRaftRemoveCommand{} +} + +func TestOperator_Raft_RemovePeer(t *testing.T) { + s, _, addr := testServer(t, nil) + defer s.Stop() + + ui := new(cli.MockUi) + c := &OperatorRaftRemoveCommand{Meta: Meta{Ui: ui}} + args := []string{"-address=" + addr, "-peer-address=nope"} + + code := c.Run(args) + if code != 1 { + t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String()) + } + + // If we get this error, it proves we sent the address all they through. + output := strings.TrimSpace(ui.ErrorWriter.String()) + if !strings.Contains(output, "address \"nope\" was not found in the Raft configuration") { + t.Fatalf("bad: %s", output) + } +} diff --git a/command/operator_raft_test.go b/command/operator_raft_test.go new file mode 100644 index 00000000000..a4ce6269be3 --- /dev/null +++ b/command/operator_raft_test.go @@ -0,0 +1,11 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Raft_Implements(t *testing.T) { + var _ cli.Command = &OperatorRaftCommand{} +} diff --git a/command/operator_test.go b/command/operator_test.go new file mode 100644 index 00000000000..485f9754494 --- /dev/null +++ b/command/operator_test.go @@ -0,0 +1,11 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" +) + +func TestOperator_Implements(t *testing.T) { + var _ cli.Command = &OperatorCommand{} +} diff --git a/commands.go b/commands.go index 3def861c439..9bc568deab1 100644 --- a/commands.go +++ b/commands.go @@ -115,6 +115,30 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { }, nil }, + "operator": func() (cli.Command, error) { + return &command.OperatorCommand{ + Meta: meta, + }, nil + }, + + "operator raft": func() (cli.Command, error) { + return &command.OperatorRaftCommand{ + Meta: meta, + }, nil + }, + + "operator raft list-peers": func() (cli.Command, error) { + return &command.OperatorRaftListCommand{ + Meta: meta, + }, nil + }, + + "operator raft remove-peer": func() (cli.Command, error) { + return &command.OperatorRaftRemoveCommand{ + Meta: meta, + }, nil + }, + "plan": func() (cli.Command, error) { return &command.PlanCommand{ Meta: meta, diff --git a/main.go b/main.go index 35e26efd37b..cc42fd27987 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,8 @@ func RunCustom(args []string, commands map[string]cli.CommandFactory) int { switch k { case "executor": case "syslog": + case "operator raft", "operator raft list-peers", "operator raft remove-peer": + case "job dispatch": case "fs ls", "fs cat", "fs stat": case "check": default: diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go new file mode 100644 index 00000000000..576962aa313 --- /dev/null +++ b/nomad/operator_endpoint.go @@ -0,0 +1,107 @@ +package nomad + +import ( + "fmt" + "net" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/raft" + "github.com/hashicorp/serf/serf" +) + +// Operator endpoint is used to perform low-level operator tasks for Nomad. +type Operator struct { + srv *Server +} + +// RaftGetConfiguration is used to retrieve the current Raft configuration. +func (op *Operator) RaftGetConfiguration(args *structs.GenericRequest, reply *structs.RaftConfigurationResponse) error { + if done, err := op.srv.forward("Operator.RaftGetConfiguration", args, args, reply); done { + return err + } + + // We can't fetch the leader and the configuration atomically with + // the current Raft API. + future := op.srv.raft.GetConfiguration() + if err := future.Error(); err != nil { + return err + } + + // Index the Nomad information about the servers. + serverMap := make(map[raft.ServerAddress]serf.Member) + for _, member := range op.srv.serf.Members() { + valid, parts := isNomadServer(member) + if !valid { + continue + } + + addr := (&net.TCPAddr{IP: member.Addr, Port: parts.Port}).String() + serverMap[raft.ServerAddress(addr)] = member + } + + // Fill out the reply. + leader := op.srv.raft.Leader() + reply.Index = future.Index() + for _, server := range future.Configuration().Servers { + node := "(unknown)" + if member, ok := serverMap[server.Address]; ok { + node = member.Name + } + + entry := &structs.RaftServer{ + ID: server.ID, + Node: node, + Address: server.Address, + Leader: server.Address == leader, + Voter: server.Suffrage == raft.Voter, + } + reply.Servers = append(reply.Servers, entry) + } + return nil +} + +// RaftRemovePeerByAddress is used to kick a stale peer (one that it in the Raft +// quorum but no longer known to Serf or the catalog) by address in the form of +// "IP:port". The reply argument is not used, but it required to fulfill the RPC +// interface. +func (op *Operator) RaftRemovePeerByAddress(args *structs.RaftPeerByAddressRequest, reply *struct{}) error { + if done, err := op.srv.forward("Operator.RaftRemovePeerByAddress", args, args, reply); done { + return err + } + + // Since this is an operation designed for humans to use, we will return + // an error if the supplied address isn't among the peers since it's + // likely they screwed up. + { + future := op.srv.raft.GetConfiguration() + if err := future.Error(); err != nil { + return err + } + for _, s := range future.Configuration().Servers { + if s.Address == args.Address { + goto REMOVE + } + } + return fmt.Errorf("address %q was not found in the Raft configuration", + args.Address) + } + +REMOVE: + // The Raft library itself will prevent various forms of foot-shooting, + // like making a configuration with no voters. Some consideration was + // given here to adding more checks, but it was decided to make this as + // low-level and direct as possible. We've got ACL coverage to lock this + // down, and if you are an operator, it's assumed you know what you are + // doing if you are calling this. If you remove a peer that's known to + // Serf, for example, it will come back when the leader does a reconcile + // pass. + future := op.srv.raft.RemovePeer(args.Address) + if err := future.Error(); err != nil { + op.srv.logger.Printf("[WARN] nomad.operator: Failed to remove Raft peer %q: %v", + args.Address, err) + return err + } + + op.srv.logger.Printf("[WARN] nomad.operator: Removed Raft peer %q", args.Address) + return nil +} diff --git a/nomad/operator_endpoint_test.go b/nomad/operator_endpoint_test.go new file mode 100644 index 00000000000..c5ab5a611ee --- /dev/null +++ b/nomad/operator_endpoint_test.go @@ -0,0 +1,109 @@ +package nomad + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/hashicorp/raft" +) + +func TestOperator_RaftGetConfiguration(t *testing.T) { + s1 := testServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + arg := structs.GenericRequest{ + QueryOptions: structs.QueryOptions{ + Region: s1.config.Region, + }, + } + var reply structs.RaftConfigurationResponse + if err := msgpackrpc.CallWithCodec(codec, "Operator.RaftGetConfiguration", &arg, &reply); err != nil { + t.Fatalf("err: %v", err) + } + + future := s1.raft.GetConfiguration() + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + if len(future.Configuration().Servers) != 1 { + t.Fatalf("bad: %v", future.Configuration().Servers) + } + me := future.Configuration().Servers[0] + expected := structs.RaftConfigurationResponse{ + Servers: []*structs.RaftServer{ + &structs.RaftServer{ + ID: me.ID, + Node: fmt.Sprintf("%v.%v", s1.config.NodeName, s1.config.Region), + Address: me.Address, + Leader: true, + Voter: true, + }, + }, + Index: future.Index(), + } + if !reflect.DeepEqual(reply, expected) { + t.Fatalf("bad: got %+v; want %+v", reply, expected) + } +} + +func TestOperator_RaftRemovePeerByAddress(t *testing.T) { + s1 := testServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Try to remove a peer that's not there. + arg := structs.RaftPeerByAddressRequest{ + Address: raft.ServerAddress(fmt.Sprintf("127.0.0.1:%d", getPort())), + } + arg.Region = s1.config.Region + var reply struct{} + err := msgpackrpc.CallWithCodec(codec, "Operator.RaftRemovePeerByAddress", &arg, &reply) + if err == nil || !strings.Contains(err.Error(), "not found in the Raft configuration") { + t.Fatalf("err: %v", err) + } + + // Add it manually to Raft. + { + future := s1.raft.AddPeer(arg.Address) + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + } + + // Make sure it's there. + { + future := s1.raft.GetConfiguration() + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + configuration := future.Configuration() + if len(configuration.Servers) != 2 { + t.Fatalf("bad: %v", configuration) + } + } + + // Remove it, now it should go through. + if err := msgpackrpc.CallWithCodec(codec, "Operator.RaftRemovePeerByAddress", &arg, &reply); err != nil { + t.Fatalf("err: %v", err) + } + + // Make sure it's not there. + { + future := s1.raft.GetConfiguration() + if err := future.Error(); err != nil { + t.Fatalf("err: %v", err) + } + configuration := future.Configuration() + if len(configuration.Servers) != 1 { + t.Fatalf("bad: %v", configuration) + } + } +} diff --git a/nomad/server.go b/nomad/server.go index e3cae7278f3..3d1446f33b1 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -162,6 +162,7 @@ type endpoints struct { Region *Region Periodic *Periodic System *System + Operator *Operator } // NewServer is used to construct a new Nomad server from the @@ -639,25 +640,27 @@ func (s *Server) setupVaultClient() error { // setupRPC is used to setup the RPC listener func (s *Server) setupRPC(tlsWrap tlsutil.RegionWrapper) error { // Create endpoints - s.endpoints.Status = &Status{s} - s.endpoints.Node = &Node{srv: s} - s.endpoints.Job = &Job{s} + s.endpoints.Alloc = &Alloc{s} s.endpoints.Eval = &Eval{s} + s.endpoints.Job = &Job{s} + s.endpoints.Node = &Node{srv: s} + s.endpoints.Operator = &Operator{s} + s.endpoints.Periodic = &Periodic{s} s.endpoints.Plan = &Plan{s} - s.endpoints.Alloc = &Alloc{s} s.endpoints.Region = &Region{s} - s.endpoints.Periodic = &Periodic{s} + s.endpoints.Status = &Status{s} s.endpoints.System = &System{s} // Register the handlers - s.rpcServer.Register(s.endpoints.Status) - s.rpcServer.Register(s.endpoints.Node) - s.rpcServer.Register(s.endpoints.Job) + s.rpcServer.Register(s.endpoints.Alloc) s.rpcServer.Register(s.endpoints.Eval) + s.rpcServer.Register(s.endpoints.Job) + s.rpcServer.Register(s.endpoints.Node) + s.rpcServer.Register(s.endpoints.Operator) + s.rpcServer.Register(s.endpoints.Periodic) s.rpcServer.Register(s.endpoints.Plan) - s.rpcServer.Register(s.endpoints.Alloc) s.rpcServer.Register(s.endpoints.Region) - s.rpcServer.Register(s.endpoints.Periodic) + s.rpcServer.Register(s.endpoints.Status) s.rpcServer.Register(s.endpoints.System) list, err := net.ListenTCP("tcp", s.config.RPCAddr) @@ -1055,7 +1058,6 @@ func (s *Server) GetConfig() *Config { return s.config } -// TODO(alex) we need a outage guide // peersInfoContent is used to help operators understand what happened to the // peers.json file. This is written to a file called peers.info in the same // location. @@ -1079,5 +1081,5 @@ creating the peers.json file, and that all servers receive the same configuration. Once the peers.json file is successfully ingested and applied, it will be deleted. -Please see https://www.consul.io/docs/guides/outage.html for more information. +Please see https://www.nomadproject.io/guides/outage.html for more information. ` diff --git a/nomad/structs/operator.go b/nomad/structs/operator.go new file mode 100644 index 00000000000..93b99f6fb7f --- /dev/null +++ b/nomad/structs/operator.go @@ -0,0 +1,49 @@ +package structs + +import ( + "github.com/hashicorp/raft" +) + +// RaftServer has information about a server in the Raft configuration. +type RaftServer struct { + // ID is the unique ID for the server. These are currently the same + // as the address, but they will be changed to a real GUID in a future + // release of Nomad. + ID raft.ServerID + + // Node is the node name of the server, as known by Nomad, or this + // will be set to "(unknown)" otherwise. + Node string + + // Address is the IP:port of the server, used for Raft communications. + Address raft.ServerAddress + + // Leader is true if this server is the current cluster leader. + Leader bool + + // Voter is true if this server has a vote in the cluster. This might + // be false if the server is staging and still coming online, or if + // it's a non-voting server, which will be added in a future release of + // Nomad. + Voter bool +} + +// RaftConfigrationResponse is returned when querying for the current Raft +// configuration. +type RaftConfigurationResponse struct { + // Servers has the list of servers in the Raft configuration. + Servers []*RaftServer + + // Index has the Raft index of this configuration. + Index uint64 +} + +// RaftPeerByAddressRequest is used by the Operator endpoint to apply a Raft +// operation on a specific Raft peer by address in the form of "IP:port". +type RaftPeerByAddressRequest struct { + // Address is the peer to remove, in the form "IP:port". + Address raft.ServerAddress + + // WriteRequest holds the Region for this request. + WriteRequest +} diff --git a/website/source/assets/stylesheets/_global.scss b/website/source/assets/stylesheets/_global.scss index e03d118aaa2..a7ff8e342bd 100755 --- a/website/source/assets/stylesheets/_global.scss +++ b/website/source/assets/stylesheets/_global.scss @@ -83,8 +83,15 @@ pre { } //fixed grid below 992 to prevent smaller responsive sizes -@media (max-width: 992px) { +@media (min-width: 768px) and (max-width: 992px) { .container{ - max-width: 970px; + width: 100%; + } +} + +//guarantees nav list fits at tablet viewport size +@media (min-width: 768px) and (max-width: 800px) { + #header .navbar-nav a { + font-size: 12px !important; } } diff --git a/website/source/docs/commands/operator-index.html.md.erb b/website/source/docs/commands/operator-index.html.md.erb new file mode 100644 index 00000000000..1eef2bf3f9d --- /dev/null +++ b/website/source/docs/commands/operator-index.html.md.erb @@ -0,0 +1,31 @@ +--- +layout: "docs" +page_title: "Commands: operator" +sidebar_current: "docs-commands-operator" +description: > + The operator command provides cluster-level tools for Nomad operators. +--- + +# Nomad Operator + +Command: `nomad operator` + +The `operator` command provides cluster-level tools for Nomad operators, such +as interacting with the Raft subsystem. This was added in Nomad 0.5.5. + +~> Use this command with extreme caution, as improper use could lead to a Nomad +outage and even loss of data. + +See the [Outage Recovery](/guides/outage.html) guide for some examples of how +this command is used. For an API to perform these operations programatically, +please see the documentation for the [Operator](/guides/outage.html) +endpoint. + +## Usage + +Usage: `nomad operator [options]` + +Run `nomad operator ` with no arguments for help on that subcommand. +The following subcommands are available: + +* `raft` - View and modify Nomad's Raft configuration. diff --git a/website/source/docs/commands/operator-raft-list-peers.html.md.erb b/website/source/docs/commands/operator-raft-list-peers.html.md.erb new file mode 100644 index 00000000000..ab2e405d217 --- /dev/null +++ b/website/source/docs/commands/operator-raft-list-peers.html.md.erb @@ -0,0 +1,61 @@ +--- +layout: "docs" +page_title: "Commands: operator raft list-peers" +sidebar_current: "docs-commands-operator-raft-list-peers" +description: > + Display the current Raft peer configuration. +--- + +# Command: `operator raft list-peers` + +The Raft list-peers command is used to display the current Raft peer +configuration. + +See the [Outage Recovery](/guides/outage.html) guide for some examples of how +this command is used. For an API to perform these operations programatically, +please see the documentation for the [Operator](/docs/http/operator.html) +endpoint. + +## Usage + +``` +nomad operator raft list-peers [options] +``` + +## General Options + +<%= partial "docs/commands/_general_options" %> + +## List Peers Options + +* `-stale`: The stale argument defaults to "false" which means the leader +provides the result. If the cluster is in an outage state without a leader, you +may need to set `-stale` to "true" to get the configuration from a non-leader +server. + +## Examples + +An example output with three servers is as follows: + +``` +$ nomad operator raft list-peers +Node ID Address State Voter +nomad-server01.global 10.10.11.5:4647 10.10.11.5:4647 follower true +nomad-server02.global 10.10.11.6:4647 10.10.11.6:4647 leader true +nomad-server03.global 10.10.11.7:4647 10.10.11.7:4647 follower true +``` + +- `Node` is the node name of the server, as known to Nomad, or "(unknown)" if +the node is stale and not known. + +- `ID` is the ID of the server. This is the same as the `Address` but may be +upgraded to a GUID in a future version of Nomad. + +- `Address` is the IP:port for the server. + +- `State` is either "follower" or "leader" depending on the server's role in the +Raft configuration. + +- `Voter` is "true" or "false", indicating if the server has a vote in the Raft +configuration. Future versions of Nomad may add support for non-voting servers. + diff --git a/website/source/docs/commands/operator-raft-remove-peer.html.md.erb b/website/source/docs/commands/operator-raft-remove-peer.html.md.erb new file mode 100644 index 00000000000..fae613d2bfe --- /dev/null +++ b/website/source/docs/commands/operator-raft-remove-peer.html.md.erb @@ -0,0 +1,41 @@ +--- +layout: "docs" +page_title: "Commands: operator raft remove-peer" +sidebar_current: "docs-commands-operator-raft-remove-peer" +description: > + Remove a Nomad server from the Raft configuration. +--- + +# Command: `operator raft remove-peer` + +Remove the Nomad server with given address from the Raft configuration. + +There are rare cases where a peer may be left behind in the Raft quorum even +though the server is no longer present and known to the cluster. This command +can be used to remove the failed server so that it is no longer affects the Raft +quorum. If the server still shows in the output of the [`nomad +server-members`](/docs/commands/server-members.html) command, it is preferable +to clean up by simply running [`nomad +server-force-leave`](/docs/commands/server-force-leave.html) instead of this +command. + +See the [Outage Recovery](/guides/outage.html) guide for some examples of how +this command is used. For an API to perform these operations programatically, +please see the documentation for the [Operator](/docs/http/operator.html) +endpoint. + +## Usage + +``` +nomad operator raft remove-peer [options] +``` + +## General Options + +<%= partial "docs/commands/_general_options" %> + +## Remove Peer Options + +* `-peer-address`: Remove a Nomad server with given address from the Raft +configuration. The format is "IP:port" + diff --git a/website/source/docs/http/operator.html.md b/website/source/docs/http/operator.html.md new file mode 100644 index 00000000000..43f8ba6e30a --- /dev/null +++ b/website/source/docs/http/operator.html.md @@ -0,0 +1,166 @@ +--- +layout: "http" +page_title: "HTTP API: /v1/operator/" +sidebar_current: "docs-http-operator" +description: > + The '/v1/operator/' endpoints provides cluster-level tools for Nomad + operators. +--- + +# /v1/operator + +The Operator endpoint provides cluster-level tools for Nomad operators, such +as interacting with the Raft subsystem. This was added in Nomad 0.5.5 + +~> Use this interface with extreme caution, as improper use could lead to a + Nomad outage and even loss of data. + +See the [Outage Recovery](/guides/outage.html) guide for some examples of how +these capabilities are used. For a CLI to perform these operations manually, please +see the documentation for the [`nomad operator`](/docs/commands/operator-index.html) +command. + +By default, the agent's local region is used; another region can be specified +using the `?region=` query parameter. + +## GET + +
+
Description
+
+ Query the status of a client node registered with Nomad. +
+ +
Method
+
GET
+ +
URL
+
`/v1/operator/raft/configuration`
+ +
Parameters
+
+
    +
  • + stale + optional + If the cluster doesn't currently have a leader an error will be + returned. You can use the `?stale` query parameter to read the Raft + configuration from any of the Nomad servers. +
    • +
+
+ +
Returns
+
+ + ```javascript +{ + "Servers": [ + { + "ID": "127.0.0.1:4647", + "Node": "alice", + "Address": "127.0.0.1:4647", + "Leader": true, + "Voter": true + }, + { + "ID": "127.0.0.2:4647", + "Node": "bob", + "Address": "127.0.0.2:4647", + "Leader": false, + "Voter": true + }, + { + "ID": "127.0.0.3:4647", + "Node": "carol", + "Address": "127.0.0.3:4647", + "Leader": false, + "Voter": true + } + ], + "Index": 22 +} + ``` + +
+ +
Field Reference
+
+ +
    +
  • + Servers + The returned `Servers` array has information about the servers in the Raft + peer configuration. See the `Server` block for a description of its fields: +
    • +
  • + Index + The `Index` value is the Raft corresponding to this configuration. The + latest configuration may not yet be committed if changes are in flight. +
    • +
+ + `Server` Fields: +
    +
  • + ID + `ID` is the ID of the server. This is the same as the `Address` but may + be upgraded to a GUID in a future version of Nomad. +
    • +
  • + Node + `Node` is the node name of the server, as known to Nomad, or "(unknown)" if + the node is stale and not known. +
    • +
  • + Address + `Address` is the IP:port for the server. +
    • +
  • + Leader + `Leader` is either "true" or "false" depending on the server's role in the + Raft configuration. +
    • +
  • + Voter + `Voter` is "true" or "false", indicating if the server has a vote in the Raft + configuration. Future versions of Nomad may add support for non-voting servers. +
    • +
+ +
+
+ + +## DELETE + +
+
Description
+
+ Remove the Nomad server with given address from the Raft configuration. The + return code signifies success or failure. +
+ +
Method
+
DELETE
+ +
URL
+
`/v1/operator/raft/peer`
+ +
Parameters
+
+
    +
  • + address + required + The address specifies the server to remove and is given as an `IP:port`. + The port number is usually 4647, unless configured otherwise. Nothing is + required in the body of the request. +
    • +
+
+ +
Returns
+
None
+ +
diff --git a/website/source/docs/cluster/automatic.html.md b/website/source/guides/cluster/automatic.html.md similarity index 98% rename from website/source/docs/cluster/automatic.html.md rename to website/source/guides/cluster/automatic.html.md index 442e35337db..8263d4324ca 100644 --- a/website/source/docs/cluster/automatic.html.md +++ b/website/source/guides/cluster/automatic.html.md @@ -1,7 +1,7 @@ --- -layout: "docs" +layout: "guides" page_title: "Automatically Bootstrapping a Nomad Cluster" -sidebar_current: "docs-cluster-automatic" +sidebar_current: "guides-cluster-automatic" description: |- Learn how to automatically bootstrap a Nomad cluster using Consul. By having a Consul agent installed on each host, Nomad can automatically discover other diff --git a/website/source/docs/cluster/bootstrapping.html.md b/website/source/guides/cluster/bootstrapping.html.md similarity index 79% rename from website/source/docs/cluster/bootstrapping.html.md rename to website/source/guides/cluster/bootstrapping.html.md index d94aab68f9b..53d6541a52f 100644 --- a/website/source/docs/cluster/bootstrapping.html.md +++ b/website/source/guides/cluster/bootstrapping.html.md @@ -1,7 +1,7 @@ --- -layout: "docs" +layout: "guides" page_title: "Bootstrapping a Nomad Cluster" -sidebar_current: "docs-cluster-bootstrap" +sidebar_current: "guides-cluster-bootstrap" description: |- Learn how to bootstrap a Nomad cluster. --- @@ -17,8 +17,8 @@ page](/docs/internals/architecture.html). There are two strategies for bootstrapping a Nomad cluster: -1. Automatic bootstrapping -1. Manual bootstrapping +1. Automatic bootstrapping +1. Manual bootstrapping Please refer to the specific documentation links above or in the sidebar for more detailed information about each strategy. diff --git a/website/source/docs/cluster/federation.md b/website/source/guides/cluster/federation.md similarity index 94% rename from website/source/docs/cluster/federation.md rename to website/source/guides/cluster/federation.md index 06435f77ab9..8cf70b4250f 100644 --- a/website/source/docs/cluster/federation.md +++ b/website/source/guides/cluster/federation.md @@ -1,7 +1,7 @@ --- -layout: "docs" +layout: "guides" page_title: "Federating a Nomad Cluster" -sidebar_current: "docs-cluster-federation" +sidebar_current: "guides-cluster-federation" description: |- Learn how to join Nomad servers across multiple regions so users can submit jobs to any server in any region using global federation. diff --git a/website/source/docs/cluster/manual.html.md b/website/source/guides/cluster/manual.html.md similarity index 97% rename from website/source/docs/cluster/manual.html.md rename to website/source/guides/cluster/manual.html.md index 0b32c3ee1f2..57cb3f3df6e 100644 --- a/website/source/docs/cluster/manual.html.md +++ b/website/source/guides/cluster/manual.html.md @@ -1,7 +1,7 @@ --- -layout: "docs" +layout: "guides" page_title: "Manually Bootstrapping a Nomad Cluster" -sidebar_current: "docs-cluster-manual" +sidebar_current: "guides-cluster-manual" description: |- Learn how to manually bootstrap a Nomad cluster using the server-join command. This section also discusses Nomad federation across multiple diff --git a/website/source/docs/cluster/requirements.html.md b/website/source/guides/cluster/requirements.html.md similarity index 98% rename from website/source/docs/cluster/requirements.html.md rename to website/source/guides/cluster/requirements.html.md index 8f22143dfba..a8ea9f4a753 100644 --- a/website/source/docs/cluster/requirements.html.md +++ b/website/source/guides/cluster/requirements.html.md @@ -1,7 +1,7 @@ --- -layout: "docs" +layout: "guides" page_title: "Nomad Client and Server Requirements" -sidebar_current: "docs-cluster-requirements" +sidebar_current: "guides-cluster-requirements" description: |- Learn about Nomad client and server requirements such as memory and CPU recommendations, network topologies, and more. diff --git a/website/source/guides/index.html.markdown b/website/source/guides/index.html.markdown new file mode 100644 index 00000000000..f5e9c6187c1 --- /dev/null +++ b/website/source/guides/index.html.markdown @@ -0,0 +1,15 @@ +--- +layout: "guides" +page_title: "Guides" +sidebar_current: "guides-home" +description: |- + Welcome to the Nomad guides! The section provides various guides for common + Nomad workflows and actions. +--- + +# Nomad Guides + +Welcome to the Nomad guides! If you are just getting started with Nomad, please +start with the [Nomad introduction](/intro/index.html) instead and then continue +on to the guides. The guides provide examples for common Nomad workflows and +actions for both users and operators of Nomad. diff --git a/website/source/guides/outage.html.markdown b/website/source/guides/outage.html.markdown new file mode 100644 index 00000000000..5e8537f7444 --- /dev/null +++ b/website/source/guides/outage.html.markdown @@ -0,0 +1,188 @@ +--- +layout: "guides" +page_title: "Outage Recovery" +sidebar_current: "guides-outage-recovery" +description: |- + Don't panic! This is a critical first step. Depending on your deployment + configuration, it may take only a single server failure for cluster + unavailability. Recovery requires an operator to intervene, but recovery is + straightforward. +--- + +# Outage Recovery + +Don't panic! This is a critical first step. + +Depending on your +[deployment configuration](/docs/internals/consensus.html#deployment_table), it +may take only a single server failure for cluster unavailability. Recovery +requires an operator to intervene, but the process is straightforward. + +~> This guide is for recovery from a Nomad outage due to a majority of server +nodes in a datacenter being lost. If you are looking to add or remove servers, +see the [bootstrapping guide](/guides/cluster/bootstrapping.html). + +## Failure of a Single Server Cluster + +If you had only a single server and it has failed, simply restart it. A +single server configuration requires the +[`-bootstrap-expect=1`](/docs/agent/configuration/server.html#bootstrap_expect) +flag. If the server cannot be recovered, you need to bring up a new +server. See the [bootstrapping guide](/guides/cluster/bootstrapping.html) +for more detail. + +In the case of an unrecoverable server failure in a single server cluster, data +loss is inevitable since data was not replicated to any other servers. This is +why a single server deploy is **never** recommended. + +## Failure of a Server in a Multi-Server Cluster + +If you think the failed server is recoverable, the easiest option is to bring +it back online and have it rejoin the cluster with the same IP address, returning +the cluster to a fully healthy state. Similarly, even if you need to rebuild a +new Nomad server to replace the failed node, you may wish to do that immediately. +Keep in mind that the rebuilt server needs to have the same IP address as the failed +server. Again, once this server is online and has rejoined, the cluster will return +to a fully healthy state. + +Both of these strategies involve a potentially lengthy time to reboot or rebuild +a failed server. If this is impractical or if building a new server with the same +IP isn't an option, you need to remove the failed server. Usually, you can issue +a [`nomad server-force-leave`](/docs/commands/server-force-leave.html) command +to remove the failed server if it's still a member of the cluster. + +If [`nomad server-force-leave`](/docs/commands/server-force-leave.html) isn't +able to remove the server, you have two methods available to remove it, +depending on your version of Nomad: + +* In Nomad 0.5.5 and later, you can use the [`nomad operator raft + remove-peer`](/docs/commands/operator-raft-remove-peer.html) command to remove + the stale peer server on the fly with no downtime. + +* In versions of Nomad prior to 0.5.5, you can manually remove the stale peer + server using the `raft/peers.json` recovery file on all remaining servers. See + the [section below](#manual-recovery-using-peers-json) for details on this + procedure. This process requires Nomad downtime to complete. + +In Nomad 0.5.5 and later, you can use the [`nomad operator raft +list-peers`](/docs/commands/operator-raft-list-peers.html) command to inspect +the Raft configuration: + +``` +$ nomad operator raft list-peers +Node ID Address State Voter +nomad-server01.global 10.10.11.5:4647 10.10.11.5:4647 follower true +nomad-server02.global 10.10.11.6:4647 10.10.11.6:4647 leader true +nomad-server03.global 10.10.11.7:4647 10.10.11.7:4647 follower true +``` + +## Failure of Multiple Servers in a Multi-Server Cluster + +In the event that multiple servers are lost, causing a loss of quorum and a +complete outage, partial recovery is possible using data on the remaining +servers in the cluster. There may be data loss in this situation because multiple +servers were lost, so information about what's committed could be incomplete. +The recovery process implicitly commits all outstanding Raft log entries, so +it's also possible to commit data that was uncommitted before the failure. + +See the [section below](#manual-recovery-using-peers-json) for details of the +recovery procedure. You simply include just the remaining servers in the +`raft/peers.json` recovery file. The cluster should be able to elect a leader +once the remaining servers are all restarted with an identical `raft/peers.json` +configuration. + +Any new servers you introduce later can be fresh with totally clean data directories +and joined using Nomad's `server-join` command. + +In extreme cases, it should be possible to recover with just a single remaining +server by starting that single server with itself as the only peer in the +`raft/peers.json` recovery file. + +Prior to Nomad 0.5.5 it wasn't always possible to recover from certain +types of outages with `raft/peers.json` because this was ingested before any Raft +log entries were played back. In Nomad 0.5.5 and later, the `raft/peers.json` +recovery file is final, and a snapshot is taken after it is ingested, so you are +guaranteed to start with your recovered configuration. This does implicitly commit +all Raft log entries, so should only be used to recover from an outage, but it +should allow recovery from any situation where there's some cluster data available. + +## Manual Recovery Using peers.json + +To begin, stop all remaining servers. You can attempt a graceful leave, +but it will not work in most cases. Do not worry if the leave exits with an +error. The cluster is in an unhealthy state, so this is expected. + +In Nomad 0.5.5 and later, the `peers.json` file is no longer present +by default and is only used when performing recovery. This file will be deleted +after Nomad starts and ingests this file. Nomad 0.5.5 also uses a new, automatically- +created `raft/peers.info` file to avoid ingesting the `raft/peers.json` file on the +first start after upgrading. Be sure to leave `raft/peers.info` in place for proper +operation. + +Using `raft/peers.json` for recovery can cause uncommitted Raft log entries to be +implicitly committed, so this should only be used after an outage where no +other option is available to recover a lost server. Make sure you don't have +any automated processes that will put the peers file in place on a +periodic basis. + +The next step is to go to the +[`-data-dir`](/docs/agent/configuration/index.html#data_dir) of each Nomad +server. Inside that directory, there will be a `raft/` sub-directory. We need to +create a `raft/peers.json` file. It should look something like: + +```javascript +[ + "10.0.1.8:4647", + "10.0.1.6:4647", + "10.0.1.7:4647" +] +``` + +Simply create entries for all remaining servers. You must confirm +that servers you do not include here have indeed failed and will not later +rejoin the cluster. Ensure that this file is the same across all remaining +server nodes. + +At this point, you can restart all the remaining servers. In Nomad 0.5.5 and +later you will see them ingest recovery file: + +```text +... +2016/08/16 14:39:20 [INFO] nomad: found peers.json file, recovering Raft configuration... +2016/08/16 14:39:20 [INFO] nomad.fsm: snapshot created in 12.484µs +2016/08/16 14:39:20 [INFO] snapshot: Creating new snapshot at /tmp/peers/raft/snapshots/2-5-1471383560779.tmp +2016/08/16 14:39:20 [INFO] nomad: deleted peers.json file after successful recovery +2016/08/16 14:39:20 [INFO] raft: Restored from snapshot 2-5-1471383560779 +2016/08/16 14:39:20 [INFO] raft: Initial configuration (index=1): [{Suffrage:Voter ID:10.212.15.121:4647 Address:10.212.15.121:4647}] +... +``` + +If any servers managed to perform a graceful leave, you may need to have them +rejoin the cluster using the [`server-join`](/docs/commands/server-join.html) command: + +```text +$ nomad server-join +Successfully joined cluster by contacting 1 nodes. +``` + +It should be noted that any existing member can be used to rejoin the cluster +as the gossip protocol will take care of discovering the server nodes. + +At this point, the cluster should be in an operable state again. One of the +nodes should claim leadership and emit a log like: + +```text +[INFO] nomad: cluster leadership acquired +``` + +In Nomad 0.5.5 and later, you can use the [`nomad operator raft +list-peers`](/docs/commands/operator-raft-list-peers.html) command to inspect +the Raft configuration: + +``` +$ nomad operator raft list-peers +Node ID Address State Voter +nomad-server01.global 10.10.11.5:4647 10.10.11.5:4647 follower true +nomad-server02.global 10.10.11.6:4647 10.10.11.6:4647 leader true +nomad-server03.global 10.10.11.7:4647 10.10.11.7:4647 follower true +``` diff --git a/website/source/layouts/_header.erb b/website/source/layouts/_header.erb index e8e27e91f88..9fab78fd43f 100644 --- a/website/source/layouts/_header.erb +++ b/website/source/layouts/_header.erb @@ -27,6 +27,7 @@ diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 48fb8ebddad..9c62e22b393 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -228,6 +228,17 @@ > node-status + > + operator + + > plan diff --git a/website/source/layouts/guides.erb b/website/source/layouts/guides.erb new file mode 100644 index 00000000000..e18758d1053 --- /dev/null +++ b/website/source/layouts/guides.erb @@ -0,0 +1,31 @@ +<% wrap_layout :inner do %> + <% content_for :sidebar do %> + + <% end %> + + <%= yield %> +<% end %> diff --git a/website/source/layouts/http.erb b/website/source/layouts/http.erb index 5fb7d7dc80b..6e6105e7f2f 100644 --- a/website/source/layouts/http.erb +++ b/website/source/layouts/http.erb @@ -117,6 +117,10 @@ Status + > + Operator + + > System