From fa2cbb6f0cf9cc71e1b51ff62af25c59da0fd742 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Wed, 25 Oct 2017 23:51:53 -0700 Subject: [PATCH 1/4] Added the purge API on node endpoints --- command/agent/node_endpoint.go | 19 +++++++++ command/agent/node_endpoint_test.go | 65 +++++++++++++++++++++++++++++ nomad/node_endpoint.go | 15 ++++++- 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/command/agent/node_endpoint.go b/command/agent/node_endpoint.go index 8720503c542..5c173346c8e 100644 --- a/command/agent/node_endpoint.go +++ b/command/agent/node_endpoint.go @@ -42,6 +42,9 @@ func (s *HTTPServer) NodeSpecificRequest(resp http.ResponseWriter, req *http.Req case strings.HasSuffix(path, "/drain"): nodeName := strings.TrimSuffix(path, "/drain") return s.nodeToggleDrain(resp, req, nodeName) + case strings.HasSuffix(path, "/purge"): + nodeName := strings.TrimSuffix(path, "/purge") + return s.nodePurge(resp, req, nodeName) default: return s.nodeQuery(resp, req, path) } @@ -142,3 +145,19 @@ func (s *HTTPServer) nodeQuery(resp http.ResponseWriter, req *http.Request, } return out.Node, nil } + +func (s *HTTPServer) nodePurge(resp http.ResponseWriter, req *http.Request, nodeID string) (interface{}, error) { + if req.Method != "POST" { + return nil, CodedError(405, ErrInvalidMethod) + } + args := structs.NodeDeregisterRequest{ + NodeID: nodeID, + } + s.parseWriteRequest(req, &args.WriteRequest) + var out structs.NodeUpdateResponse + if err := s.agent.RPC("Node.Deregister", &args, &out); err != nil { + return nil, err + } + setIndex(resp, out.Index) + return out, nil +} diff --git a/command/agent/node_endpoint_test.go b/command/agent/node_endpoint_test.go index 391c10b9d90..9f8a1edad7c 100644 --- a/command/agent/node_endpoint_test.go +++ b/command/agent/node_endpoint_test.go @@ -276,6 +276,71 @@ func TestHTTP_NodeDrain(t *testing.T) { }) } +func TestHTTP_NodePurge(t *testing.T) { + t.Parallel() + httpTest(t, nil, func(s *TestAgent) { + // Create the node + node := mock.Node() + args := structs.NodeRegisterRequest{ + Node: node, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + var resp structs.NodeUpdateResponse + if err := s.Agent.RPC("Node.Register", &args, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + // Add some allocations to the node + state := s.Agent.server.State() + alloc1 := mock.Alloc() + alloc1.NodeID = node.ID + if err := state.UpsertJobSummary(999, mock.JobSummary(alloc1.JobID)); err != nil { + t.Fatal(err) + } + err := state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Make the HTTP request to purge it + req, err := http.NewRequest("POST", "/v1/node/"+node.ID+"/purge", nil) + if err != nil { + t.Fatalf("err: %v", err) + } + respW := httptest.NewRecorder() + + // Make the request + obj, err := s.Server.NodeSpecificRequest(respW, req) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Check for the index + if respW.HeaderMap.Get("X-Nomad-Index") == "" { + t.Fatalf("missing index") + } + + // Check the response + upd := obj.(structs.NodeUpdateResponse) + if len(upd.EvalIDs) == 0 { + t.Fatalf("bad: %v", upd) + } + + // Ensure that the node is not present anymore + args1 := structs.NodeSpecificRequest{ + NodeID: node.ID, + QueryOptions: structs.QueryOptions{Region: "global"}, + } + var resp1 structs.SingleNodeResponse + if err := s.Agent.RPC("Node.GetNode", &args1, &resp1); err != nil { + t.Fatalf("err: %v", err) + } + if resp1.Node != nil { + t.Fatalf("node still exists after purging: %#v", resp1.Node) + } + }) +} + func TestHTTP_NodeQuery(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index a8778216ffd..a62c68d1bc6 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -213,6 +213,20 @@ func (n *Node) Deregister(args *structs.NodeDeregisterRequest, reply *structs.No if args.NodeID == "" { return fmt.Errorf("missing node ID for client deregistration") } + // Look for the node + snap, err := n.srv.fsm.State().Snapshot() + if err != nil { + return err + } + + ws := memdb.NewWatchSet() + node, err := snap.NodeByID(ws, args.NodeID) + if err != nil { + return err + } + if node == nil { + return fmt.Errorf("node not found") + } // Commit this update via Raft _, index, err := n.srv.raftApply(structs.NodeDeregisterRequestType, args) @@ -232,7 +246,6 @@ func (n *Node) Deregister(args *structs.NodeDeregisterRequest, reply *structs.No } // Determine if there are any Vault accessors on the node - ws := memdb.NewWatchSet() accessors, err := n.srv.State().VaultAccessorsByNode(ws, args.NodeID) if err != nil { n.srv.logger.Printf("[ERR] nomad.client: looking up accessors for node %q failed: %v", args.NodeID, err) From 634099211b1b006207f26e4cb53f2d39acd14160 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Thu, 26 Oct 2017 14:12:17 -0700 Subject: [PATCH 2/4] Added ACLs to the node de-register endpoint --- nomad/node_endpoint.go | 9 ++++- nomad/node_endpoint_test.go | 65 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index a62c68d1bc6..dd15166e11a 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -209,6 +209,13 @@ func (n *Node) Deregister(args *structs.NodeDeregisterRequest, reply *structs.No } defer metrics.MeasureSince([]string{"nomad", "client", "deregister"}, time.Now()) + // Check node permissions + if aclObj, err := n.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil && !aclObj.AllowNodeWrite() { + return structs.ErrPermissionDenied + } + // Verify the arguments if args.NodeID == "" { return fmt.Errorf("missing node ID for client deregistration") @@ -246,7 +253,7 @@ func (n *Node) Deregister(args *structs.NodeDeregisterRequest, reply *structs.No } // Determine if there are any Vault accessors on the node - accessors, err := n.srv.State().VaultAccessorsByNode(ws, args.NodeID) + accessors, err := snap.VaultAccessorsByNode(ws, args.NodeID) if err != nil { n.srv.logger.Printf("[ERR] nomad.client: looking up accessors for node %q failed: %v", args.NodeID, err) return err diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 772a40e3e04..5d156f80242 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -132,6 +132,71 @@ func TestClientEndpoint_Deregister(t *testing.T) { } } +func TestClientEndpoint_Deregister_ACL(t *testing.T) { + t.Parallel() + s1, root := testACLServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the node + node := mock.Node() + node1 := mock.Node() + state := s1.fsm.State() + if err := state.UpsertNode(1, node); err != nil { + t.Fatalf("err: %v", err) + } + if err := state.UpsertNode(2, node1); err != nil { + t.Fatalf("err: %v", err) + } + + // Create the policy and tokens + validToken := mock.CreatePolicyAndToken(t, state, 1001, "test-valid", mock.NodePolicy(acl.PolicyWrite)) + invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", mock.NodePolicy(acl.PolicyRead)) + + // Deregister without any token and expect it to fail + dereg := &structs.NodeDeregisterRequest{ + NodeID: node.ID, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + var resp structs.GenericResponse + if err := msgpackrpc.CallWithCodec(codec, "Node.Deregister", dereg, &resp); err == nil { + t.Fatalf("node de-register succeeded") + } + + // Deregister with a valid token + dereg.AuthToken = validToken.SecretID + if err := msgpackrpc.CallWithCodec(codec, "Node.Deregister", dereg, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + // Check for the node in the FSM + ws := memdb.NewWatchSet() + out, err := state.NodeByID(ws, node.ID) + if err != nil { + t.Fatalf("err: %v", err) + } + if out != nil { + t.Fatalf("unexpected node") + } + + // Deregister with an invalid token. + dereg1 := &structs.NodeDeregisterRequest{ + NodeID: node1.ID, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + dereg1.AuthToken = invalidToken.SecretID + if err := msgpackrpc.CallWithCodec(codec, "Node.Deregister", dereg1, &resp); err == nil { + t.Fatalf("rpc should not have succeeded") + } + + // Try with a root token + dereg1.AuthToken = root.SecretID + if err := msgpackrpc.CallWithCodec(codec, "Node.Deregister", dereg1, &resp); err != nil { + t.Fatalf("err: %v", err) + } +} + func TestClientEndpoint_Deregister_Vault(t *testing.T) { t.Parallel() s1 := testServer(t, nil) From f362f99748c044d0b9b4254d3833f9cd81b4e4ab Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Thu, 26 Oct 2017 14:18:34 -0700 Subject: [PATCH 3/4] Added docs for the purge API --- website/source/api/nodes.html.md | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/website/source/api/nodes.html.md b/website/source/api/nodes.html.md index 7527d10971a..a414d6b0c19 100644 --- a/website/source/api/nodes.html.md +++ b/website/source/api/nodes.html.md @@ -614,3 +614,48 @@ $ curl \ "KnownLeader": false } ``` + +## Purge Node + +This endpoint purges a node from the system. Nodes can still join the cluster if +they are alive. + +| Method | Path | Produces | +| ------- | ------------------------- | -------------------------- | +| `POST` | `/v1/node/:node_id/purge` | `application/json` | + +The table below shows this endpoint's support for +[blocking queries](/api/index.html#blocking-queries) and +[required ACLs](/api/index.html#acls). + +| Blocking Queries | ACL Required | +| ---------------- | ------------------ | +| `NO` | `node:write` | + +### Parameters + +- `:node_id` `(string: )`- Specifies the UUID of the node. This must + be the full UUID, not the short 8-character one. This is specified as part of + the path. + +### Sample Request + +```text +$ curl \ + https://nomad.rocks/v1/node/fb2170a8-257d-3c64-b14d-bc06cc94e34c/purge +``` + +### Sample Response + +```json +{ + "EvalIDs": [ + "253ec083-22a7-76c9-b8b6-2bf3d4b27bfb" + ], + "EvalCreateIndex": 91, + "NodeModifyIndex": 90, + "Index": 90, + "LastContact": 0, + "KnownLeader": false +} +``` From 220afb49b31afec9d0da73dcd65d41d074a4c2ed Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Thu, 26 Oct 2017 14:19:57 -0700 Subject: [PATCH 4/4] Changed the curl command --- website/source/api/nodes.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/api/nodes.html.md b/website/source/api/nodes.html.md index a414d6b0c19..9732127ee35 100644 --- a/website/source/api/nodes.html.md +++ b/website/source/api/nodes.html.md @@ -642,7 +642,7 @@ The table below shows this endpoint's support for ```text $ curl \ - https://nomad.rocks/v1/node/fb2170a8-257d-3c64-b14d-bc06cc94e34c/purge + -XPOST https://nomad.rocks/v1/node/fb2170a8-257d-3c64-b14d-bc06cc94e34c/purge ``` ### Sample Response