diff --git a/command/node_drain.go b/command/node_drain.go index 0911a68d8be..caa34bf0700 100644 --- a/command/node_drain.go +++ b/command/node_drain.go @@ -75,10 +75,10 @@ Node Drain Options: -m Message for the drain update operation. Registered in drain metadata as - "message" for drain enable and "cancel_message" for drain disable. + "message" during drain enable and "cancel_message" during drain disable. -meta = - Custom metadata to store on thed drain operation, can be used multiple times. + Custom metadata to store on the drain operation, can be used multiple times. -self Set the drain status of the local node. diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index a35515735eb..5d5c323ccba 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -870,6 +870,10 @@ func TestClientEndpoint_UpdateStatus_HeartbeatOnly_Advertise(t *testing.T) { require.Equal(resp.Servers[0].RPCAdvertiseAddr, advAddr) } +// TestClientEndpoint_UpdateDrain asserts the ability to initiate drain +// against a node and cancel that drain. It also asserts: +// * an evaluation is created when the node becomes eligible +// * drain metadata is properly persisted in Node.LastDrain func TestClientEndpoint_UpdateDrain(t *testing.T) { t.Parallel() require := require.New(t) @@ -984,6 +988,9 @@ func TestClientEndpoint_UpdateDrain(t *testing.T) { require.Len(out.Events, 4) } +// TestClientEndpoint_UpdatedDrainAndCompleted asserts that drain metadata +// is properly persisted in Node.LastDrain as the node drain is updated and +// completes. func TestClientEndpoint_UpdatedDrainAndCompleted(t *testing.T) { t.Parallel() require := require.New(t) @@ -1019,7 +1026,7 @@ func TestClientEndpoint_UpdatedDrainAndCompleted(t *testing.T) { NodeID: node.ID, DrainStrategy: strategy, Meta: map[string]string{ - "message": "first", + "message": "first drain", }, WriteRequest: structs.WriteRequest{Region: "global"}, } @@ -1037,14 +1044,14 @@ func TestClientEndpoint_UpdatedDrainAndCompleted(t *testing.T) { StartedAt: firstDrainUpdate, UpdatedAt: firstDrainUpdate, Status: structs.DrainStatusDraining, - Meta: map[string]string{"message": "first"}, + Meta: map[string]string{"message": "first drain"}, }, *out.LastDrain) time.Sleep(1 * time.Second) // Update the drain dereg.DrainStrategy.DrainSpec.Deadline *= 2 - dereg.Meta["message"] = "second" + dereg.Meta["message"] = "second drain" require.Nil(msgpackrpc.CallWithCodec(codec, "Node.UpdateDrain", dereg, &resp2)) require.NotZero(resp2.Index) @@ -1058,7 +1065,7 @@ func TestClientEndpoint_UpdatedDrainAndCompleted(t *testing.T) { StartedAt: firstDrainUpdate, UpdatedAt: secondDrainUpdate, Status: structs.DrainStatusDraining, - Meta: map[string]string{"message": "second"}, + Meta: map[string]string{"message": "second drain"}, }, *out.LastDrain) time.Sleep(1 * time.Second) @@ -1084,10 +1091,13 @@ func TestClientEndpoint_UpdatedDrainAndCompleted(t *testing.T) { StartedAt: firstDrainUpdate, UpdatedAt: out.LastDrain.UpdatedAt, Status: structs.DrainStatusComplete, - Meta: map[string]string{"message": "second"}, + Meta: map[string]string{"message": "second drain"}, }, *out.LastDrain) } +// TestClientEndpoint_UpdatedDrainNoop asserts that drain metadata is properly +// persisted in Node.LastDrain when calls to Node.UpdateDrain() don't affect +// the drain status. func TestClientEndpoint_UpdatedDrainNoop(t *testing.T) { t.Parallel() require := require.New(t) @@ -1160,6 +1170,9 @@ func TestClientEndpoint_UpdatedDrainNoop(t *testing.T) { require.Equal(prevDrain, out.LastDrain) } +// TestClientEndpoint_UpdateDrain_ACL asserts that Node.UpdateDrain() enforces +// node.write ACLs, and that token accessor ID is properly persisted in +// Node.LastDrain.AccessorID func TestClientEndpoint_UpdateDrain_ACL(t *testing.T) { t.Parallel() diff --git a/website/content/api-docs/nodes.mdx b/website/content/api-docs/nodes.mdx index 74656512f35..a01b1a0b868 100644 --- a/website/content/api-docs/nodes.mdx +++ b/website/content/api-docs/nodes.mdx @@ -112,6 +112,7 @@ $ curl \ } }, "ID": "f7476465-4d6e-c0de-26d0-e383c49be941", + "LastDrain": null, "ModifyIndex": 2526, "Name": "nomad-4", "NodeClass": "", @@ -180,7 +181,7 @@ $ curl \ "memory.totalbytes": "16571674624", "nomad.advertise.address": "127.0.0.1:4646", "nomad.revision": "30da2b8f6c3aa860113c9d313c695a05eff5bb97+CHANGES", - "nomad.version": "0.10.0-dev", + "nomad.version": "1.1.0", "os.name": "nixos", "os.signals": "SIGTTOU,SIGTTIN,SIGSTOP,SIGSYS,SIGXCPU,SIGBUS,SIGKILL,SIGTERM,SIGIOT,SIGILL,SIGIO,SIGQUIT,SIGSEGV,SIGUSR1,SIGXFSZ,SIGCHLD,SIGUSR2,SIGURG,SIGFPE,SIGHUP,SIGINT,SIGPROF,SIGCONT,SIGALRM,SIGPIPE,SIGTRAP,SIGTSTP,SIGWINCH,SIGABRT", "os.version": "\"19.03.173017.85f820d6e41 (Koi)\"", @@ -261,11 +262,25 @@ $ curl \ }, "Events": [ { - "CreateIndex": 0, + "CreateIndex": 6, "Details": null, "Message": "Node registered", "Subsystem": "Cluster", - "Timestamp": "2019-08-26T12:22:50+02:00" + "Timestamp": "2021-03-31T12:11:39Z" + }, + { + "CreateIndex": 11, + "Details": null, + "Message": "Node drain strategy set", + "Subsystem": "Drain", + "Timestamp": "2021-03-31T12:12:20.213412Z" + }, + { + "CreateIndex": 12, + "Details": null, + "Message": "Node drain complete", + "Subsystem": "Drain", + "Timestamp": "2021-03-31T12:12:20.213639Z" } ], "HTTPAddr": "127.0.0.1:4646", @@ -282,6 +297,15 @@ $ curl \ } }, "ID": "1ac61e33-a465-2ace-f63f-cffa1285e7eb", + "LastDrain": { + "AccessorID": "4e1b7ce1-f8aa-d7ff-09f1-55c3a0fd3988", + "Meta": { + "message": "node maintenance" + }, + "StartedAt": "2021-03-31T12:12:20Z", + "Status": "complete", + "UpdatedAt": "2021-03-31T12:12:20Z" + }, "Links": { "consul": "dc1.mew" }, @@ -289,7 +313,7 @@ $ curl \ "connect.log_level": "info", "connect.sidecar_image": "envoyproxy/envoy:v1.11.1" }, - "ModifyIndex": 9, + "ModifyIndex": 14, "Name": "mew", "NodeClass": "", "NodeResources": { @@ -929,6 +953,9 @@ The table below shows this endpoint's support for - `MarkEligible` `(bool: false)` - Specifies whether to mark a node as eligible for scheduling again when _disabling_ a drain. +- `Meta` `(json: )` - A JSON map of strings with drain operation + metadata that will be persisted in `.LastDrain.Meta`. + ### Sample Payload ```json @@ -936,6 +963,9 @@ The table below shows this endpoint's support for "DrainSpec": { "Deadline": 3600000000000, "IgnoreSystemJobs": true + }, + "Meta": { + "message": "drain for maintenance" } } ``` diff --git a/website/content/docs/commands/node/drain.mdx b/website/content/docs/commands/node/drain.mdx index 7eac8ae9cd5..171304bed2e 100644 --- a/website/content/docs/commands/node/drain.mdx +++ b/website/content/docs/commands/node/drain.mdx @@ -79,6 +79,12 @@ capability. existing drain is being cancelled but additional scheduling on the node is not desired. +- `-m`: Message for the drain update operation. Registered in drain metadata as + `"message"` during drain enable and `"cancel_message"` during drain disable. + +- `-meta =`: Custom metadata to store on the drain operation, can be + used multiple times. + - `-self`: Drain the local node. - `-yes`: Automatic yes to prompts. @@ -88,7 +94,7 @@ capability. Enable drain mode on node with ID prefix "4d2ba53b": ```shell-session -$ nomad node drain -enable f4e8a9e5 +$ nomad node drain -enable f4e8a9e5 -m "node maintenance" Are you sure you want to enable drain mode for node "f4e8a9e5-30d8-3536-1e6f-cda5c869c35e"? [y/N] y 2018-03-30T23:13:16Z: Ctrl-C to stop monitoring: will not cancel the node drain 2018-03-30T23:13:16Z: Node "f4e8a9e5-30d8-3536-1e6f-cda5c869c35e" drain strategy set