diff --git a/api/nodes.go b/api/nodes.go index 837cd149e3c..4a264eb4d33 100644 --- a/api/nodes.go +++ b/api/nodes.go @@ -521,6 +521,9 @@ type DrainStrategy struct { // ForceDeadline is the deadline time for the drain after which drains will // be forced ForceDeadline time.Time + + // StartedAt is the time the drain process started + StartedAt time.Time } // DrainSpec describes a Node's drain behavior. diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index 80f97eb3eb0..c5913c27319 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -523,8 +523,10 @@ func (n *Node) UpdateDrain(args *structs.NodeUpdateDrainRequest, return fmt.Errorf("node not found") } + now := time.Now().UTC() + // Update the timestamp of when the node status was updated - args.UpdatedAt = time.Now().Unix() + args.UpdatedAt = now.Unix() // COMPAT: Remove in 0.9. Attempt to upgrade the request if it is of the old // format. @@ -536,9 +538,19 @@ func (n *Node) UpdateDrain(args *structs.NodeUpdateDrainRequest, } } - // Mark the deadline time - if args.DrainStrategy != nil && args.DrainStrategy.Deadline.Nanoseconds() > 0 { - args.DrainStrategy.ForceDeadline = time.Now().Add(args.DrainStrategy.Deadline) + // Setup drain strategy + if args.DrainStrategy != nil { + // Mark start time for the drain + if node.DrainStrategy == nil { + args.DrainStrategy.StartedAt = now + } else { + args.DrainStrategy.StartedAt = node.DrainStrategy.StartedAt + } + + // Mark the deadline time + if args.DrainStrategy.Deadline.Nanoseconds() > 0 { + args.DrainStrategy.ForceDeadline = now.Add(args.DrainStrategy.Deadline) + } } // Construct the node event diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 48157044924..7ebf36bf879 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -906,6 +906,17 @@ func TestClientEndpoint_UpdateDrain(t *testing.T) { // now+deadline should be after the forced deadline require.True(time.Now().Add(strategy.Deadline).After(out.DrainStrategy.ForceDeadline)) + drainStartedAt := out.DrainStrategy.StartedAt + // StartedAt should be close to the time the drain started + require.WithinDuration(beforeUpdate, drainStartedAt, 1*time.Second) + + // StartedAt shouldn't change if a new request comes while still draining + require.Nil(msgpackrpc.CallWithCodec(codec, "Node.UpdateDrain", dereg, &resp2)) + ws = memdb.NewWatchSet() + out, err = state.NodeByID(ws, node.ID) + require.NoError(err) + require.True(out.DrainStrategy.StartedAt.Equal(drainStartedAt)) + // Register a system job job := mock.SystemJob() require.Nil(s1.State().UpsertJob(10, job)) @@ -923,8 +934,8 @@ func TestClientEndpoint_UpdateDrain(t *testing.T) { ws = memdb.NewWatchSet() out, err = state.NodeByID(ws, node.ID) require.NoError(err) - require.Len(out.Events, 3) - require.Equal(NodeDrainEventDrainDisabled, out.Events[2].Message) + require.Len(out.Events, 4) + require.Equal(NodeDrainEventDrainDisabled, out.Events[3].Message) // Check that calling UpdateDrain with the same DrainStrategy does not emit // a node event. @@ -932,7 +943,7 @@ func TestClientEndpoint_UpdateDrain(t *testing.T) { ws = memdb.NewWatchSet() out, err = state.NodeByID(ws, node.ID) require.NoError(err) - require.Len(out.Events, 3) + require.Len(out.Events, 4) } func TestClientEndpoint_UpdateDrain_ACL(t *testing.T) { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d96e4cf6e7e..585adec36f4 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1445,6 +1445,9 @@ type DrainStrategy struct { // ForceDeadline is the deadline time for the drain after which drains will // be forced ForceDeadline time.Time + + // StartedAt is the time the drain process started + StartedAt time.Time } func (d *DrainStrategy) Copy() *DrainStrategy {