From 472c7f0452c9ad4489da05cc138d7722233f6875 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Mon, 24 Jun 2019 12:57:19 -0700 Subject: [PATCH 01/10] add timestamps to evals --- api/evaluations.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/evaluations.go b/api/evaluations.go index f8ec9432923..fd9b43e92e8 100644 --- a/api/evaluations.go +++ b/api/evaluations.go @@ -80,6 +80,8 @@ type Evaluation struct { SnapshotIndex uint64 CreateIndex uint64 ModifyIndex uint64 + CreateTime int64 + ModifyTime int64 } // EvalIndexSort is a wrapper to sort evaluations by CreateIndex. From 0a72f11cf05d8adaf11522d969c4a6049dee13f3 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Wed, 3 Jul 2019 11:49:28 -0700 Subject: [PATCH 02/10] add timestamps to eval instances --- nomad/job_endpoint.go | 6 ++++++ nomad/leader.go | 6 ++++++ nomad/mock/mock.go | 2 ++ nomad/node_endpoint.go | 6 ++++++ nomad/periodic.go | 3 +++ nomad/state/state_store.go | 12 ++++++++++++ nomad/structs/structs.go | 3 +++ nomad/worker.go | 6 ++++++ 8 files changed, 44 insertions(+) diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 9648be01cef..7ea92a00e1b 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -217,6 +217,8 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis JobID: args.Job.ID, JobModifyIndex: reply.JobModifyIndex, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, @@ -659,6 +661,8 @@ func (j *Job) Deregister(args *structs.JobDeregisterRequest, reply *structs.JobD JobID: args.JobID, JobModifyIndex: index, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, @@ -1424,6 +1428,8 @@ func (j *Job) Dispatch(args *structs.JobDispatchRequest, reply *structs.JobDispa JobID: dispatchJob.ID, JobModifyIndex: jobCreateIndex, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, diff --git a/nomad/leader.go b/nomad/leader.go index 0b08fc60cc9..09660d48628 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -532,8 +532,12 @@ func (s *Server) reapFailedEvaluations(stopCh chan struct{}) { // due to the fairly large backoff. followupEvalWait := s.config.EvalFailedFollowupBaselineDelay + time.Duration(rand.Int63n(int64(s.config.EvalFailedFollowupDelayRange))) + + // TODO: what is this followup eval and where to create timestamp? followupEval := eval.CreateFailedFollowUpEval(followupEvalWait) updateEval.NextEval = followupEval.ID + updateEval.CreateTime = time.Now().UTC().UnixNano() + updateEval.ModifyTime = time.Now().UTC().UnixNano() // Update via Raft req := structs.EvalUpdateRequest{ @@ -570,6 +574,8 @@ func (s *Server) reapDupBlockedEvaluations(stopCh chan struct{}) { newEval := dup.Copy() newEval.Status = structs.EvalStatusCancelled newEval.StatusDescription = fmt.Sprintf("existing blocked evaluation exists for job %q", newEval.JobID) + newEval.CreateTime = time.Now().UTC().UnixNano() + newEval.ModifyTime = time.Now().UTC().UnixNano() cancel[i] = newEval } diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index d3ccda14596..12add3a1fb5 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -409,6 +409,8 @@ func Eval() *structs.Evaluation { Type: structs.JobTypeService, JobID: uuid.Generate(), Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } return eval } diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index 1fed74bbc28..f50b5c1c170 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -1095,6 +1095,8 @@ func (n *Node) batchUpdate(future *structs.BatchFuture, updates []*structs.Alloc } _, exists := evalsByJobId[namespacedID] if !exists { + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() trimmedEvals = append(trimmedEvals, eval) evalsByJobId[namespacedID] = struct{}{} } @@ -1261,6 +1263,8 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 NodeID: nodeID, NodeModifyIndex: nodeIndex, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } evals = append(evals, eval) evalIDs = append(evalIDs, eval.ID) @@ -1285,6 +1289,8 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 NodeID: nodeID, NodeModifyIndex: nodeIndex, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } evals = append(evals, eval) evalIDs = append(evalIDs, eval.ID) diff --git a/nomad/periodic.go b/nomad/periodic.go index c4178641634..f1f4eda54f1 100644 --- a/nomad/periodic.go +++ b/nomad/periodic.go @@ -72,6 +72,8 @@ func (s *Server) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { JobID: job.ID, JobModifyIndex: index, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, @@ -85,6 +87,7 @@ func (s *Server) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { return nil, err } + // TODO: do the timestamps need to be at the same time that the raft index is updated? // Update its indexes. eval.CreateIndex = evalIndex eval.ModifyIndex = evalIndex diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 09c74286400..4aaeaeeac11 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -303,6 +303,8 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR // Upsert followup evals for allocs that were preempted for _, eval := range results.PreemptionEvals { + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, eval); err != nil { return err } @@ -1724,6 +1726,8 @@ func (s *StateStore) UpsertEvalsTxn(index uint64, evals []*structs.Evaluation, t // Do a nested upsert jobs := make(map[structs.NamespacedID]string, len(evals)) for _, eval := range evals { + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, eval); err != nil { return err } @@ -2313,6 +2317,8 @@ func (s *StateStore) UpdateAllocsDesiredTransitions(index uint64, allocs map[str } for _, eval := range evals { + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, eval); err != nil { return err } @@ -2724,6 +2730,8 @@ func (s *StateStore) UpdateDeploymentStatus(index uint64, req *structs.Deploymen // Upsert the optional eval if req.Eval != nil { + req.Eval.CreateTime = time.Now().UTC().UnixNano() + req.Eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { return err } @@ -2940,6 +2948,8 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD // Upsert the optional eval if req.Eval != nil { + req.Eval.CreateTime = time.Now().UTC().UnixNano() + req.Eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { return err } @@ -3054,6 +3064,8 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl // Upsert the optional eval if req.Eval != nil { + req.Eval.CreateTime = time.Now().UTC().UnixNano() + req.Eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { return err } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index dee93eee733..849efab3d7e 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8394,6 +8394,9 @@ type Evaluation struct { // Raft Indexes CreateIndex uint64 ModifyIndex uint64 + + CreateTime int64 + ModifyTime int64 } // TerminalStatus returns if the current status is terminal and diff --git a/nomad/worker.go b/nomad/worker.go index 192a078d5dc..f4d83079ade 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -349,6 +349,8 @@ func (w *Worker) UpdateEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() // Setup the request req := structs.EvalUpdateRequest{ @@ -386,6 +388,8 @@ func (w *Worker) CreateEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() // Setup the request req := structs.EvalUpdateRequest{ @@ -447,6 +451,8 @@ func (w *Worker) ReblockEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex + eval.CreateTime = time.Now().UTC().UnixNano() + eval.ModifyTime = time.Now().UTC().UnixNano() // Setup the request req := structs.EvalUpdateRequest{ From bd0dcad0599e7ad94b0ab10ed8f264a5b67fde00 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Thu, 11 Jul 2019 09:00:14 -0700 Subject: [PATCH 03/10] add more eval timestamps --- nomad/alloc_endpoint.go | 2 ++ nomad/deploymentwatcher/deployment_watcher.go | 2 ++ nomad/drainer/drainer.go | 2 ++ nomad/job_endpoint.go | 7 +++++++ nomad/leader.go | 1 - nomad/node_endpoint.go | 10 ++++++---- nomad/plan_apply.go | 2 ++ nomad/state/state_store.go | 12 ------------ 8 files changed, 21 insertions(+), 17 deletions(-) diff --git a/nomad/alloc_endpoint.go b/nomad/alloc_endpoint.go index 995ce874aa2..138bc0418e9 100644 --- a/nomad/alloc_endpoint.go +++ b/nomad/alloc_endpoint.go @@ -244,6 +244,8 @@ func (a *Alloc) Stop(args *structs.AllocStopRequest, reply *structs.AllocStopRes JobID: alloc.Job.ID, JobModifyIndex: alloc.Job.ModifyIndex, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } transitionReq := &structs.AllocUpdateDesiredTransitionRequest{ diff --git a/nomad/deploymentwatcher/deployment_watcher.go b/nomad/deploymentwatcher/deployment_watcher.go index e41fa2054af..e92345ff305 100644 --- a/nomad/deploymentwatcher/deployment_watcher.go +++ b/nomad/deploymentwatcher/deployment_watcher.go @@ -800,6 +800,8 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation { JobID: w.j.ID, DeploymentID: w.deploymentID, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } } diff --git a/nomad/drainer/drainer.go b/nomad/drainer/drainer.go index 84ae154a541..f41bf1c85d2 100644 --- a/nomad/drainer/drainer.go +++ b/nomad/drainer/drainer.go @@ -411,6 +411,8 @@ func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs TriggeredBy: structs.EvalTriggerNodeDrain, JobID: job, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), }) } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 7ea92a00e1b..c9d7ea0840f 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -582,6 +582,8 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis JobID: job.ID, JobModifyIndex: job.ModifyIndex, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } // Create a AllocUpdateDesiredTransitionRequest request with the eval and any forced rescheduled allocs @@ -750,6 +752,8 @@ func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *st TriggeredBy: structs.EvalTriggerJobDeregister, JobID: jobNS.ID, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } args.Evals = append(args.Evals, eval) } @@ -1209,6 +1213,9 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) JobModifyIndex: updatedIndex, Status: structs.EvalStatusPending, AnnotatePlan: true, + // Timestamps are added for consistency but this eval is never persisted + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } snap.UpsertEvals(100, []*structs.Evaluation{eval}) diff --git a/nomad/leader.go b/nomad/leader.go index 09660d48628..ba61fbbf914 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -574,7 +574,6 @@ func (s *Server) reapDupBlockedEvaluations(stopCh chan struct{}) { newEval := dup.Copy() newEval.Status = structs.EvalStatusCancelled newEval.StatusDescription = fmt.Sprintf("existing blocked evaluation exists for job %q", newEval.JobID) - newEval.CreateTime = time.Now().UTC().UnixNano() newEval.ModifyTime = time.Now().UTC().UnixNano() cancel[i] = newEval } diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index f50b5c1c170..c7dfbd81626 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -1038,6 +1038,8 @@ func (n *Node) UpdateAlloc(args *structs.AllocUpdateRequest, reply *structs.Gene Type: job.Type, Priority: job.Priority, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } evals = append(evals, eval) } @@ -1263,8 +1265,8 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 NodeID: nodeID, NodeModifyIndex: nodeIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } evals = append(evals, eval) evalIDs = append(evalIDs, eval.ID) @@ -1289,8 +1291,8 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 NodeID: nodeID, NodeModifyIndex: nodeIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } evals = append(evals, eval) evalIDs = append(evalIDs, eval.ID) diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index 92873d0d52b..f376d786466 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -240,6 +240,8 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap Type: job.Type, Priority: job.Priority, Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } evals = append(evals, eval) } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 4aaeaeeac11..09c74286400 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -303,8 +303,6 @@ func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanR // Upsert followup evals for allocs that were preempted for _, eval := range results.PreemptionEvals { - eval.CreateTime = time.Now().UTC().UnixNano() - eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, eval); err != nil { return err } @@ -1726,8 +1724,6 @@ func (s *StateStore) UpsertEvalsTxn(index uint64, evals []*structs.Evaluation, t // Do a nested upsert jobs := make(map[structs.NamespacedID]string, len(evals)) for _, eval := range evals { - eval.CreateTime = time.Now().UTC().UnixNano() - eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, eval); err != nil { return err } @@ -2317,8 +2313,6 @@ func (s *StateStore) UpdateAllocsDesiredTransitions(index uint64, allocs map[str } for _, eval := range evals { - eval.CreateTime = time.Now().UTC().UnixNano() - eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, eval); err != nil { return err } @@ -2730,8 +2724,6 @@ func (s *StateStore) UpdateDeploymentStatus(index uint64, req *structs.Deploymen // Upsert the optional eval if req.Eval != nil { - req.Eval.CreateTime = time.Now().UTC().UnixNano() - req.Eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { return err } @@ -2948,8 +2940,6 @@ func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyD // Upsert the optional eval if req.Eval != nil { - req.Eval.CreateTime = time.Now().UTC().UnixNano() - req.Eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { return err } @@ -3064,8 +3054,6 @@ func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.Appl // Upsert the optional eval if req.Eval != nil { - req.Eval.CreateTime = time.Now().UTC().UnixNano() - req.Eval.ModifyTime = time.Now().UTC().UnixNano() if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { return err } From a8d464cd99899489dff2fcfa2f3549f0909dde71 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Mon, 22 Jul 2019 16:39:47 -0700 Subject: [PATCH 04/10] refactor for code review: adjust correct places to update/create timestamps for evals --- nomad/leader.go | 2 -- nomad/periodic.go | 1 - nomad/structs/structs.go | 7 +++++++ nomad/worker.go | 2 -- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nomad/leader.go b/nomad/leader.go index ba61fbbf914..e28918027e8 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -533,10 +533,8 @@ func (s *Server) reapFailedEvaluations(stopCh chan struct{}) { followupEvalWait := s.config.EvalFailedFollowupBaselineDelay + time.Duration(rand.Int63n(int64(s.config.EvalFailedFollowupDelayRange))) - // TODO: what is this followup eval and where to create timestamp? followupEval := eval.CreateFailedFollowUpEval(followupEvalWait) updateEval.NextEval = followupEval.ID - updateEval.CreateTime = time.Now().UTC().UnixNano() updateEval.ModifyTime = time.Now().UTC().UnixNano() // Update via Raft diff --git a/nomad/periodic.go b/nomad/periodic.go index f1f4eda54f1..5abe2a1bdb2 100644 --- a/nomad/periodic.go +++ b/nomad/periodic.go @@ -87,7 +87,6 @@ func (s *Server) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { return nil, err } - // TODO: do the timestamps need to be at the same time that the raft index is updated? // Update its indexes. eval.CreateIndex = evalIndex eval.ModifyIndex = evalIndex diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 849efab3d7e..6bd5775773b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8507,6 +8507,9 @@ func (e *Evaluation) NextRollingEval(wait time.Duration) *Evaluation { Status: EvalStatusPending, Wait: wait, PreviousEval: e.ID, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), + // TODO(@jasmine): is a NextRollingEval technically created now or when original eval was created? } } @@ -8530,6 +8533,8 @@ func (e *Evaluation) CreateBlockedEval(classEligibility map[string]bool, ClassEligibility: classEligibility, EscapedComputedClass: escaped, QuotaLimitReached: quotaReached, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } } @@ -8549,6 +8554,8 @@ func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { Status: EvalStatusPending, Wait: wait, PreviousEval: e.ID, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } } diff --git a/nomad/worker.go b/nomad/worker.go index f4d83079ade..c18b9e405a9 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -349,7 +349,6 @@ func (w *Worker) UpdateEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex - eval.CreateTime = time.Now().UTC().UnixNano() eval.ModifyTime = time.Now().UTC().UnixNano() // Setup the request @@ -451,7 +450,6 @@ func (w *Worker) ReblockEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex - eval.CreateTime = time.Now().UTC().UnixNano() eval.ModifyTime = time.Now().UTC().UnixNano() // Setup the request From 9a384b8a206ba20c96ce1abf1838970723198e9e Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Wed, 24 Jul 2019 12:20:05 -0700 Subject: [PATCH 05/10] tests --- nomad/job_endpoint_test.go | 36 ++++++++++++++++++++++++++++++++++++ nomad/node_endpoint_test.go | 12 ++++++++++++ 2 files changed, 48 insertions(+) diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index a5204bf1b91..c98084587cd 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -96,6 +96,12 @@ func TestJobEndpoint_Register(t *testing.T) { if eval.Status != structs.EvalStatusPending { t.Fatalf("bad: %#v", eval) } + if eval.CreateTime == 0 { + t.Fatalf("eval CreateTime is unset: %#v", eval) + } + if eval.ModifyTime == 0 { + t.Fatalf("eval ModifyTime is unset: %#v", eval) + } } func TestJobEndpoint_Register_ACL(t *testing.T) { @@ -302,6 +308,12 @@ func TestJobEndpoint_Register_Existing(t *testing.T) { if eval.Status != structs.EvalStatusPending { t.Fatalf("bad: %#v", eval) } + if eval.CreateTime == 0 { + t.Fatalf("eval CreateTime is unset: %#v", eval) + } + if eval.ModifyTime == 0 { + t.Fatalf("eval ModifyTime is unset: %#v", eval) + } if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { t.Fatalf("err: %v", err) @@ -1500,6 +1512,12 @@ func TestJobEndpoint_Evaluate(t *testing.T) { if eval.Status != structs.EvalStatusPending { t.Fatalf("bad: %#v", eval) } + if eval.CreateTime == 0 { + t.Fatalf("eval CreateTime is unset: %#v", eval) + } + if eval.ModifyTime == 0 { + t.Fatalf("eval ModifyTime is unset: %#v", eval) + } } func TestJobEndpoint_ForceRescheduleEvaluate(t *testing.T) { @@ -1569,6 +1587,8 @@ func TestJobEndpoint_ForceRescheduleEvaluate(t *testing.T) { require.Equal(eval.JobID, job.ID) require.Equal(eval.JobModifyIndex, resp.JobModifyIndex) require.Equal(eval.Status, structs.EvalStatusPending) + require.NotZero(eval.CreateTime) + require.NotZero(eval.ModifyTime) // Lookup the alloc, verify DesiredTransition ForceReschedule alloc, err = state.AllocByID(ws, alloc.ID) @@ -1647,6 +1667,8 @@ func TestJobEndpoint_Evaluate_ACL(t *testing.T) { require.Equal(eval.JobID, job.ID) require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) require.Equal(eval.Status, structs.EvalStatusPending) + require.NotZero(eval.CreateTime) + require.NotZero(eval.ModifyTime) } func TestJobEndpoint_Evaluate_Periodic(t *testing.T) { @@ -1790,6 +1812,8 @@ func TestJobEndpoint_Deregister(t *testing.T) { require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) require.Equal(job.ID, eval.JobID) require.Equal(structs.EvalStatusPending, eval.Status) + require.NotZero(eval.CreateTime) + require.NotZero(eval.ModifyTime) // Deregister and purge dereg2 := &structs.JobDeregisterRequest{ @@ -1820,6 +1844,8 @@ func TestJobEndpoint_Deregister(t *testing.T) { require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) require.Equal(job.ID, eval.JobID) require.Equal(structs.EvalStatusPending, eval.Status) + require.NotZero(eval.CreateTime) + require.NotZero(eval.ModifyTime) } func TestJobEndpoint_Deregister_ACL(t *testing.T) { @@ -1899,6 +1925,8 @@ func TestJobEndpoint_Deregister_ACL(t *testing.T) { require.Equal(eval.JobID, job.ID) require.Equal(eval.JobModifyIndex, validResp2.JobModifyIndex) require.Equal(eval.Status, structs.EvalStatusPending) + require.NotZero(eval.CreateTime) + require.NotZero(eval.ModifyTime) } func TestJobEndpoint_Deregister_Nonexistent(t *testing.T) { @@ -1959,6 +1987,12 @@ func TestJobEndpoint_Deregister_Nonexistent(t *testing.T) { if eval.Status != structs.EvalStatusPending { t.Fatalf("bad: %#v", eval) } + if eval.CreateTime == 0 { + t.Fatalf("eval CreateTime is unset: %#v", eval) + } + if eval.ModifyTime == 0 { + t.Fatalf("eval ModifyTime is unset: %#v", eval) + } } func TestJobEndpoint_Deregister_Periodic(t *testing.T) { @@ -2165,6 +2199,8 @@ func TestJobEndpoint_BatchDeregister(t *testing.T) { require.Equal(structs.EvalTriggerJobDeregister, eval.TriggeredBy) require.Equal(expectedJob.ID, eval.JobID) require.Equal(structs.EvalStatusPending, eval.Status) + require.NotZero(eval.CreateTime) + require.NotZero(eval.ModifyTime) } } diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 6b7b90e2f40..11d0a605498 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -2351,6 +2351,12 @@ func TestClientEndpoint_CreateNodeEvals(t *testing.T) { if eval.JobID != expJobID { t.Fatalf("JobID incorrect on type %v: %#v", schedType, eval) } + if eval.CreateTime == 0 { + t.Fatalf("CreateTime is unset on type %v: %#v", schedType, eval) + } + if eval.ModifyTime == 0 { + t.Fatalf("ModifyTime is unset on type %v: %#v", schedType, eval) + } } } @@ -2433,6 +2439,12 @@ func TestClientEndpoint_Evaluate(t *testing.T) { if eval.Status != structs.EvalStatusPending { t.Fatalf("bad: %#v", eval) } + if eval.CreateTime == 0 { + t.Fatalf("CreateTime is unset: %#v", eval) + } + if eval.ModifyTime == 0 { + t.Fatalf("ModifyTime is unset: %#v", eval) + } } func TestClientEndpoint_Evaluate_ACL(t *testing.T) { From 56e60d3d64be78f26b6e6ca2dc5c083a8f666e53 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Wed, 24 Jul 2019 12:21:02 -0700 Subject: [PATCH 06/10] linting fixes --- api/evaluations.go | 4 ++-- nomad/job_endpoint.go | 4 ++-- nomad/mock/mock.go | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/evaluations.go b/api/evaluations.go index fd9b43e92e8..c8404dfc4c6 100644 --- a/api/evaluations.go +++ b/api/evaluations.go @@ -80,8 +80,8 @@ type Evaluation struct { SnapshotIndex uint64 CreateIndex uint64 ModifyIndex uint64 - CreateTime int64 - ModifyTime int64 + CreateTime int64 + ModifyTime int64 } // EvalIndexSort is a wrapper to sort evaluations by CreateIndex. diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index c9d7ea0840f..2563f7c4205 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1214,8 +1214,8 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) Status: structs.EvalStatusPending, AnnotatePlan: true, // Timestamps are added for consistency but this eval is never persisted - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } snap.UpsertEvals(100, []*structs.Evaluation{eval}) diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 12add3a1fb5..305ae48a83e 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -403,14 +403,14 @@ func PeriodicJob() *structs.Job { func Eval() *structs.Evaluation { eval := &structs.Evaluation{ - ID: uuid.Generate(), - Namespace: structs.DefaultNamespace, - Priority: 50, - Type: structs.JobTypeService, - JobID: uuid.Generate(), - Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + ID: uuid.Generate(), + Namespace: structs.DefaultNamespace, + Priority: 50, + Type: structs.JobTypeService, + JobID: uuid.Generate(), + Status: structs.EvalStatusPending, + CreateTime: time.Now().UTC().UnixNano(), + ModifyTime: time.Now().UTC().UnixNano(), } return eval } From 0ce1aca1b86ee72177ef9180a5bd26b47539d442 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Tue, 6 Aug 2019 09:33:44 -0700 Subject: [PATCH 07/10] add pretty printed timestamps to eval status command --- command/eval_status.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/command/eval_status.go b/command/eval_status.go index 99312fdcbb1..372967abbc1 100644 --- a/command/eval_status.go +++ b/command/eval_status.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strings" + "time" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/api/contexts" @@ -203,9 +204,21 @@ func (c *EvalStatusCommand) Run(args []string) int { statusDesc = eval.Status } + // Format eval timestamps + var formattedCreateTime, formattedModifyTime string + if verbose { + formattedCreateTime = formatUnixNanoTime(eval.CreateTime) + formattedModifyTime = formatUnixNanoTime(eval.ModifyTime) + } else { + formattedCreateTime = prettyTimeDiff(time.Unix(0, eval.CreateTime), time.Now()) + formattedModifyTime = prettyTimeDiff(time.Unix(0, eval.ModifyTime), time.Now()) + } + // Format the evaluation data basic := []string{ fmt.Sprintf("ID|%s", limit(eval.ID, length)), + fmt.Sprintf("Create Time|%s", formattedCreateTime), + fmt.Sprintf("Modify Time|%s", formattedModifyTime), fmt.Sprintf("Status|%s", eval.Status), fmt.Sprintf("Status Description|%s", statusDesc), fmt.Sprintf("Type|%s", eval.Type), From 04b943ce61c8e0fe1970dc1a585be0e87442b293 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Tue, 6 Aug 2019 10:15:52 -0700 Subject: [PATCH 08/10] eval CreateTime and ModifyTime are equal if eval is unmodified --- nomad/alloc_endpoint.go | 5 ++-- nomad/deploymentwatcher/deployment_watcher.go | 5 ++-- nomad/drainer/drainer.go | 5 ++-- nomad/job_endpoint.go | 30 +++++++++++-------- nomad/mock/mock.go | 5 ++-- nomad/node_endpoint.go | 18 ++++++----- nomad/periodic.go | 5 ++-- nomad/plan_apply.go | 4 +-- nomad/structs/structs.go | 16 +++++----- nomad/worker.go | 6 ++-- 10 files changed, 58 insertions(+), 41 deletions(-) diff --git a/nomad/alloc_endpoint.go b/nomad/alloc_endpoint.go index 138bc0418e9..d78642b8633 100644 --- a/nomad/alloc_endpoint.go +++ b/nomad/alloc_endpoint.go @@ -235,6 +235,7 @@ func (a *Alloc) Stop(args *structs.AllocStopRequest, reply *structs.AllocStopRes return fmt.Errorf(structs.ErrUnknownAllocationPrefix) } + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: alloc.Namespace, @@ -244,8 +245,8 @@ func (a *Alloc) Stop(args *structs.AllocStopRequest, reply *structs.AllocStopRes JobID: alloc.Job.ID, JobModifyIndex: alloc.Job.ModifyIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } transitionReq := &structs.AllocUpdateDesiredTransitionRequest{ diff --git a/nomad/deploymentwatcher/deployment_watcher.go b/nomad/deploymentwatcher/deployment_watcher.go index e92345ff305..4c18757cb4d 100644 --- a/nomad/deploymentwatcher/deployment_watcher.go +++ b/nomad/deploymentwatcher/deployment_watcher.go @@ -791,6 +791,7 @@ func (w *deploymentWatcher) createBatchedUpdate(allowReplacements []string, forI // getEval returns an evaluation suitable for the deployment func (w *deploymentWatcher) getEval() *structs.Evaluation { + now := time.Now().UTC().UnixNano() return &structs.Evaluation{ ID: uuid.Generate(), Namespace: w.j.Namespace, @@ -800,8 +801,8 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation { JobID: w.j.ID, DeploymentID: w.deploymentID, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } } diff --git a/nomad/drainer/drainer.go b/nomad/drainer/drainer.go index f41bf1c85d2..c92666c4e0a 100644 --- a/nomad/drainer/drainer.go +++ b/nomad/drainer/drainer.go @@ -402,6 +402,7 @@ func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs } evals := make([]*structs.Evaluation, 0, len(jobs)) + now := time.Now().UTC().UnixNano() for job, alloc := range jobs { evals = append(evals, &structs.Evaluation{ ID: uuid.Generate(), @@ -411,8 +412,8 @@ func (n *NodeDrainer) drainAllocs(future *structs.BatchFuture, allocs []*structs TriggeredBy: structs.EvalTriggerNodeDrain, JobID: job, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, }) } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 2563f7c4205..88bbd478013 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -208,6 +208,7 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis } // Create a new evaluation + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: args.RequestNamespace(), @@ -217,8 +218,8 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis JobID: args.Job.ID, JobModifyIndex: reply.JobModifyIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, @@ -573,6 +574,7 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis } // Create a new evaluation + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: args.RequestNamespace(), @@ -582,8 +584,8 @@ func (j *Job) Evaluate(args *structs.JobEvaluateRequest, reply *structs.JobRegis JobID: job.ID, JobModifyIndex: job.ModifyIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } // Create a AllocUpdateDesiredTransitionRequest request with the eval and any forced rescheduled allocs @@ -654,6 +656,7 @@ func (j *Job) Deregister(args *structs.JobDeregisterRequest, reply *structs.JobD // Create a new evaluation // XXX: The job priority / type is strange for this, since it's not a high // priority even if the job was. + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: args.RequestNamespace(), @@ -663,8 +666,8 @@ func (j *Job) Deregister(args *structs.JobDeregisterRequest, reply *structs.JobD JobID: args.JobID, JobModifyIndex: index, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, @@ -744,6 +747,7 @@ func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *st } // Create a new evaluation + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: jobNS.Namespace, @@ -752,8 +756,8 @@ func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *st TriggeredBy: structs.EvalTriggerJobDeregister, JobID: jobNS.ID, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } args.Evals = append(args.Evals, eval) } @@ -1203,6 +1207,7 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) } // Create an eval and mark it as requiring annotations and insert that as well + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: args.RequestNamespace(), @@ -1214,8 +1219,8 @@ func (j *Job) Plan(args *structs.JobPlanRequest, reply *structs.JobPlanResponse) Status: structs.EvalStatusPending, AnnotatePlan: true, // Timestamps are added for consistency but this eval is never persisted - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } snap.UpsertEvals(100, []*structs.Evaluation{eval}) @@ -1426,6 +1431,7 @@ func (j *Job) Dispatch(args *structs.JobDispatchRequest, reply *structs.JobDispa // If the job is periodic, we don't create an eval. if !dispatchJob.IsPeriodic() { // Create a new evaluation + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: args.RequestNamespace(), @@ -1435,8 +1441,8 @@ func (j *Job) Dispatch(args *structs.JobDispatchRequest, reply *structs.JobDispa JobID: dispatchJob.ID, JobModifyIndex: jobCreateIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 305ae48a83e..1614b5a006c 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -402,6 +402,7 @@ func PeriodicJob() *structs.Job { } func Eval() *structs.Evaluation { + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: structs.DefaultNamespace, @@ -409,8 +410,8 @@ func Eval() *structs.Evaluation { Type: structs.JobTypeService, JobID: uuid.Generate(), Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } return eval } diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index c7dfbd81626..ffcbe323dbd 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -1038,8 +1038,8 @@ func (n *Node) UpdateAlloc(args *structs.AllocUpdateRequest, reply *structs.Gene Type: job.Type, Priority: job.Priority, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now.UTC().UnixNano(), + ModifyTime: now.UTC().UnixNano(), } evals = append(evals, eval) } @@ -1097,8 +1097,9 @@ func (n *Node) batchUpdate(future *structs.BatchFuture, updates []*structs.Alloc } _, exists := evalsByJobId[namespacedID] if !exists { - eval.CreateTime = time.Now().UTC().UnixNano() - eval.ModifyTime = time.Now().UTC().UnixNano() + now := time.Now().UTC().UnixNano() + eval.CreateTime = now + eval.ModifyTime = now trimmedEvals = append(trimmedEvals, eval) evalsByJobId[namespacedID] = struct{}{} } @@ -1246,6 +1247,7 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 var evals []*structs.Evaluation var evalIDs []string jobIDs := make(map[string]struct{}) + now := time.Now().UTC().UnixNano() for _, alloc := range allocs { // Deduplicate on JobID @@ -1265,8 +1267,8 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 NodeID: nodeID, NodeModifyIndex: nodeIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } evals = append(evals, eval) evalIDs = append(evalIDs, eval.ID) @@ -1291,8 +1293,8 @@ func (n *Node) createNodeEvals(nodeID string, nodeIndex uint64) ([]string, uint6 NodeID: nodeID, NodeModifyIndex: nodeIndex, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } evals = append(evals, eval) evalIDs = append(evalIDs, eval.ID) diff --git a/nomad/periodic.go b/nomad/periodic.go index 5abe2a1bdb2..0a64fac7dfa 100644 --- a/nomad/periodic.go +++ b/nomad/periodic.go @@ -63,6 +63,7 @@ func (s *Server) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { } // Create a new evaluation + now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), Namespace: job.Namespace, @@ -72,8 +73,8 @@ func (s *Server) DispatchJob(job *structs.Job) (*structs.Evaluation, error) { JobID: job.ID, JobModifyIndex: index, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } update := &structs.EvalUpdateRequest{ Evals: []*structs.Evaluation{eval}, diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index f376d786466..37beaf2285c 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -240,8 +240,8 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap Type: job.Type, Priority: job.Priority, Status: structs.EvalStatusPending, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } evals = append(evals, eval) } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 6bd5775773b..534b9d9f5af 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8496,6 +8496,7 @@ func (e *Evaluation) MakePlan(j *Job) *Plan { // NextRollingEval creates an evaluation to followup this eval for rolling updates func (e *Evaluation) NextRollingEval(wait time.Duration) *Evaluation { + now := time.Now().UTC().UnixNano() return &Evaluation{ ID: uuid.Generate(), Namespace: e.Namespace, @@ -8507,8 +8508,8 @@ func (e *Evaluation) NextRollingEval(wait time.Duration) *Evaluation { Status: EvalStatusPending, Wait: wait, PreviousEval: e.ID, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, // TODO(@jasmine): is a NextRollingEval technically created now or when original eval was created? } } @@ -8519,7 +8520,7 @@ func (e *Evaluation) NextRollingEval(wait time.Duration) *Evaluation { // quota limit was reached. func (e *Evaluation) CreateBlockedEval(classEligibility map[string]bool, escaped bool, quotaReached string) *Evaluation { - + now := time.Now().UTC().UnixNano() return &Evaluation{ ID: uuid.Generate(), Namespace: e.Namespace, @@ -8533,8 +8534,8 @@ func (e *Evaluation) CreateBlockedEval(classEligibility map[string]bool, ClassEligibility: classEligibility, EscapedComputedClass: escaped, QuotaLimitReached: quotaReached, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } } @@ -8543,6 +8544,7 @@ func (e *Evaluation) CreateBlockedEval(classEligibility map[string]bool, // be retried by the eval_broker. Callers should copy the created eval's ID to // into the old eval's NextEval field. func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { + now := time.Now().UTC().UnixNano() return &Evaluation{ ID: uuid.Generate(), Namespace: e.Namespace, @@ -8554,8 +8556,8 @@ func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { Status: EvalStatusPending, Wait: wait, PreviousEval: e.ID, - CreateTime: time.Now().UTC().UnixNano(), - ModifyTime: time.Now().UTC().UnixNano(), + CreateTime: now, + ModifyTime: now, } } diff --git a/nomad/worker.go b/nomad/worker.go index c18b9e405a9..daf8d49c001 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -387,8 +387,10 @@ func (w *Worker) CreateEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex - eval.CreateTime = time.Now().UTC().UnixNano() - eval.ModifyTime = time.Now().UTC().UnixNano() + + now := time.Now().UTC().UnixNano() + eval.CreateTime = now + eval.ModifyTime = now // Setup the request req := structs.EvalUpdateRequest{ From a405a3adb41ca862f9b9fce151888553f9a86e0f Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Tue, 6 Aug 2019 10:46:52 -0700 Subject: [PATCH 09/10] guarantee eval ModifyTime will always be after CreateTime --- nomad/leader.go | 4 ++-- nomad/structs/structs.go | 12 ++++++++++++ nomad/worker.go | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/nomad/leader.go b/nomad/leader.go index e28918027e8..a6ea8416e66 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -535,7 +535,7 @@ func (s *Server) reapFailedEvaluations(stopCh chan struct{}) { followupEval := eval.CreateFailedFollowUpEval(followupEvalWait) updateEval.NextEval = followupEval.ID - updateEval.ModifyTime = time.Now().UTC().UnixNano() + updateEval.UpdateModifyTime() // Update via Raft req := structs.EvalUpdateRequest{ @@ -572,7 +572,7 @@ func (s *Server) reapDupBlockedEvaluations(stopCh chan struct{}) { newEval := dup.Copy() newEval.Status = structs.EvalStatusCancelled newEval.StatusDescription = fmt.Sprintf("existing blocked evaluation exists for job %q", newEval.JobID) - newEval.ModifyTime = time.Now().UTC().UnixNano() + newEval.UpdateModifyTime() cancel[i] = newEval } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 534b9d9f5af..2bf699b60c6 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8561,6 +8561,18 @@ func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { } } +// UpdateModifyTime takes into account that clocks on different servers may be +// slightly out of sync. Even in case of a leader change, it will guarantee that +// ModifyTime will always be after CreateTime. +func (e *Evaluation) UpdateModifyTime() { + now := time.Now().UTC().UnixNano() + if now <= e.CreateTime { + e.ModifyTime = e.CreateTime + 1 + } else { + e.ModifyTime = now + } +} + // Plan is used to submit a commit plan for task allocations. These // are submitted to the leader which verifies that resources have // not been overcommitted before admitting the plan. diff --git a/nomad/worker.go b/nomad/worker.go index daf8d49c001..240dc4f3d33 100644 --- a/nomad/worker.go +++ b/nomad/worker.go @@ -349,7 +349,7 @@ func (w *Worker) UpdateEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex - eval.ModifyTime = time.Now().UTC().UnixNano() + eval.UpdateModifyTime() // Setup the request req := structs.EvalUpdateRequest{ @@ -452,7 +452,7 @@ func (w *Worker) ReblockEval(eval *structs.Evaluation) error { // Store the snapshot index in the eval eval.SnapshotIndex = w.snapshotIndex - eval.ModifyTime = time.Now().UTC().UnixNano() + eval.UpdateModifyTime() // Setup the request req := structs.EvalUpdateRequest{ From d855aaa97d80e13ef4eff110ca2098f812dcb16d Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Tue, 6 Aug 2019 15:24:56 -0700 Subject: [PATCH 10/10] comment formatting --- nomad/structs/structs.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 2bf699b60c6..102b72d19c9 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8510,7 +8510,6 @@ func (e *Evaluation) NextRollingEval(wait time.Duration) *Evaluation { PreviousEval: e.ID, CreateTime: now, ModifyTime: now, - // TODO(@jasmine): is a NextRollingEval technically created now or when original eval was created? } } @@ -8562,8 +8561,8 @@ func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { } // UpdateModifyTime takes into account that clocks on different servers may be -// slightly out of sync. Even in case of a leader change, it will guarantee that -// ModifyTime will always be after CreateTime. +// slightly out of sync. Even in case of a leader change, this method will +// guarantee that ModifyTime will always be after CreateTime. func (e *Evaluation) UpdateModifyTime() { now := time.Now().UTC().UnixNano() if now <= e.CreateTime {